Add utility functions for database operations

- Introduced a new utils.js file containing various utility functions for database operations.
- Implemented functions for parsing filters, converting objects to camelCase, extracting configuration blocks, and managing audit logs.
- Added functionality to handle ObjectId conversions and filter generation based on query parameters.
- Enhanced object ID handling with functions to flatten and expand ObjectIds for better integration with MongoDB.

src/database/utils.js

@@ -0,0 +1,524 @@
+import { ObjectId } from 'mongodb';
+import { auditLogModel } from './schemas/management/auditlog.schema.js';
+import { etcdServer } from './etcd.js';
+
+function parseFilter(property, value) {
+  if (typeof value === 'string') {
+    var trimmed = value.trim();
+    if (trimmed.charAt(3) == ':') {
+      trimmed = value.split(':')[1];
+    }
+
+    // Handle booleans
+    if (trimmed.toLowerCase() === 'true') return { [property]: true };
+    if (trimmed.toLowerCase() === 'false') return { [property]: false };
+
+    // Handle ObjectId (24-char hex)
+
+    if (/^[a-f\d]{24}$/i.test(trimmed) && trimmed.length >= 24) {
+      return { [property]: new ObjectId(trimmed) };
+    }
+
+    // Handle numbers
+    if (!isNaN(trimmed)) {
+      return { [property]: parseFloat(trimmed) };
+    }
+
+    // Default to case-insensitive regex for non-numeric strings
+    return {
+      [property]: {
+        $regex: trimmed,
+        $options: 'i'
+      }
+    };
+  }
+
+  // Handle actual booleans, numbers, objects, etc.
+  return { [property]: value };
+}
+
+function convertToCamelCase(obj) {
+  const result = {};
+
+  for (const key in obj) {
+    if (Object.prototype.hasOwnProperty.call(obj, key)) {
+      const value = obj[key];
+
+      // Convert the key to camelCase
+      let camelKey = key
+        // First handle special cases with spaces, brackets and other characters
+        .replace(/\s*\[.*?\]\s*/g, '') // Remove brackets and their contents
+        .replace(/\s+/g, ' ') // Normalize spaces
+        .trim()
+        // Split by common separators (space, underscore, hyphen)
+        .split(/[\s_-]/)
+        // Convert to camelCase
+        .map((word, index) => {
+          // Remove any non-alphanumeric characters
+          word = word.replace(/[^a-zA-Z0-9]/g, '');
+
+          // Lowercase first word, uppercase others
+          return index === 0
+            ? word.toLowerCase()
+            : word.charAt(0).toUpperCase() + word.slice(1).toLowerCase();
+        })
+        .join('');
+
+      // Handle values that are objects recursively
+      if (
+        value !== null &&
+        typeof value === 'object' &&
+        !Array.isArray(value)
+      ) {
+        result[camelKey] = convertToCamelCase(value);
+      } else {
+        result[camelKey] = value;
+      }
+    }
+  }
+
+  return result;
+}
+
+function extractConfigBlock(fileContent, useCamelCase = true) {
+  const configObject = {};
+
+  // Extract header information
+  const headerBlockRegex =
+    /; HEADER_BLOCK_START([\s\S]*?)(?:; HEADER_BLOCK_END|$)/;
+  const headerBlockMatch = fileContent.match(headerBlockRegex);
+  if (headerBlockMatch && headerBlockMatch[1]) {
+    const headerLines = headerBlockMatch[1].split('\n');
+    headerLines.forEach(line => {
+      const keyValueRegex = /^\s*;\s*([^:]+?):\s*(.*?)\s*$/;
+      const simpleValueRegex = /^\s*;\s*(.*?)\s*$/;
+
+      // Try key-value format first
+      let match = line.match(keyValueRegex);
+      if (match) {
+        const key = match[1].trim();
+        let value = match[2].trim();
+
+        // Try to convert value to appropriate type
+        if (!isNaN(value) && value !== '') {
+          value = Number(value);
+        }
+        configObject[key] = value;
+      } else {
+        // Try the simple format like "; generated by OrcaSlicer 2.1.1 on 2025-04-28 at 13:30:11"
+        match = line.match(simpleValueRegex);
+        if (match && match[1] && !match[1].includes('HEADER_BLOCK')) {
+          const text = match[1].trim();
+
+          // Extract slicer info
+          const slicerMatch = text.match(
+            /generated by (.*?) on (.*?) at (.*?)$/
+          );
+          if (slicerMatch) {
+            configObject['slicer'] = slicerMatch[1].trim();
+            configObject['date'] = slicerMatch[2].trim();
+            configObject['time'] = slicerMatch[3].trim();
+          } else {
+            // Just add as a general header entry if it doesn't match any specific pattern
+            const key = `header_${Object.keys(configObject).length}`;
+            configObject[key] = text;
+          }
+        }
+      }
+    });
+  }
+
+  // Extract thumbnail data
+  const thumbnailBlockRegex =
+    /; THUMBNAIL_BLOCK_START([\s\S]*?)(?:; THUMBNAIL_BLOCK_END|$)/;
+  const thumbnailBlockMatch = fileContent.match(thumbnailBlockRegex);
+  if (thumbnailBlockMatch && thumbnailBlockMatch[1]) {
+    const thumbnailLines = thumbnailBlockMatch[1].split('\n');
+    let base64Data = '';
+    let thumbnailInfo = {};
+
+    thumbnailLines.forEach(line => {
+      // Extract thumbnail dimensions and size from the line "thumbnail begin 640x640 27540"
+      const thumbnailHeaderRegex = /^\s*;\s*thumbnail begin (\d+)x(\d+) (\d+)/;
+      const match = line.match(thumbnailHeaderRegex);
+
+      if (match) {
+        thumbnailInfo.width = parseInt(match[1], 10);
+        thumbnailInfo.height = parseInt(match[2], 10);
+        thumbnailInfo.size = parseInt(match[3], 10);
+      } else if (
+        line.trim().startsWith('; ') &&
+        !line.includes('THUMBNAIL_BLOCK')
+      ) {
+        // Collect base64 data (remove the leading semicolon and space and thumbnail end)
+        const dataLine = line.trim().substring(2);
+        if (dataLine && dataLine != 'thumbnail end') {
+          base64Data += dataLine;
+        }
+      }
+    });
+
+    // Add thumbnail data to config object
+    if (base64Data) {
+      configObject.thumbnail = {
+        data: base64Data,
+        ...thumbnailInfo
+      };
+    }
+  }
+
+  // Extract CONFIG_BLOCK
+  const configBlockRegex =
+    /; CONFIG_BLOCK_START([\s\S]*?)(?:; CONFIG_BLOCK_END|$)/;
+  const configBlockMatch = fileContent.match(configBlockRegex);
+  if (configBlockMatch && configBlockMatch[1]) {
+    // Extract each config line
+    const configLines = configBlockMatch[1].split('\n');
+    // Process each line
+    configLines.forEach(line => {
+      // Check if the line starts with a semicolon and has an equals sign
+      const configLineRegex = /^\s*;\s*([^=]+?)\s*=\s*(.*?)\s*$/;
+      const match = line.match(configLineRegex);
+      if (match) {
+        const key = match[1].trim();
+        let value = match[2].trim();
+        // Try to convert value to appropriate type
+        if (value === 'true' || value === 'false') {
+          value = value === 'true';
+        } else if (!isNaN(value) && value !== '') {
+          // Check if it's a number (but not a percentage)
+          if (!value.includes('%')) {
+            value = Number(value);
+          }
+        }
+        configObject[key] = value;
+      }
+    });
+  }
+
+  // Extract additional variables that appear after EXECUTABLE_BLOCK_END
+  const additionalVarsRegex =
+    /; EXECUTABLE_BLOCK_(?:START|END)([\s\S]*?)(?:; CONFIG_BLOCK_START|$)/i;
+  const additionalVarsMatch = fileContent.match(additionalVarsRegex);
+  if (additionalVarsMatch && additionalVarsMatch[1]) {
+    const additionalLines = additionalVarsMatch[1].split('\n');
+    additionalLines.forEach(line => {
+      // Match both standard format and the special case for "total filament cost"
+      const varRegex =
+        /^\s*;\s*((?:filament used|filament cost|total filament used|total filament cost|total layers count|estimated printing time)[^=]*?)\s*=\s*(.*?)\s*$/;
+      const match = line.match(varRegex);
+      if (match) {
+        const key = match[1].replace(/\[([^\]]+)\]/g, '$1').trim();
+        let value = match[2].trim();
+        // Clean up values - remove units in brackets and handle special cases
+        if (key.includes('filament used')) {
+          // Extract just the numeric value, ignoring units in brackets
+          const numMatch = value.match(/(\d+\.\d+)/);
+          if (numMatch) {
+            value = parseFloat(numMatch[1]);
+          }
+        } else if (key.includes('filament cost')) {
+          // Extract just the numeric value
+          const numMatch = value.match(/(\d+\.\d+)/);
+          if (numMatch) {
+            value = parseFloat(numMatch[1]);
+          }
+        } else if (key.includes('total layers count')) {
+          value = parseInt(value, 10);
+        } else if (key.includes('estimated printing time')) {
+          // Keep as string but trim any additional whitespace
+          value = value.trim();
+        }
+        configObject[key] = value;
+      }
+    });
+  }
+
+  // Also extract extrusion width settings
+  const extrusionWidthRegex = /;\s*(.*?)\s*extrusion width\s*=\s*(.*?)mm/g;
+  let extrusionMatch;
+  while ((extrusionMatch = extrusionWidthRegex.exec(fileContent)) !== null) {
+    const settingName = extrusionMatch[1].trim();
+    const settingValue = parseFloat(extrusionMatch[2].trim());
+    configObject[`${settingName} extrusion width`] = settingValue;
+  }
+
+  // Extract additional parameters after CONFIG_BLOCK_END if they exist
+  const postConfigParams = /; CONFIG_BLOCK_END\s*\n([\s\S]*?)$/;
+  const postConfigMatch = fileContent.match(postConfigParams);
+  if (postConfigMatch && postConfigMatch[1]) {
+    const postConfigLines = postConfigMatch[1].split('\n');
+    postConfigLines.forEach(line => {
+      // Match lines with format "; parameter_name = value"
+      const paramRegex = /^\s*;\s*([^=]+?)\s*=\s*(.*?)\s*$/;
+      const match = line.match(paramRegex);
+      if (match) {
+        const key = match[1].trim();
+        let value = match[2].trim();
+
+        // Try to convert value to appropriate type
+        if (value === 'true' || value === 'false') {
+          value = value === 'true';
+        } else if (!isNaN(value) && value !== '') {
+          // Check if it's a number (but not a percentage)
+          if (!value.includes('%')) {
+            value = Number(value);
+          }
+        }
+
+        // Add to config object if not already present
+        if (!configObject[key]) {
+          configObject[key] = value;
+        }
+      }
+    });
+  }
+
+  // Apply camelCase conversion if requested
+  return useCamelCase ? convertToCamelCase(configObject) : configObject;
+}
+
+function getChangedValues(oldObj, newObj, old = false) {
+  const changes = {};
+
+  const combinedObj = { ...oldObj, ...newObj };
+
+  // Check all keys in the new object
+  for (const key in combinedObj) {
+    // Skip if the key is _id or timestamps
+    if (key === 'createdAt' || key === 'updatedAt' || key === '_id') continue;
+
+    const oldVal = oldObj ? oldObj[key] : undefined;
+    const newVal = newObj ? newObj[key] : undefined;
+
+    // If both values are objects (but not arrays or null), recurse
+    if (
+      oldVal &&
+      newVal &&
+      typeof oldVal === 'object' &&
+      typeof newVal === 'object' &&
+      !Array.isArray(oldVal) &&
+      !Array.isArray(newVal) &&
+      oldVal !== null &&
+      newVal !== null
+    ) {
+      if (oldVal?._id || newVal?._id) {
+        if (JSON.stringify(oldVal?._id) !== JSON.stringify(newVal?._id)) {
+          changes[key] = old ? oldVal : newVal;
+        }
+      } else {
+        const nestedChanges = getChangedValues(oldVal, newVal, old);
+        if (Object.keys(nestedChanges).length > 0) {
+          changes[key] = nestedChanges;
+        }
+      }
+    } else if (JSON.stringify(oldVal) !== JSON.stringify(newVal)) {
+      // If the old value is different from the new value, include it
+      changes[key] = old ? oldVal : newVal;
+    }
+  }
+
+  return changes;
+}
+
+async function newAuditLog(newValue, parentId, parentType, owner, ownerType) {
+  // Filter out createdAt and updatedAt from newValue
+  const filteredNewValue = { ...newValue };
+  delete filteredNewValue.createdAt;
+  delete filteredNewValue.updatedAt;
+  const auditLog = new auditLogModel({
+    changes: {
+      new: filteredNewValue
+    },
+    parent: parentId,
+    parentType,
+    owner: owner._id,
+    ownerType: ownerType,
+    operation: 'new'
+  });
+
+  await auditLog.save();
+
+  await distributeNew(auditLog._id, 'auditLog');
+}
+
+async function editAuditLog(
+  oldValue,
+  newValue,
+  parentId,
+  parentType,
+  owner,
+  ownerType
+) {
+  // Get only the changed values
+  const changedOldValues = getChangedValues(oldValue, newValue, true);
+  const changedNewValues = getChangedValues(oldValue, newValue, false);
+
+  // If no values changed, don't create an audit log
+  if (
+    Object.keys(changedOldValues).length === 0 ||
+    Object.keys(changedNewValues).length === 0
+  ) {
+    return;
+  }
+
+  const auditLog = new auditLogModel({
+    changes: {
+      old: changedOldValues,
+      new: changedNewValues
+    },
+    parent: parentId,
+    parentType,
+    owner: owner._id,
+    ownerType: ownerType,
+    operation: 'edit'
+  });
+
+  await auditLog.save();
+
+  await distributeNew(auditLog._id, 'auditLog');
+}
+
+async function deleteAuditLog(
+  deleteValue,
+  parentId,
+  parentType,
+  owner,
+  ownerType
+) {
+  const auditLog = new auditLogModel({
+    changes: {
+      old: deleteValue
+    },
+    parent: parentId,
+    parentType,
+    owner: owner._id,
+    ownerType: ownerType,
+    operation: 'delete'
+  });
+
+  await auditLog.save();
+
+  await distributeNew(auditLog._id, 'auditLog');
+}
+
+async function getAuditLogs(idOrIds) {
+  if (Array.isArray(idOrIds)) {
+    return auditLogModel.find({ parent: { $in: idOrIds } }).populate('owner');
+  } else {
+    return auditLogModel.find({ parent: idOrIds }).populate('owner');
+  }
+}
+
+async function distributeUpdate(value, id, type) {
+  await etcdServer.setKey(`/${type}s/${id}/object`, value);
+}
+
+async function distributeNew(id, type) {
+  await etcdServer.setKey(`/${type}s/new`, id);
+}
+
+function flatternObjectIds(object) {
+  if (!object || typeof object !== 'object') {
+    return object;
+  }
+
+  const result = {};
+
+  for (const [key, value] of Object.entries(object)) {
+    if (value && typeof value === 'object' && value._id) {
+      // If the value is an object with _id, convert to just the _id
+      result[key] = value._id;
+    } else {
+      // Keep primitive values as is
+      result[key] = value;
+    }
+  }
+
+  return result;
+}
+
+function expandObjectIds(input) {
+  // Helper to check if a value is an ObjectId or a 24-char hex string
+  function isObjectId(val) {
+    // Check for MongoDB ObjectId instance
+    if (val instanceof ObjectId) return true;
+    // Check for exactly 24 hex characters (no special characters)
+    if (typeof val === 'string' && /^[a-fA-F\d]{24}$/.test(val)) return true;
+    return false;
+  }
+
+  // Recursive function
+  function expand(value) {
+    if (Array.isArray(value)) {
+      return value.map(expand);
+    } else if (
+      value &&
+      typeof value === 'object' &&
+      !(value instanceof ObjectId)
+    ) {
+      var result = {};
+      for (const [key, val] of Object.entries(value)) {
+        if (key === '_id') {
+          // Do not expand keys that are already named _id
+          result[key] = val;
+        } else if (isObjectId(val)) {
+          result[key] = { _id: val };
+        } else if (Array.isArray(val)) {
+          result[key] = val.map(expand);
+        } else if (val instanceof Date) {
+          result[key] = val;
+        } else if (val && typeof val === 'object') {
+          result[key] = expand(val);
+        } else {
+          result[key] = val;
+        }
+      }
+      return result;
+    } else if (isObjectId(value)) {
+      return { _id: value };
+    } else {
+      return value;
+    }
+  }
+
+  return expand(input);
+}
+
+// Returns a filter object based on allowed filters and req.query
+function getFilter(query, allowedFilters, parse = true) {
+  let filter = {};
+  for (const [key, value] of Object.entries(query)) {
+    if (allowedFilters.includes(key)) {
+      const parsedFilter = parse ? parseFilter(key, value) : { [key]: value };
+      filter = { ...filter, ...parsedFilter };
+    }
+  }
+  return filter;
+}
+
+// Converts a properties argument (string or array) to an array of strings
+function convertPropertiesString(properties) {
+  if (typeof properties === 'string') {
+    return properties.split(',');
+  } else if (!Array.isArray(properties)) {
+    return [];
+  }
+  return properties;
+}
+
+export {
+  parseFilter,
+  convertToCamelCase,
+  extractConfigBlock,
+  newAuditLog,
+  editAuditLog,
+  deleteAuditLog,
+  getAuditLogs,
+  flatternObjectIds,
+  expandObjectIds,
+  distributeUpdate,
+  distributeNew,
+  getFilter, // <-- add here
+  convertPropertiesString
+};