From 6cc2a07ee08174185864a02fbdf56375f3afd9a7 Mon Sep 17 00:00:00 2001 From: Tom Butcher Date: Mon, 18 Aug 2025 01:09:14 +0100 Subject: [PATCH] Add utility functions for database operations - Introduced a new utils.js file containing various utility functions for database operations. - Implemented functions for parsing filters, converting objects to camelCase, extracting configuration blocks, and managing audit logs. - Added functionality to handle ObjectId conversions and filter generation based on query parameters. - Enhanced object ID handling with functions to flatten and expand ObjectIds for better integration with MongoDB. --- src/database/utils.js | 524 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 524 insertions(+) create mode 100644 src/database/utils.js diff --git a/src/database/utils.js b/src/database/utils.js new file mode 100644 index 0000000..c803162 --- /dev/null +++ b/src/database/utils.js @@ -0,0 +1,524 @@ +import { ObjectId } from 'mongodb'; +import { auditLogModel } from './schemas/management/auditlog.schema.js'; +import { etcdServer } from './etcd.js'; + +function parseFilter(property, value) { + if (typeof value === 'string') { + var trimmed = value.trim(); + if (trimmed.charAt(3) == ':') { + trimmed = value.split(':')[1]; + } + + // Handle booleans + if (trimmed.toLowerCase() === 'true') return { [property]: true }; + if (trimmed.toLowerCase() === 'false') return { [property]: false }; + + // Handle ObjectId (24-char hex) + + if (/^[a-f\d]{24}$/i.test(trimmed) && trimmed.length >= 24) { + return { [property]: new ObjectId(trimmed) }; + } + + // Handle numbers + if (!isNaN(trimmed)) { + return { [property]: parseFloat(trimmed) }; + } + + // Default to case-insensitive regex for non-numeric strings + return { + [property]: { + $regex: trimmed, + $options: 'i' + } + }; + } + + // Handle actual booleans, numbers, objects, etc. + return { [property]: value }; +} + +function convertToCamelCase(obj) { + const result = {}; + + for (const key in obj) { + if (Object.prototype.hasOwnProperty.call(obj, key)) { + const value = obj[key]; + + // Convert the key to camelCase + let camelKey = key + // First handle special cases with spaces, brackets and other characters + .replace(/\s*\[.*?\]\s*/g, '') // Remove brackets and their contents + .replace(/\s+/g, ' ') // Normalize spaces + .trim() + // Split by common separators (space, underscore, hyphen) + .split(/[\s_-]/) + // Convert to camelCase + .map((word, index) => { + // Remove any non-alphanumeric characters + word = word.replace(/[^a-zA-Z0-9]/g, ''); + + // Lowercase first word, uppercase others + return index === 0 + ? word.toLowerCase() + : word.charAt(0).toUpperCase() + word.slice(1).toLowerCase(); + }) + .join(''); + + // Handle values that are objects recursively + if ( + value !== null && + typeof value === 'object' && + !Array.isArray(value) + ) { + result[camelKey] = convertToCamelCase(value); + } else { + result[camelKey] = value; + } + } + } + + return result; +} + +function extractConfigBlock(fileContent, useCamelCase = true) { + const configObject = {}; + + // Extract header information + const headerBlockRegex = + /; HEADER_BLOCK_START([\s\S]*?)(?:; HEADER_BLOCK_END|$)/; + const headerBlockMatch = fileContent.match(headerBlockRegex); + if (headerBlockMatch && headerBlockMatch[1]) { + const headerLines = headerBlockMatch[1].split('\n'); + headerLines.forEach(line => { + const keyValueRegex = /^\s*;\s*([^:]+?):\s*(.*?)\s*$/; + const simpleValueRegex = /^\s*;\s*(.*?)\s*$/; + + // Try key-value format first + let match = line.match(keyValueRegex); + if (match) { + const key = match[1].trim(); + let value = match[2].trim(); + + // Try to convert value to appropriate type + if (!isNaN(value) && value !== '') { + value = Number(value); + } + configObject[key] = value; + } else { + // Try the simple format like "; generated by OrcaSlicer 2.1.1 on 2025-04-28 at 13:30:11" + match = line.match(simpleValueRegex); + if (match && match[1] && !match[1].includes('HEADER_BLOCK')) { + const text = match[1].trim(); + + // Extract slicer info + const slicerMatch = text.match( + /generated by (.*?) on (.*?) at (.*?)$/ + ); + if (slicerMatch) { + configObject['slicer'] = slicerMatch[1].trim(); + configObject['date'] = slicerMatch[2].trim(); + configObject['time'] = slicerMatch[3].trim(); + } else { + // Just add as a general header entry if it doesn't match any specific pattern + const key = `header_${Object.keys(configObject).length}`; + configObject[key] = text; + } + } + } + }); + } + + // Extract thumbnail data + const thumbnailBlockRegex = + /; THUMBNAIL_BLOCK_START([\s\S]*?)(?:; THUMBNAIL_BLOCK_END|$)/; + const thumbnailBlockMatch = fileContent.match(thumbnailBlockRegex); + if (thumbnailBlockMatch && thumbnailBlockMatch[1]) { + const thumbnailLines = thumbnailBlockMatch[1].split('\n'); + let base64Data = ''; + let thumbnailInfo = {}; + + thumbnailLines.forEach(line => { + // Extract thumbnail dimensions and size from the line "thumbnail begin 640x640 27540" + const thumbnailHeaderRegex = /^\s*;\s*thumbnail begin (\d+)x(\d+) (\d+)/; + const match = line.match(thumbnailHeaderRegex); + + if (match) { + thumbnailInfo.width = parseInt(match[1], 10); + thumbnailInfo.height = parseInt(match[2], 10); + thumbnailInfo.size = parseInt(match[3], 10); + } else if ( + line.trim().startsWith('; ') && + !line.includes('THUMBNAIL_BLOCK') + ) { + // Collect base64 data (remove the leading semicolon and space and thumbnail end) + const dataLine = line.trim().substring(2); + if (dataLine && dataLine != 'thumbnail end') { + base64Data += dataLine; + } + } + }); + + // Add thumbnail data to config object + if (base64Data) { + configObject.thumbnail = { + data: base64Data, + ...thumbnailInfo + }; + } + } + + // Extract CONFIG_BLOCK + const configBlockRegex = + /; CONFIG_BLOCK_START([\s\S]*?)(?:; CONFIG_BLOCK_END|$)/; + const configBlockMatch = fileContent.match(configBlockRegex); + if (configBlockMatch && configBlockMatch[1]) { + // Extract each config line + const configLines = configBlockMatch[1].split('\n'); + // Process each line + configLines.forEach(line => { + // Check if the line starts with a semicolon and has an equals sign + const configLineRegex = /^\s*;\s*([^=]+?)\s*=\s*(.*?)\s*$/; + const match = line.match(configLineRegex); + if (match) { + const key = match[1].trim(); + let value = match[2].trim(); + // Try to convert value to appropriate type + if (value === 'true' || value === 'false') { + value = value === 'true'; + } else if (!isNaN(value) && value !== '') { + // Check if it's a number (but not a percentage) + if (!value.includes('%')) { + value = Number(value); + } + } + configObject[key] = value; + } + }); + } + + // Extract additional variables that appear after EXECUTABLE_BLOCK_END + const additionalVarsRegex = + /; EXECUTABLE_BLOCK_(?:START|END)([\s\S]*?)(?:; CONFIG_BLOCK_START|$)/i; + const additionalVarsMatch = fileContent.match(additionalVarsRegex); + if (additionalVarsMatch && additionalVarsMatch[1]) { + const additionalLines = additionalVarsMatch[1].split('\n'); + additionalLines.forEach(line => { + // Match both standard format and the special case for "total filament cost" + const varRegex = + /^\s*;\s*((?:filament used|filament cost|total filament used|total filament cost|total layers count|estimated printing time)[^=]*?)\s*=\s*(.*?)\s*$/; + const match = line.match(varRegex); + if (match) { + const key = match[1].replace(/\[([^\]]+)\]/g, '$1').trim(); + let value = match[2].trim(); + // Clean up values - remove units in brackets and handle special cases + if (key.includes('filament used')) { + // Extract just the numeric value, ignoring units in brackets + const numMatch = value.match(/(\d+\.\d+)/); + if (numMatch) { + value = parseFloat(numMatch[1]); + } + } else if (key.includes('filament cost')) { + // Extract just the numeric value + const numMatch = value.match(/(\d+\.\d+)/); + if (numMatch) { + value = parseFloat(numMatch[1]); + } + } else if (key.includes('total layers count')) { + value = parseInt(value, 10); + } else if (key.includes('estimated printing time')) { + // Keep as string but trim any additional whitespace + value = value.trim(); + } + configObject[key] = value; + } + }); + } + + // Also extract extrusion width settings + const extrusionWidthRegex = /;\s*(.*?)\s*extrusion width\s*=\s*(.*?)mm/g; + let extrusionMatch; + while ((extrusionMatch = extrusionWidthRegex.exec(fileContent)) !== null) { + const settingName = extrusionMatch[1].trim(); + const settingValue = parseFloat(extrusionMatch[2].trim()); + configObject[`${settingName} extrusion width`] = settingValue; + } + + // Extract additional parameters after CONFIG_BLOCK_END if they exist + const postConfigParams = /; CONFIG_BLOCK_END\s*\n([\s\S]*?)$/; + const postConfigMatch = fileContent.match(postConfigParams); + if (postConfigMatch && postConfigMatch[1]) { + const postConfigLines = postConfigMatch[1].split('\n'); + postConfigLines.forEach(line => { + // Match lines with format "; parameter_name = value" + const paramRegex = /^\s*;\s*([^=]+?)\s*=\s*(.*?)\s*$/; + const match = line.match(paramRegex); + if (match) { + const key = match[1].trim(); + let value = match[2].trim(); + + // Try to convert value to appropriate type + if (value === 'true' || value === 'false') { + value = value === 'true'; + } else if (!isNaN(value) && value !== '') { + // Check if it's a number (but not a percentage) + if (!value.includes('%')) { + value = Number(value); + } + } + + // Add to config object if not already present + if (!configObject[key]) { + configObject[key] = value; + } + } + }); + } + + // Apply camelCase conversion if requested + return useCamelCase ? convertToCamelCase(configObject) : configObject; +} + +function getChangedValues(oldObj, newObj, old = false) { + const changes = {}; + + const combinedObj = { ...oldObj, ...newObj }; + + // Check all keys in the new object + for (const key in combinedObj) { + // Skip if the key is _id or timestamps + if (key === 'createdAt' || key === 'updatedAt' || key === '_id') continue; + + const oldVal = oldObj ? oldObj[key] : undefined; + const newVal = newObj ? newObj[key] : undefined; + + // If both values are objects (but not arrays or null), recurse + if ( + oldVal && + newVal && + typeof oldVal === 'object' && + typeof newVal === 'object' && + !Array.isArray(oldVal) && + !Array.isArray(newVal) && + oldVal !== null && + newVal !== null + ) { + if (oldVal?._id || newVal?._id) { + if (JSON.stringify(oldVal?._id) !== JSON.stringify(newVal?._id)) { + changes[key] = old ? oldVal : newVal; + } + } else { + const nestedChanges = getChangedValues(oldVal, newVal, old); + if (Object.keys(nestedChanges).length > 0) { + changes[key] = nestedChanges; + } + } + } else if (JSON.stringify(oldVal) !== JSON.stringify(newVal)) { + // If the old value is different from the new value, include it + changes[key] = old ? oldVal : newVal; + } + } + + return changes; +} + +async function newAuditLog(newValue, parentId, parentType, owner, ownerType) { + // Filter out createdAt and updatedAt from newValue + const filteredNewValue = { ...newValue }; + delete filteredNewValue.createdAt; + delete filteredNewValue.updatedAt; + const auditLog = new auditLogModel({ + changes: { + new: filteredNewValue + }, + parent: parentId, + parentType, + owner: owner._id, + ownerType: ownerType, + operation: 'new' + }); + + await auditLog.save(); + + await distributeNew(auditLog._id, 'auditLog'); +} + +async function editAuditLog( + oldValue, + newValue, + parentId, + parentType, + owner, + ownerType +) { + // Get only the changed values + const changedOldValues = getChangedValues(oldValue, newValue, true); + const changedNewValues = getChangedValues(oldValue, newValue, false); + + // If no values changed, don't create an audit log + if ( + Object.keys(changedOldValues).length === 0 || + Object.keys(changedNewValues).length === 0 + ) { + return; + } + + const auditLog = new auditLogModel({ + changes: { + old: changedOldValues, + new: changedNewValues + }, + parent: parentId, + parentType, + owner: owner._id, + ownerType: ownerType, + operation: 'edit' + }); + + await auditLog.save(); + + await distributeNew(auditLog._id, 'auditLog'); +} + +async function deleteAuditLog( + deleteValue, + parentId, + parentType, + owner, + ownerType +) { + const auditLog = new auditLogModel({ + changes: { + old: deleteValue + }, + parent: parentId, + parentType, + owner: owner._id, + ownerType: ownerType, + operation: 'delete' + }); + + await auditLog.save(); + + await distributeNew(auditLog._id, 'auditLog'); +} + +async function getAuditLogs(idOrIds) { + if (Array.isArray(idOrIds)) { + return auditLogModel.find({ parent: { $in: idOrIds } }).populate('owner'); + } else { + return auditLogModel.find({ parent: idOrIds }).populate('owner'); + } +} + +async function distributeUpdate(value, id, type) { + await etcdServer.setKey(`/${type}s/${id}/object`, value); +} + +async function distributeNew(id, type) { + await etcdServer.setKey(`/${type}s/new`, id); +} + +function flatternObjectIds(object) { + if (!object || typeof object !== 'object') { + return object; + } + + const result = {}; + + for (const [key, value] of Object.entries(object)) { + if (value && typeof value === 'object' && value._id) { + // If the value is an object with _id, convert to just the _id + result[key] = value._id; + } else { + // Keep primitive values as is + result[key] = value; + } + } + + return result; +} + +function expandObjectIds(input) { + // Helper to check if a value is an ObjectId or a 24-char hex string + function isObjectId(val) { + // Check for MongoDB ObjectId instance + if (val instanceof ObjectId) return true; + // Check for exactly 24 hex characters (no special characters) + if (typeof val === 'string' && /^[a-fA-F\d]{24}$/.test(val)) return true; + return false; + } + + // Recursive function + function expand(value) { + if (Array.isArray(value)) { + return value.map(expand); + } else if ( + value && + typeof value === 'object' && + !(value instanceof ObjectId) + ) { + var result = {}; + for (const [key, val] of Object.entries(value)) { + if (key === '_id') { + // Do not expand keys that are already named _id + result[key] = val; + } else if (isObjectId(val)) { + result[key] = { _id: val }; + } else if (Array.isArray(val)) { + result[key] = val.map(expand); + } else if (val instanceof Date) { + result[key] = val; + } else if (val && typeof val === 'object') { + result[key] = expand(val); + } else { + result[key] = val; + } + } + return result; + } else if (isObjectId(value)) { + return { _id: value }; + } else { + return value; + } + } + + return expand(input); +} + +// Returns a filter object based on allowed filters and req.query +function getFilter(query, allowedFilters, parse = true) { + let filter = {}; + for (const [key, value] of Object.entries(query)) { + if (allowedFilters.includes(key)) { + const parsedFilter = parse ? parseFilter(key, value) : { [key]: value }; + filter = { ...filter, ...parsedFilter }; + } + } + return filter; +} + +// Converts a properties argument (string or array) to an array of strings +function convertPropertiesString(properties) { + if (typeof properties === 'string') { + return properties.split(','); + } else if (!Array.isArray(properties)) { + return []; + } + return properties; +} + +export { + parseFilter, + convertToCamelCase, + extractConfigBlock, + newAuditLog, + editAuditLog, + deleteAuditLog, + getAuditLogs, + flatternObjectIds, + expandObjectIds, + distributeUpdate, + distributeNew, + getFilter, // <-- add here + convertPropertiesString +};