From 234652f7b0b4ad85d646ac6d90806bfb7aa99f2f Mon Sep 17 00:00:00 2001 From: Tom Butcher Date: Mon, 10 Aug 2026 00:56:59 +0100 Subject: [PATCH] Enhance database utility functions for improved property value retrieval and reference handling This update introduces several new functions in `database.js` to enhance the handling of Mongoose model references and property values. Key additions include `getObjectRefPathInfo`, `resolveMongooseModel`, and `fetchBasicObjectsByIds`, which improve the efficiency of fetching related objects based on property references. Additionally, the `getPropertyValues` function has been updated to support dynamic references and filtering, ensuring more robust data retrieval. Minor adjustments were also made to the `getJobPropertyValuesRouteHandler` to accommodate the new filtering capabilities. --- src/database/database.js | 159 +++++++++++++++++++++++++++++++- src/services/production/jobs.js | 4 +- src/utils.js | 2 + 3 files changed, 159 insertions(+), 6 deletions(-) diff --git a/src/database/database.js b/src/database/database.js index 8823056..576541f 100644 --- a/src/database/database.js +++ b/src/database/database.js @@ -5,6 +5,7 @@ import log4js from 'log4js'; import crypto from 'crypto'; import sharp from 'sharp'; import { encode as encodeBlurHash } from 'blurhash'; +import mongoose from 'mongoose'; import { uploadFile, deleteFile, downloadFile, BUCKETS } from './ceph.js'; import { deleteAuditLog, @@ -25,11 +26,13 @@ import { deleteNotification, flatternObjectIds, } from '../utils.js'; -import { getAllModels } from '../services/misc/model.js'; +import { getAllModels, getModelByName } from '../services/misc/model.js'; import { redisServer } from './redis.js'; import { auditLogModel } from './schemas/management/auditlog.schema.js'; import { convertObjectIdStringsInFilter } from './utils.js'; +const BASIC_OBJECT_FIELDS = '_id _reference name color state'; + const logger = log4js.getLogger('Database'); logger.level = config.server.logLevel; @@ -651,11 +654,158 @@ export const listPropertyValues = async ({ model, property, filter = {}, search return await model.aggregate(aggregateCommand); }; -export const getPropertyValues = async ({ model, property }) => { +function getObjectRefPathInfo(model, property) { + if (!model?.schema || !property) return null; + + const path = model.schema.path(property); + if (!path) return null; + + const schemaType = path.instance === 'Array' ? path.caster : path; + if (!schemaType) return null; + + const instance = schemaType.instance; + if (instance !== 'ObjectID' && instance !== 'ObjectId') return null; + + return { + ref: schemaType.options?.ref, + refPath: schemaType.options?.refPath, + }; +} + +function resolveMongooseModel(refName) { + if (!refName || typeof refName !== 'string') return null; + try { + return mongoose.model(refName); + } catch { + return getModelByName(refName)?.model ?? null; + } +} + +function inferTypeFieldFromRefFunction(refFn) { + if (typeof refFn !== 'function') return null; + const matches = [...refFn.toString().matchAll(/this\.(\w+)/g)].map((m) => m[1]); + return matches.find((name) => name.endsWith('Type') || name === 'type') || matches[0] || null; +} + +function getArrayParentPaths(model, property) { + if (!model?.schema || !property?.includes('.')) return []; + + const parents = []; + const parts = property.split('.'); + for (let i = 1; i < parts.length; i++) { + const parentPath = parts.slice(0, i).join('.'); + const parentSchemaPath = model.schema.path(parentPath); + if (parentSchemaPath?.instance === 'Array') { + parents.push(parentPath); + } + } + return parents; +} + +async function fetchBasicObjectsByIds(refName, ids) { + const uniqueIds = [ + ...new Map( + (ids || []) + .filter((id) => id != null) + .map((id) => [id.toString(), id]) + ).values(), + ]; + if (uniqueIds.length === 0) return []; + + const withObjectType = (value) => { + const expanded = expandObjectIds(value); + const base = + expanded && typeof expanded === 'object' && !Array.isArray(expanded) + ? expanded + : { _id: expanded }; + return { ...base, objectType: refName }; + }; + + const refModel = resolveMongooseModel(refName); + if (!refModel) { + return uniqueIds.map((id) => withObjectType(id)); + } + + const objects = await refModel + .find({ _id: { $in: uniqueIds } }) + .select(BASIC_OBJECT_FIELDS) + .lean(); + + const foundById = new Map(objects.map((obj) => [obj._id.toString(), obj])); + return uniqueIds.map((id) => withObjectType(foundById.get(id.toString()) || id)); +} + +export const getPropertyValues = async ({ model, property, filter = {} }) => { if (!property) { return []; } - return await model.distinct(property); + + const convertedFilter = + filter && Object.keys(filter).length > 0 ? convertObjectIdStringsInFilter(filter) : {}; + + if (property === 'state') { + return (await model.distinct('state.type', convertedFilter)).map((type) => ({ type })); + } + + const pathInfo = getObjectRefPathInfo(model, property); + const hasFixedRef = typeof pathInfo?.ref === 'string'; + const hasDynamicRef = Boolean(pathInfo?.refPath) || typeof pathInfo?.ref === 'function'; + + if (hasDynamicRef) { + const typeField = pathInfo.refPath || inferTypeFieldFromRefFunction(pathInfo.ref); + if (typeField) { + const pipeline = []; + if (Object.keys(convertedFilter).length > 0) { + pipeline.push({ $match: convertedFilter }); + } + + for (const parentPath of getArrayParentPaths(model, property)) { + pipeline.push({ + $unwind: { + path: `$${parentPath}`, + preserveNullAndEmptyArrays: false, + }, + }); + } + + pipeline.push( + { $match: { [property]: { $ne: null }, [typeField]: { $ne: null } } }, + { $group: { _id: { id: `$${property}`, type: `$${typeField}` } } } + ); + + const pairs = await model.aggregate(pipeline); + const idsByRef = new Map(); + + for (const pair of pairs) { + const id = pair?._id?.id; + const typeValue = pair?._id?.type; + if (id == null || typeValue == null) continue; + + let refName = typeValue; + if (typeof pathInfo.ref === 'function') { + refName = pathInfo.ref.call({ [typeField]: typeValue }); + } + if (!refName) continue; + + if (!idsByRef.has(refName)) idsByRef.set(refName, []); + idsByRef.get(refName).push(id); + } + + const results = []; + for (const [refName, ids] of idsByRef) { + results.push(...(await fetchBasicObjectsByIds(refName, ids))); + } + return results; + } + } + + if (hasFixedRef) { + const ids = await model.distinct(property, convertedFilter); + return fetchBasicObjectsByIds(pathInfo.ref, ids); + } + + const result = await model.distinct(property, convertedFilter); + return result.map((item) => expandObjectIds(item)); }; // Helper to build nested structure for listObjectsByProperty @@ -1269,8 +1419,7 @@ export async function createFileThumbnails(fileData, fileObject) { const isImageFileType = fileObject?.type?.startsWith('image/'); const hasThumbnailSource = - (hasGCodeThumbnailMeta && gcodeThumbnail.data.length > 0) || - (isImageFileType && fileData); + (hasGCodeThumbnailMeta && gcodeThumbnail.data.length > 0) || (isImageFileType && fileData); if (!hasThumbnailSource || !fileObject?._id) { return; diff --git a/src/services/production/jobs.js b/src/services/production/jobs.js index 18a4f5d..0de8493 100644 --- a/src/services/production/jobs.js +++ b/src/services/production/jobs.js @@ -13,6 +13,7 @@ import { getModelHistory, searchObjects, checkStates, + getPropertyValues, getObjectNeighbors, } from '../../database/database.js'; const logger = log4js.getLogger('Jobs'); @@ -75,10 +76,11 @@ export const listJobsByPropertiesRouteHandler = async ( res.send(result); }; -export const getJobPropertyValuesRouteHandler = async (req, res, property) => { +export const getJobPropertyValuesRouteHandler = async (req, res, property, filter = {}) => { const result = await getPropertyValues({ model: jobModel, property, + filter, }); res.send(result); }; diff --git a/src/utils.js b/src/utils.js index 6d9d9d2..dc189c1 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1498,6 +1498,8 @@ function expandObjectIds(input) { function expand(value) { if (Array.isArray(value)) { return value.map(expand); + } else if (value instanceof Date) { + return value; } else if (value && typeof value === 'object' && !(value instanceof mongoose.Types.ObjectId)) { var result = {}; for (const [key, val] of Object.entries(value)) {