Enhance database utility functions for improved property value retrieval and reference handling

This update introduces several new functions in `database.js` to enhance the handling of Mongoose model references and property values. Key additions include `getObjectRefPathInfo`, `resolveMongooseModel`, and `fetchBasicObjectsByIds`, which improve the efficiency of fetching related objects based on property references. Additionally, the `getPropertyValues` function has been updated to support dynamic references and filtering, ensuring more robust data retrieval. Minor adjustments were also made to the `getJobPropertyValuesRouteHandler` to accommodate the new filtering capabilities.

src/database/database.js

@@ -5,6 +5,7 @@ import log4js from 'log4js';
 import crypto from 'crypto';
 import sharp from 'sharp';
 import { encode as encodeBlurHash } from 'blurhash';
+import mongoose from 'mongoose';
 import { uploadFile, deleteFile, downloadFile, BUCKETS } from './ceph.js';
 import {
   deleteAuditLog,
@@ -25,11 +26,13 @@ import {
   deleteNotification,
   flatternObjectIds,
 } from '../utils.js';
-import { getAllModels } from '../services/misc/model.js';
+import { getAllModels, getModelByName } from '../services/misc/model.js';
 import { redisServer } from './redis.js';
 import { auditLogModel } from './schemas/management/auditlog.schema.js';
 import { convertObjectIdStringsInFilter } from './utils.js';
 
+const BASIC_OBJECT_FIELDS = '_id _reference name color state';
+
 const logger = log4js.getLogger('Database');
 logger.level = config.server.logLevel;
 
@@ -651,11 +654,158 @@ export const listPropertyValues = async ({ model, property, filter = {}, search
   return await model.aggregate(aggregateCommand);
 };
 
-export const getPropertyValues = async ({ model, property }) => {
+function getObjectRefPathInfo(model, property) {
+  if (!model?.schema || !property) return null;
+
+  const path = model.schema.path(property);
+  if (!path) return null;
+
+  const schemaType = path.instance === 'Array' ? path.caster : path;
+  if (!schemaType) return null;
+
+  const instance = schemaType.instance;
+  if (instance !== 'ObjectID' && instance !== 'ObjectId') return null;
+
+  return {
+    ref: schemaType.options?.ref,
+    refPath: schemaType.options?.refPath,
+  };
+}
+
+function resolveMongooseModel(refName) {
+  if (!refName || typeof refName !== 'string') return null;
+  try {
+    return mongoose.model(refName);
+  } catch {
+    return getModelByName(refName)?.model ?? null;
+  }
+}
+
+function inferTypeFieldFromRefFunction(refFn) {
+  if (typeof refFn !== 'function') return null;
+  const matches = [...refFn.toString().matchAll(/this\.(\w+)/g)].map((m) => m[1]);
+  return matches.find((name) => name.endsWith('Type') || name === 'type') || matches[0] || null;
+}
+
+function getArrayParentPaths(model, property) {
+  if (!model?.schema || !property?.includes('.')) return [];
+
+  const parents = [];
+  const parts = property.split('.');
+  for (let i = 1; i < parts.length; i++) {
+    const parentPath = parts.slice(0, i).join('.');
+    const parentSchemaPath = model.schema.path(parentPath);
+    if (parentSchemaPath?.instance === 'Array') {
+      parents.push(parentPath);
+    }
+  }
+  return parents;
+}
+
+async function fetchBasicObjectsByIds(refName, ids) {
+  const uniqueIds = [
+    ...new Map(
+      (ids || [])
+        .filter((id) => id != null)
+        .map((id) => [id.toString(), id])
+    ).values(),
+  ];
+  if (uniqueIds.length === 0) return [];
+
+  const withObjectType = (value) => {
+    const expanded = expandObjectIds(value);
+    const base =
+      expanded && typeof expanded === 'object' && !Array.isArray(expanded)
+        ? expanded
+        : { _id: expanded };
+    return { ...base, objectType: refName };
+  };
+
+  const refModel = resolveMongooseModel(refName);
+  if (!refModel) {
+    return uniqueIds.map((id) => withObjectType(id));
+  }
+
+  const objects = await refModel
+    .find({ _id: { $in: uniqueIds } })
+    .select(BASIC_OBJECT_FIELDS)
+    .lean();
+
+  const foundById = new Map(objects.map((obj) => [obj._id.toString(), obj]));
+  return uniqueIds.map((id) => withObjectType(foundById.get(id.toString()) || id));
+}
+
+export const getPropertyValues = async ({ model, property, filter = {} }) => {
   if (!property) {
     return [];
   }
-  return await model.distinct(property);
+
+  const convertedFilter =
+    filter && Object.keys(filter).length > 0 ? convertObjectIdStringsInFilter(filter) : {};
+
+  if (property === 'state') {
+    return (await model.distinct('state.type', convertedFilter)).map((type) => ({ type }));
+  }
+
+  const pathInfo = getObjectRefPathInfo(model, property);
+  const hasFixedRef = typeof pathInfo?.ref === 'string';
+  const hasDynamicRef = Boolean(pathInfo?.refPath) || typeof pathInfo?.ref === 'function';
+
+  if (hasDynamicRef) {
+    const typeField = pathInfo.refPath || inferTypeFieldFromRefFunction(pathInfo.ref);
+    if (typeField) {
+      const pipeline = [];
+      if (Object.keys(convertedFilter).length > 0) {
+        pipeline.push({ $match: convertedFilter });
+      }
+
+      for (const parentPath of getArrayParentPaths(model, property)) {
+        pipeline.push({
+          $unwind: {
+            path: `$${parentPath}`,
+            preserveNullAndEmptyArrays: false,
+          },
+        });
+      }
+
+      pipeline.push(
+        { $match: { [property]: { $ne: null }, [typeField]: { $ne: null } } },
+        { $group: { _id: { id: `$${property}`, type: `$${typeField}` } } }
+      );
+
+      const pairs = await model.aggregate(pipeline);
+      const idsByRef = new Map();
+
+      for (const pair of pairs) {
+        const id = pair?._id?.id;
+        const typeValue = pair?._id?.type;
+        if (id == null || typeValue == null) continue;
+
+        let refName = typeValue;
+        if (typeof pathInfo.ref === 'function') {
+          refName = pathInfo.ref.call({ [typeField]: typeValue });
+        }
+        if (!refName) continue;
+
+        if (!idsByRef.has(refName)) idsByRef.set(refName, []);
+        idsByRef.get(refName).push(id);
+      }
+
+      const results = [];
+      for (const [refName, ids] of idsByRef) {
+        results.push(...(await fetchBasicObjectsByIds(refName, ids)));
+      }
+      return results;
+    }
+  }
+
+  if (hasFixedRef) {
+    const ids = await model.distinct(property, convertedFilter);
+    return fetchBasicObjectsByIds(pathInfo.ref, ids);
+  }
+
+  const result = await model.distinct(property, convertedFilter);
+  return result.map((item) => expandObjectIds(item));
 };
 
 // Helper to build nested structure for listObjectsByProperty
@@ -1269,8 +1419,7 @@ export async function createFileThumbnails(fileData, fileObject) {
     const isImageFileType = fileObject?.type?.startsWith('image/');
 
     const hasThumbnailSource =
-      (hasGCodeThumbnailMeta && gcodeThumbnail.data.length > 0) ||
-      (isImageFileType && fileData);
+      (hasGCodeThumbnailMeta && gcodeThumbnail.data.length > 0) || (isImageFileType && fileData);
 
     if (!hasThumbnailSource || !fileObject?._id) {
       return;

src/services/production/jobs.js

@@ -13,6 +13,7 @@ import {
   getModelHistory,
   searchObjects,
   checkStates,
+  getPropertyValues,
   getObjectNeighbors,
 } from '../../database/database.js';
 const logger = log4js.getLogger('Jobs');
@@ -75,10 +76,11 @@ export const listJobsByPropertiesRouteHandler = async (
   res.send(result);
 };
 
-export const getJobPropertyValuesRouteHandler = async (req, res, property) => {
+export const getJobPropertyValuesRouteHandler = async (req, res, property, filter = {}) => {
   const result = await getPropertyValues({
     model: jobModel,
     property,
+    filter,
   });
   res.send(result);
 };

src/utils.js

@@ -1498,6 +1498,8 @@ function expandObjectIds(input) {
   function expand(value) {
     if (Array.isArray(value)) {
       return value.map(expand);
+    } else if (value instanceof Date) {
+      return value;
     } else if (value && typeof value === 'object' && !(value instanceof mongoose.Types.ObjectId)) {
       var result = {};
       for (const [key, val] of Object.entries(value)) {