From e563538e8925dcf4955839de5aed4a8ae70eee6a Mon Sep 17 00:00:00 2001 From: Tom Butcher Date: Sat, 22 Aug 2026 20:02:36 +0100 Subject: [PATCH] Refactor database utility functions and enhance document job schema - Reformatted the `NOTIFICATION_EXCLUDED_MODELS` array for improved readability in `utils.js`. - Updated the `editNotification` and `createNotification` functions for better code clarity by breaking long lines. - Added a new `quantity` field to the `documentJobSchema`, ensuring it is required with a default value of 1 and a minimum of 1. - Introduced new filtering capabilities in `updatemanager.js` to handle complex filter conditions, enhancing the overall filtering logic. - Updated tests to mock new filter functionalities, ensuring proper integration and functionality verification. --- src/database/filter.js | 922 ++++++++++++++++++ .../schemas/management/documentjob.schema.js | 6 + src/database/utils.js | 35 +- src/updates/__tests__/updatemanager.test.js | 12 + src/updates/updatemanager.js | 150 ++- 5 files changed, 1098 insertions(+), 27 deletions(-) create mode 100644 src/database/filter.js diff --git a/src/database/filter.js b/src/database/filter.js new file mode 100644 index 0000000..4719048 --- /dev/null +++ b/src/database/filter.js @@ -0,0 +1,922 @@ +import mongoose from 'mongoose'; +import log4js from 'log4js'; +import { loadConfig } from '../config.js'; + +const config = loadConfig(); +const logger = log4js.getLogger('Filter'); +logger.level = config.server.logLevel; + +let modelsCache = null; + +async function getModels() { + if (!modelsCache) { + modelsCache = (await import('./schemas/models.js')).models; + } + return modelsCache; +} + +function getModelsSync() { + return modelsCache || {}; +} + +function getModelByName(name) { + return ( + Object.values(getModelsSync()).find(entry => entry.type === name) || null + ); +} + +function getModelByPrefix(prefix) { + return getModelsSync()[prefix] || null; +} + +async function getObjectTypeModel(objectType) { + const models = await getModels(); + const entry = Object.values(models).find( + item => item.type === objectType || item.model?.modelName === objectType + ); + return entry?.model ?? null; +} + +// --------------------------------------------------------------------------- +// Filter expression parsing (Microsoft Dynamics NAV / Business Central style) +// +// Supported syntax (all compiled down to MongoDB query operators): +// = equal to 377 -> { field: 377 } +// <> not equal to <>0 -> { field: { $ne: 0 } } +// > greater than >1200 -> { field: { $gt: 1200 } } +// >= greater than or equal >=1200 -> { field: { $gte: 1200 } } +// < less than <1200 -> { field: { $lt: 1200 } } +// <= less than or equal <=1200 -> { field: { $lte: 1200 } } +// .. interval 1100..2100 -> { field: { $gte: 1100, $lte: 2100 } } +// open-ended intervals ..2500 / 23.. +// | either / or 1200|1300 -> { field: { $in: [1200, 1300] } } +// & and <2000&>1000 -> { field: { $gt: 1000, $lt: 2000 } } +// ( ) grouping / precedence 30|(>=10&<=20) +// * any number of characters Co* / *Co / *Co* +// ? a single character Hans?n +// @ ignore case @location (text matching is case-insensitive) +// +// Date-typed fields (detected by name, e.g. *At / *Date / *Time) interpret bare +// values as calendar days/datetimes, e.g. "22" => the whole of day 22 of the +// current month/year, "22..23" => start of 22 through end of 23. +// +// Note: BC's space-delimited date shorthand is locale dependent; here numeric +// date operands are read day-first ("D M Y H Min S") with missing components +// filled from the current date and the interval boundary. +// --------------------------------------------------------------------------- + +function buildWildcardRegexPattern(input) { + // Escape all regex special chars except * and ? (which we treat as wildcards) + const escaped = String(input).replace(/[.+^${}()|[\]\\]/g, '\\$&'); + // * matches any run of characters, ? matches exactly one character + const withWildcards = escaped.replace(/\*/g, '.*').replace(/\?/g, '.'); + // Anchor so that, without wildcards, this is an exact match + return `^${withWildcards}$`; +} + +function looksLikeDateField(property) { + const last = String(property).split('.').pop(); + return ( + /[a-z](?:At|Date|Time)$/.test(last) || + /^(?:date|time|datetime)$/i.test(last) + ); +} + +const NO_MATCH_CONDITION = { op: { $in: [] } }; + +function isBooleanField(property, model = null) { + const schema = model?.schema; + if (schema) { + const path = schema.path(property); + return path?.instance === 'Boolean'; + } + + let found = false; + for (const entry of Object.values(getModelsSync())) { + const path = entry.model?.schema?.path(property); + if (!path) continue; + found = true; + if (path.instance !== 'Boolean') return false; + } + return found; +} + +function isObjectIdPath(property) { + return property === '_id' || property.endsWith('._id'); +} + +function getBaseProperty(property) { + if (property === '_id') return '_id'; + if (property.endsWith('._id')) return property.slice(0, -4); + return property; +} + +function getEmbeddedSchemaType(path) { + if (!path) return null; + return ( + path.embeddedSchemaType ?? path.$embeddedSchemaType ?? path.caster ?? null + ); +} + +function getObjectIdSchemaTypeFromPath(path) { + if (!path) return null; + const schemaType = + path.instance === 'Array' ? getEmbeddedSchemaType(path) : path; + if (!schemaType) return null; + if ( + schemaType.instance === 'ObjectId' || + schemaType.instance === 'ObjectID' + ) { + return schemaType; + } + return null; +} + +function getSchemaPathFromModels(property, model = null) { + if (model?.schema?.path(property)) return model.schema.path(property); + for (const entry of Object.values(getModelsSync())) { + const path = entry.model?.schema?.path(property); + if (path) return path; + } + return null; +} + +function getFilterFieldKind(property, model = null) { + const baseProperty = getBaseProperty(property); + + const inspectPath = path => { + if (!path) return null; + + const objectIdType = getObjectIdSchemaTypeFromPath(path); + if (isObjectIdPath(property) || objectIdType) { + const ref = objectIdType?.options?.ref ?? path.options?.ref; + if (property.endsWith('._id') && property !== '_id' && ref) { + return { kind: 'objectId', property: baseProperty }; + } + if (ref) { + return { kind: 'objectRef', property: baseProperty, ref }; + } + return { kind: 'objectId', property: baseProperty }; + } + + return { kind: 'default', property: baseProperty }; + }; + + if (model) { + return ( + inspectPath(model.schema?.path(baseProperty)) || { + kind: 'default', + property: baseProperty + } + ); + } + + const path = getSchemaPathFromModels(baseProperty); + const kind = inspectPath(path); + if (kind) return kind; + + return { kind: 'default', property: baseProperty }; +} + +function parsePrefixedValue(value) { + const trimmed = String(value).trim(); + if (trimmed.length > 4 && trimmed.charAt(3) === ':') { + return { + prefix: trimmed.slice(0, 3).toUpperCase(), + suffix: trimmed.slice(4).trim(), + hadPrefix: true + }; + } + return { prefix: null, suffix: trimmed, hadPrefix: false }; +} + +function buildRegexOp(pattern, useOptions = true) { + const op = { $regex: pattern }; + if (useOptions) op.$options = 'i'; + return op; +} + +function getSchemaRefName(property, model = null) { + const baseProperty = getBaseProperty(property); + const path = + model?.schema?.path(baseProperty) ?? getSchemaPathFromModels(baseProperty); + const embeddedType = getEmbeddedSchemaType(path); + return path?.options?.ref ?? embeddedType?.options?.ref ?? null; +} + +function getRefModelEntryFromSchemaRef(refName) { + if (!refName) return null; + + let refModel; + try { + refModel = mongoose.model(refName); + } catch { + refModel = getModelByName(refName)?.model ?? null; + } + + if (!refModel) return null; + + const registryEntry = getModelByName(refName); + return { + model: refModel, + idField: registryEntry?.idField ?? '_id', + referenceField: registryEntry?.referenceField ?? '_reference' + }; +} + +function getRefModelEntryFromPrefix(prefix) { + const entry = getModelByPrefix(prefix); + if (!entry?.model) return null; + return { + model: entry.model, + idField: entry.idField ?? '_id', + referenceField: entry.referenceField ?? '_reference' + }; +} + +function stripPrefixFromOperand(operand) { + const { suffix, hadPrefix } = parsePrefixedValue(String(operand).trim()); + return hadPrefix ? suffix : String(operand).trim(); +} + +function getRefModelEntryForToken(token, fallbackRefName) { + const { prefix, hadPrefix } = parsePrefixedValue(String(token).trim()); + if (hadPrefix) { + const entry = getRefModelEntryFromPrefix(prefix); + if (entry) return entry; + } + return getRefModelEntryFromSchemaRef(fallbackRefName); +} + +function parseBooleanOperand(value) { + const lower = String(value).trim().toLowerCase(); + if (lower === 'yes') return true; + if (lower === 'no') return false; + if (lower === 'true') return true; + if (lower === 'false') return false; + if (lower === '1') return true; + if (lower === '0') return false; + if (lower === 'on') return true; + if (lower === 'off') return false; + if (lower === 'y') return true; + if (lower === 'n') return false; + if (typeof value === 'boolean') return value; + return undefined; +} + +function buildBooleanEquality(value) { + const bool = parseBooleanOperand(value); + if (bool === undefined) return NO_MATCH_CONDITION; + return { value: bool }; +} + +function buildBooleanComparison(name, value) { + if (name === 'eq') return buildBooleanEquality(value); + + if (name === 'ne') { + const bool = parseBooleanOperand(value); + if (bool === undefined) return NO_MATCH_CONDITION; + return { op: { $ne: bool } }; + } + + return NO_MATCH_CONDITION; +} + +function isObjectIdString(value) { + return /^[a-f\d]{24}$/i.test(value); +} + +function isNumeric(value) { + return value.trim() !== '' && !isNaN(value); +} + +function startOfDay(date) { + const d = new Date(date); + d.setHours(0, 0, 0, 0); + return d; +} + +function endOfDay(date) { + const d = new Date(date); + d.setHours(23, 59, 59, 999); + return d; +} + +function normalizeYear(year) { + if (year >= 100) return year; + return year < 70 ? 2000 + year : 1900 + year; +} + +// Parses a single date operand to a Date, filling missing parts from the +// current date and the supplied interval boundary ('start' | 'end'). +function parseDateOperand(value, boundary = 'start') { + const text = String(value).trim(); + if (!text) return null; + const end = boundary === 'end'; + + // Values with explicit separators (ISO and similar) are parsed directly. + if (/[-/T]/.test(text) || /\d:\d/.test(text)) { + const parsed = new Date(text); + if (isNaN(parsed.getTime())) return null; + if (!/[T:]/.test(text)) { + return end ? endOfDay(parsed) : startOfDay(parsed); + } + return parsed; + } + + // Numeric, day-first form: "D [M] [Y] [H] [Min] [S]". + const parts = text.split(/\s+/); + if (!parts.every(part => /^\d+$/.test(part))) return null; + const nums = parts.map(Number); + const now = new Date(); + const day = nums[0]; + const month = nums.length >= 2 ? nums[1] : now.getMonth() + 1; + const year = nums.length >= 3 ? normalizeYear(nums[2]) : now.getFullYear(); + const hour = nums.length >= 4 ? nums[3] : end ? 23 : 0; + const minute = nums.length >= 5 ? nums[4] : end ? 59 : 0; + const second = nums.length >= 6 ? nums[5] : end ? 59 : 0; + const ms = end ? 999 : 0; + const date = new Date(year, month - 1, day, hour, minute, second, ms); + return isNaN(date.getTime()) ? null : date; +} + +// Coerces a literal to its strongest matching scalar type. +function coerceScalar(value) { + const lower = value.toLowerCase(); + if (lower === 'true') return true; + if (lower === 'false') return false; + if (isObjectIdString(value)) return new mongoose.Types.ObjectId(value); + if (isNumeric(value)) return Number(value); + return value; +} + +// Coerces an interval/comparison boundary, preferring a date for date fields. +function coerceBoundary(value, isDateField, boundary) { + if (isDateField) { + const date = parseDateOperand(value, boundary); + if (date) return date; + } + return coerceScalar(value); +} + +// Splits on a separator while respecting parentheses depth. +function splitTopLevel(str, separator) { + const parts = []; + let depth = 0; + let current = ''; + for (const ch of str) { + if (ch === '(') depth++; + else if (ch === ')') depth = Math.max(0, depth - 1); + + if (ch === separator && depth === 0) { + parts.push(current); + current = ''; + } else { + current += ch; + } + } + parts.push(current); + return parts; +} + +// True when a single outer pair of parentheses wraps the whole token. +function isWrappedInParens(str) { + if (!str.startsWith('(') || !str.endsWith(')')) return false; + let depth = 0; + for (let i = 0; i < str.length; i++) { + if (str[i] === '(') depth++; + else if (str[i] === ')') { + depth--; + if (depth === 0 && i < str.length - 1) return false; + } + } + return depth === 0; +} + +// Parses a filter expression into an AST of or / and / leaf nodes. +function parseExpression(str) { + const orParts = splitTopLevel(str, '|'); + if (orParts.length > 1) { + return { type: 'or', items: orParts.map(parseExpression) }; + } + const andParts = splitTopLevel(str, '&'); + if (andParts.length > 1) { + return { type: 'and', items: andParts.map(parseExpression) }; + } + const trimmed = str.trim(); + if (isWrappedInParens(trimmed)) { + return parseExpression(trimmed.slice(1, -1)); + } + return { type: 'leaf', token: trimmed }; +} + +// Strips a leading @ (ignore-case marker); text matching is already case-insensitive. +function stripIgnoreCase(str) { + return str.startsWith('@') ? str.slice(1) : str; +} + +// Builds an equality condition (value, date range, or wildcard regex). +function buildEquality( + value, + isDateField, + isBooleanField = false, + isObjectIdField = false +) { + if (isBooleanField) return buildBooleanEquality(value); + if (isObjectIdField) { + const { suffix } = parsePrefixedValue(value); + if (/[*?]/.test(suffix)) return NO_MATCH_CONDITION; + if (isObjectIdString(suffix)) + return { value: new mongoose.Types.ObjectId(suffix) }; + return NO_MATCH_CONDITION; + } + if (isDateField) { + const start = parseDateOperand(value, 'start'); + const end = parseDateOperand(value, 'end'); + if (start && end) return { op: { $gte: start, $lte: end } }; + } + const lower = value.toLowerCase(); + if (lower === 'true') return { value: true }; + if (lower === 'false') return { value: false }; + if (isObjectIdString(value)) + return { value: new mongoose.Types.ObjectId(value) }; + if (isNumeric(value)) return { value: Number(value) }; + return { op: buildRegexOp(buildWildcardRegexPattern(value)) }; +} + +// Builds a comparison condition for a single operator. +function buildComparison( + name, + value, + isDateField, + isBooleanField = false, + isObjectIdField = false +) { + if (isBooleanField) return buildBooleanComparison(name, value); + if (isObjectIdField) { + const { suffix } = parsePrefixedValue(value); + if (/[*?]/.test(suffix)) return NO_MATCH_CONDITION; + if (name === 'eq') return buildEquality(suffix, false, false, true); + if (name === 'ne' && isObjectIdString(suffix)) { + return { op: { $ne: new mongoose.Types.ObjectId(suffix) } }; + } + return NO_MATCH_CONDITION; + } + if (name === 'eq') return buildEquality(value, isDateField); + + if (name === 'ne') { + if (isDateField) { + const start = parseDateOperand(value, 'start'); + const end = parseDateOperand(value, 'end'); + if (start && end) return { op: { $not: { $gte: start, $lte: end } } }; + } + if (/[*?]/.test(value)) { + return { op: { $not: buildRegexOp(buildWildcardRegexPattern(value)) } }; + } + return { op: { $ne: coerceScalar(value) } }; + } + + // For dates, < and >= align to the start of the day, > and <= to the end. + const boundary = name === 'gt' || name === 'lte' ? 'end' : 'start'; + return { op: { [`$${name}`]: coerceBoundary(value, isDateField, boundary) } }; +} + +function parseLeafComparisonOperators( + token, + isDateField, + isBooleanField, + isObjectIdField +) { + const operators = [ + ['<>', 'ne'], + ['>=', 'gte'], + ['<=', 'lte'], + ['>', 'gt'], + ['<', 'lt'], + ['=', 'eq'] + ]; + for (const [symbol, name] of operators) { + if (token.startsWith(symbol)) { + return buildComparison( + name, + stripIgnoreCase(token.slice(symbol.length).trim()), + isDateField, + isBooleanField, + isObjectIdField + ); + } + } + return null; +} + +// Parses a leaf token (range, comparison, or plain value) into a condition. +function parseLeafCondition( + rawToken, + isDateField, + isBooleanField = false, + isObjectIdField = false +) { + const token = rawToken.trim(); + if (token === '') { + if (isBooleanField || isObjectIdField) return NO_MATCH_CONDITION; + return { op: buildRegexOp('^$') }; + } + + // Interval: a..b, ..b, a.. + const rangeIdx = token.indexOf('..'); + if (rangeIdx !== -1) { + if (isBooleanField || isObjectIdField) return NO_MATCH_CONDITION; + const lo = stripIgnoreCase(token.slice(0, rangeIdx).trim()); + const hi = stripIgnoreCase(token.slice(rangeIdx + 2).trim()); + const op = {}; + if (lo !== '') op.$gte = coerceBoundary(lo, isDateField, 'start'); + if (hi !== '') op.$lte = coerceBoundary(hi, isDateField, 'end'); + // A bare ".." places no constraint on the field. + if (Object.keys(op).length === 0) return { query: {} }; + return { op }; + } + + const comparison = parseLeafComparisonOperators( + token, + isDateField, + isBooleanField, + isObjectIdField + ); + if (comparison) return comparison; + + return buildEquality( + stripIgnoreCase(token), + isDateField, + isBooleanField, + isObjectIdField + ); +} + +// Converts a condition descriptor into a MongoDB query object for `property`. +function conditionToQuery(cond, property) { + if (cond.query) return cond.query; + if (cond.op) return { [property]: cond.op }; + return { [property]: cond.value }; +} + +function isOnlyRegex(op) { + return Object.keys(op).every(key => key === '$regex' || key === '$options'); +} + +// Combines AND-ed conditions, merging operator objects on the same field where possible. +function combineAnd(children, property) { + if (children.some(child => child.query)) { + return { + query: { $and: children.map(child => conditionToQuery(child, property)) } + }; + } + const merged = {}; + for (const child of children) { + if (child.value !== undefined) { + merged.$eq = child.value; + } else if (child.op) { + Object.assign(merged, child.op); + } + } + return { op: merged }; +} + +// Combines OR-ed conditions, collapsing to $in or a single regex when possible. +function combineOr(children, property, isObjectIdField = false) { + if (children.every(child => child.value !== undefined)) { + return { op: { $in: children.map(child => child.value) } }; + } + if ( + !isObjectIdField && + children.every(child => child.op && isOnlyRegex(child.op)) + ) { + const pattern = children.map(child => child.op.$regex).join('|'); + const op = { $regex: pattern }; + if (children[0].op.$options) op.$options = children[0].op.$options; + return { op }; + } + return { + query: { $or: children.map(child => conditionToQuery(child, property)) } + }; +} + +function buildCondition( + node, + property, + isDateField, + isBooleanField = false, + isObjectIdField = false +) { + if (node.type === 'leaf') { + return parseLeafCondition( + node.token, + isDateField, + isBooleanField, + isObjectIdField + ); + } + const children = node.items.map(item => + buildCondition(item, property, isDateField, isBooleanField, isObjectIdField) + ); + return node.type === 'and' + ? combineAnd(children, property) + : combineOr(children, property, isObjectIdField); +} + +function idsToCondition(ids) { + if (!ids || ids.length === 0) return NO_MATCH_CONDITION; + if (ids.length === 1) return { value: ids[0] }; + return { op: { $in: ids } }; +} + +function extractObjectIdFromOperand(operand) { + const { suffix } = parsePrefixedValue( + stripIgnoreCase(String(operand).trim()) + ); + if (isObjectIdString(suffix)) { + return new mongoose.Types.ObjectId(suffix); + } + return null; +} + +const REF_OPERATOR_SYMBOLS = { + eq: '', + ne: '<>', + gt: '>', + gte: '>=', + lt: '<', + lte: '<=' +}; + +function buildRefListExpression(operator, operand) { + const text = String(operand).trim(); + if (operator === 'eq') return text; + return `${REF_OPERATOR_SYMBOLS[operator]}${text}`; +} + +// Pre-list the referenced model, matching the expression against name and/or _reference via parseFilter. +async function listRefModelIds(refModelEntry, expression) { + const refModel = refModelEntry.model; + const orClauses = []; + + if (refModel.schema.path('name')) { + orClauses.push(await parseFilter('name', expression, refModel)); + } + if (refModel.schema.path('_reference')) { + orClauses.push(await parseFilter('_reference', expression, refModel)); + } + + if (orClauses.length === 0) return []; + + const query = orClauses.length === 1 ? orClauses[0] : { $or: orClauses }; + const docs = await refModel.find(query).select('_id').lean(); + return docs.map(doc => doc._id); +} + +function conditionToIds(cond) { + if (cond.query) return null; + if (cond.value !== undefined) return [cond.value]; + if (cond.op?.$in) return cond.op.$in; + return null; +} + +function combineOrRefIds(children) { + const ids = new Set(); + for (const child of children) { + if (child.query) return child; + const childIds = conditionToIds(child); + if (!childIds) return child; + childIds.forEach(id => ids.add(String(id))); + } + return idsToCondition([...ids].map(id => new mongoose.Types.ObjectId(id))); +} + +function combineAndRefIds(children) { + let ids = null; + for (const child of children) { + if (child.query) return child; + const childIds = conditionToIds(child); + if (!childIds || childIds.length === 0) return NO_MATCH_CONDITION; + const set = new Set(childIds.map(String)); + if (ids === null) { + ids = set; + } else { + ids = new Set([...ids].filter(id => set.has(id))); + } + if (ids.size === 0) return NO_MATCH_CONDITION; + } + return idsToCondition([...ids].map(id => new mongoose.Types.ObjectId(id))); +} + +async function resolveRefOperand(operand, refModelEntry, operator = 'eq') { + if (!refModelEntry?.model) return NO_MATCH_CONDITION; + + const lookupOperand = String(operand).trim(); + const objectId = extractObjectIdFromOperand(lookupOperand); + + if (objectId) { + if (operator === 'ne') return { op: { $ne: objectId } }; + if (operator === 'eq') return { value: objectId }; + return NO_MATCH_CONDITION; + } + + const expression = buildRefListExpression(operator, lookupOperand); + const ids = await listRefModelIds(refModelEntry, expression); + + if (operator === 'ne') { + if (ids.length === 0) return { query: {} }; + return { op: { $nin: ids } }; + } + + return idsToCondition(ids); +} + +async function resolveRefLeaf(rawToken, fallbackRefName) { + const token = rawToken.trim(); + if (token === '') return NO_MATCH_CONDITION; + + const refModelEntry = getRefModelEntryForToken(token, fallbackRefName); + const lookupToken = stripPrefixFromOperand(token); + + if (!refModelEntry?.model) { + const objectId = extractObjectIdFromOperand(lookupToken); + if (objectId) return { value: objectId }; + return NO_MATCH_CONDITION; + } + + const rangeIdx = lookupToken.indexOf('..'); + if (rangeIdx !== -1) { + const ids = await listRefModelIds(refModelEntry, lookupToken); + return idsToCondition(ids); + } + + const operators = [ + ['<>', 'ne'], + ['>=', 'gte'], + ['<=', 'lte'], + ['>', 'gt'], + ['<', 'lt'], + ['=', 'eq'] + ]; + for (const [symbol, name] of operators) { + if (lookupToken.startsWith(symbol)) { + return resolveRefOperand( + lookupToken.slice(symbol.length), + refModelEntry, + name + ); + } + } + + const objectId = extractObjectIdFromOperand(lookupToken); + if (objectId) return { value: objectId }; + + const ids = await listRefModelIds(refModelEntry, lookupToken); + return idsToCondition(ids); +} + +async function resolveRefCondition(node, fallbackRefName) { + if (node.type === 'leaf') { + return resolveRefLeaf(node.token, fallbackRefName); + } + + const children = await Promise.all( + node.items.map(item => resolveRefCondition(item, fallbackRefName)) + ); + + return node.type === 'and' + ? combineAndRefIds(children) + : combineOrRefIds(children); +} + +async function resolveObjectRefFilter( + filterProperty, + expression, + fallbackRefName +) { + const tree = parseExpression(expression); + const condition = await resolveRefCondition(tree, fallbackRefName); + return conditionToQuery(condition, filterProperty); +} + +async function parseFilter(property, value, model = null) { + // Normalize state filter to state.type for schemas with state: { type } + if (property === 'state') { + property = 'state.type'; + } + + const fieldKind = getFilterFieldKind(property, model); + const filterProperty = fieldKind.property; + const isBoolField = isBooleanField(property, model); + + if (value?._id !== undefined && value?._id !== null) { + return { [filterProperty]: new mongoose.Types.ObjectId(value._id) }; + } + + // Non-string values pass through; boolean fields only accept actual booleans. + if (typeof value !== 'string') { + if (isBoolField && typeof value !== 'boolean') { + return { [filterProperty]: { $in: [] } }; + } + return { [filterProperty]: value }; + } + + const trimmed = value.trim(); + + if ( + (fieldKind.kind === 'objectId' || fieldKind.kind === 'objectRef') && + property != '_id' + ) { + const refName = fieldKind.ref ?? getSchemaRefName(property, model); + return resolveObjectRefFilter(filterProperty, trimmed, refName); + } + + let expression = trimmed; + if (fieldKind.kind === 'default' && expression.charAt(3) === ':') { + const afterColon = value.split(':')[1]; + expression = afterColon != null ? afterColon.trim() : ''; + } + + const isDateField = looksLikeDateField(property); + const isObjectIdField = fieldKind.kind === 'objectId'; + const tree = parseExpression(expression); + const condition = buildCondition( + tree, + filterProperty, + isDateField, + isBoolField, + isObjectIdField + ); + return conditionToQuery(condition, filterProperty); +} + +// Merges per-field filter clauses, falling back to $and when clauses use +// logical operators or target the same field (so nothing is silently lost). +function mergeFilterClauses(clauses) { + const valid = clauses.filter( + clause => clause && Object.keys(clause).length > 0 + ); + if (valid.length === 0) return {}; + if (valid.length === 1) return valid[0]; + + const seen = new Set(); + let needsAnd = false; + for (const clause of valid) { + for (const key of Object.keys(clause)) { + if (key === '$or' || key === '$and' || seen.has(key)) { + needsAnd = true; + } + seen.add(key); + } + } + + return needsAnd ? { $and: valid } : Object.assign({}, ...valid); +} + +function normalizeFilterQuery(query = {}) { + const queryClean = { ...query }; + + // qs turns `order._id=` into `{ order: { _id } }`. Lift it so it can be parsed + // as the order object filter instead of a nested query object. + if ( + queryClean.order && + typeof queryClean.order === 'object' && + !Array.isArray(queryClean.order) + ) { + if (queryClean.order._id !== undefined) { + queryClean['order._id'] = queryClean.order._id; + } + delete queryClean.order; + } + + return queryClean; +} + +// Returns a filter object based on allowed filters and req.query +async function getFilter(query, allowedFilters, parse = true, model = null) { + await getModels(); + const clauses = []; + const queryClean = normalizeFilterQuery(query); + for (const key of ['sortProperty', 'sortOrder', 'page', 'limit']) { + if (key in queryClean) delete queryClean[key]; + } + logger.info('queryExcludingSortAndOrder', queryClean); + if ( + !parse && + queryClean['order._id'] !== undefined && + allowedFilters.includes('order') + ) { + queryClean.order = queryClean['order._id']; + delete queryClean['order._id']; + } + for (const [key, value] of Object.entries(queryClean)) { + if (allowedFilters.includes(key)) { + clauses.push( + parse ? await parseFilter(key, value, model) : { [key]: value } + ); + } + } + return mergeFilterClauses(clauses); +} + +export { parseFilter, getFilter, getObjectTypeModel }; diff --git a/src/database/schemas/management/documentjob.schema.js b/src/database/schemas/management/documentjob.schema.js index 2e0a637..8fbaf7c 100644 --- a/src/database/schemas/management/documentjob.schema.js +++ b/src/database/schemas/management/documentjob.schema.js @@ -31,6 +31,12 @@ const documentJobSchema = new Schema( ref: 'documentPrinter', required: true, }, + quantity: { + type: Number, + required: true, + default: 1, + min: 1, + }, content: { type: String, required: false, diff --git a/src/database/utils.js b/src/database/utils.js index a23da52..47a7871 100644 --- a/src/database/utils.js +++ b/src/database/utils.js @@ -6,7 +6,11 @@ import { natsServer } from './nats.js'; import { customAlphabet } from 'nanoid'; -const NOTIFICATION_EXCLUDED_MODELS = ['notification', 'userNotifier', 'auditLog']; +const NOTIFICATION_EXCLUDED_MODELS = [ + 'notification', + 'userNotifier', + 'auditLog' +]; const SENSITIVE_KEYS = ['secret']; function omitSensitive(obj) { @@ -504,8 +508,12 @@ async function editNotification( const user = notificationUserFromOwner(owner, ownerType); const objectName = oldValue?.name ?? newValue?.name ?? modelEntry?.label ?? parentType; - const changedOldValues = omitSensitive(getChangedValues(oldValue, newValue, true)); - const changedNewValues = omitSensitive(getChangedValues(oldValue, newValue, false)); + const changedOldValues = omitSensitive( + getChangedValues(oldValue, newValue, true) + ); + const changedNewValues = omitSensitive( + getChangedValues(oldValue, newValue, false) + ); if ( Object.keys(changedOldValues).length === 0 || @@ -556,7 +564,13 @@ async function notfiyObjectUserNotifiers( } } -async function createNotification(user, title, message, type = 'info', metadata) { +async function createNotification( + user, + title, + message, + type = 'info', + metadata +) { const notification = new notificationModel({ user, title, @@ -637,17 +651,7 @@ function expandObjectIds(input) { return expand(input); } -// Returns a filter object based on allowed filters and req.query -function getFilter(query, allowedFilters, parse = true) { - let filter = {}; - for (const [key, value] of Object.entries(query)) { - if (allowedFilters.includes(key)) { - const parsedFilter = parse ? parseFilter(key, value) : { [key]: value }; - filter = { ...filter, ...parsedFilter }; - } - } - return filter; -} +export { getFilter } from './filter.js'; // Converts a properties argument (string or array) to an array of strings function convertPropertiesString(properties) { @@ -675,6 +679,5 @@ export { editNotification, notfiyObjectUserNotifiers, createNotification, - getFilter, // <-- add here convertPropertiesString }; diff --git a/src/updates/__tests__/updatemanager.test.js b/src/updates/__tests__/updatemanager.test.js index 5369338..b92cdab 100644 --- a/src/updates/__tests__/updatemanager.test.js +++ b/src/updates/__tests__/updatemanager.test.js @@ -28,8 +28,14 @@ jest.unstable_mockModule('../../config.js', () => ({ })) })); +jest.unstable_mockModule('../../database/filter.js', () => ({ + getFilter: jest.fn(async query => query), + getObjectTypeModel: jest.fn(async () => null) +})); + const { UpdateManager } = await import('../updatemanager.js'); const { natsServer } = await import('../../database/nats.js'); +const { getFilter } = await import('../../database/filter.js'); describe('UpdateManager', () => { let mockSocketClient; @@ -79,6 +85,12 @@ describe('UpdateManager', () => { await updateManager.subscribeToObjectNew('printer', filter); + expect(getFilter).toHaveBeenCalledWith( + filter, + Object.keys(filter), + true, + null + ); expect(natsServer.subscribe).toHaveBeenCalledWith( 'printers.new', 'test-socket-id:{"state.type":"ready"}', diff --git a/src/updates/updatemanager.js b/src/updates/updatemanager.js index 973d885..8bfdb20 100644 --- a/src/updates/updatemanager.js +++ b/src/updates/updatemanager.js @@ -3,6 +3,7 @@ import _ from 'lodash'; import { loadConfig } from '../config.js'; import { natsServer } from '../database/nats.js'; import { expandObjectIds } from '../database/utils.js'; +import { getFilter, getObjectTypeModel } from '../database/filter.js'; import { formatTraceData } from '../utils.js'; const config = loadConfig(); @@ -13,6 +14,18 @@ logger.level = config.server.logLevel; const normalizeFilter = filter => filter && typeof filter === 'object' && !Array.isArray(filter) ? filter : {}; +const unwrapId = value => { + if ( + value && + typeof value === 'object' && + !Array.isArray(value) && + value._id != null + ) { + return value._id; + } + return value; +}; + const getFilterValue = (object, key) => { if (key.endsWith('._id')) { const refPath = key.slice(0, -4); @@ -23,21 +36,94 @@ const getFilterValue = (object, key) => { return ref; } - return _.get(object, key); + return unwrapId(_.get(object, key)); }; const valuesMatch = (actual, expected) => { - if (actual == expected) { + const left = unwrapId(actual); + const right = unwrapId(expected); + + if (left == right) { return true; } - if (actual != null && expected != null) { - return String(actual) === String(expected); + if (left != null && right != null) { + return String(left) === String(right); } return false; }; +const isOperatorObject = value => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return false; + } + if (value instanceof Date || value._bsontype === 'ObjectId') { + return false; + } + const keys = Object.keys(value); + return keys.length > 0 && keys.every(key => key.startsWith('$')); +}; + +const compareValues = (actual, expected, operator) => { + if (actual == null || expected == null) { + return false; + } + if (operator === '$gt') return actual > expected; + if (operator === '$gte') return actual >= expected; + if (operator === '$lt') return actual < expected; + if (operator === '$lte') return actual <= expected; + return false; +}; + +const matchesOperators = (actual, operators) => { + if (Object.prototype.hasOwnProperty.call(operators, '$eq')) { + if (!valuesMatch(actual, operators.$eq)) return false; + } + if (Object.prototype.hasOwnProperty.call(operators, '$ne')) { + if (valuesMatch(actual, operators.$ne)) return false; + } + if (Object.prototype.hasOwnProperty.call(operators, '$in')) { + if ( + !Array.isArray(operators.$in) || + !operators.$in.some(value => valuesMatch(actual, value)) + ) { + return false; + } + } + if (Object.prototype.hasOwnProperty.call(operators, '$nin')) { + if ( + Array.isArray(operators.$nin) && + operators.$nin.some(value => valuesMatch(actual, value)) + ) { + return false; + } + } + if (Object.prototype.hasOwnProperty.call(operators, '$regex')) { + const regex = new RegExp(operators.$regex, operators.$options || ''); + if (!regex.test(String(actual ?? ''))) return false; + } + if (Object.prototype.hasOwnProperty.call(operators, '$not')) { + if (matchesExpected(actual, operators.$not)) return false; + } + for (const operator of ['$gt', '$gte', '$lt', '$lte']) { + if ( + Object.prototype.hasOwnProperty.call(operators, operator) && + !compareValues(actual, operators[operator], operator) + ) { + return false; + } + } + return true; +}; + +const matchesExpected = (actual, expected) => { + if (isOperatorObject(expected)) { + return matchesOperators(actual, expected); + } + return valuesMatch(actual, expected); +}; + const matchesFilter = (object, filter) => { if (!filter || Object.keys(filter).length === 0) { return true; @@ -52,8 +138,20 @@ const matchesFilter = (object, filter) => { ? object : { _id: object }; + if (Array.isArray(filter.$and)) { + return filter.$and.every(clause => matchesFilter(normalizedObject, clause)); + } + if (Array.isArray(filter.$or)) { + return filter.$or.some(clause => matchesFilter(normalizedObject, clause)); + } + for (const [key, expectedValue] of Object.entries(filter)) { - if (!valuesMatch(getFilterValue(normalizedObject, key), expectedValue)) { + if (key === '$and' || key === '$or') { + continue; + } + if ( + !matchesExpected(getFilterValue(normalizedObject, key), expectedValue) + ) { return false; } } @@ -61,6 +159,22 @@ const matchesFilter = (object, filter) => { return true; }; +const resolveObjectTypeFilter = async (objectType, filter = {}) => { + const normalizedFilter = normalizeFilter(filter); + if (Object.keys(normalizedFilter).length === 0) { + return { normalizedFilter, processedFilter: normalizedFilter }; + } + + const model = await getObjectTypeModel(objectType); + const processedFilter = await getFilter( + normalizedFilter, + Object.keys(normalizedFilter), + true, + model + ); + return { normalizedFilter, processedFilter }; +}; + const stableStringify = value => { if (Array.isArray(value)) { return `[${value.map(stableStringify).join(',')}]`; @@ -94,11 +208,17 @@ export class UpdateManager { return matchesFilter(value, normalizeFilter(filter)); } - emitObjectTypeEvent(eventName, objectType, filter, value) { + emitObjectTypeEvent( + eventName, + objectType, + filter, + value, + matchFilter = filter + ) { const normalizedFilter = normalizeFilter(filter); const matches = this.matchesObjectTypeFilter( objectType, - normalizedFilter, + matchFilter, value ); @@ -121,7 +241,10 @@ export class UpdateManager { } async subscribeToObjectNew(objectType, filter = {}) { - const normalizedFilter = normalizeFilter(filter); + const { normalizedFilter, processedFilter } = await resolveObjectTypeFilter( + objectType, + filter + ); const subject = `${objectType}s.new`; const owner = getSubscriptionOwner( this.socketClient.socketId, @@ -134,7 +257,8 @@ export class UpdateManager { 'objectNew', objectType, normalizedFilter, - value + value, + processedFilter ); }); @@ -143,7 +267,10 @@ export class UpdateManager { } async subscribeToObjectDelete(objectType, filter = {}) { - const normalizedFilter = normalizeFilter(filter); + const { normalizedFilter, processedFilter } = await resolveObjectTypeFilter( + objectType, + filter + ); const subject = `${objectType}s.delete`; const owner = getSubscriptionOwner( this.socketClient.socketId, @@ -156,7 +283,8 @@ export class UpdateManager { 'objectDelete', objectType, normalizedFilter, - value + value, + processedFilter ); });