Tom Butcher
e98828bfc7
Some checks failed
farmcontrol/farmcontrol-api/pipeline/head There was a failure building this commit
This commit modifies the log level in the configuration file from "trace" to "debug" for improved logging clarity. It also introduces new routes for fulfillment, return, and payment policies in the index file, enhancing the marketplace integration capabilities. Additionally, the initialization process is updated to start the marketplace worker, ensuring that the application can handle marketplace operations effectively. Various schemas are updated to include new fields and relationships for policies, improving the overall functionality and flexibility of the marketplace management system.
1740 lines
50 KiB
JavaScript
1740 lines
50 KiB
JavaScript
import config from '../config.js';
|
|
import { fileModel } from './schemas/management/file.schema.js';
|
|
import _ from 'lodash';
|
|
import log4js from 'log4js';
|
|
import crypto from 'crypto';
|
|
import sharp from 'sharp';
|
|
import { encode as encodeBlurHash } from 'blurhash';
|
|
import mongoose from 'mongoose';
|
|
import { uploadFile, deleteFile, downloadFile, BUCKETS } from './ceph.js';
|
|
import {
|
|
deleteAuditLog,
|
|
distributeDelete,
|
|
expandObjectIds,
|
|
modelHasRef,
|
|
getFieldsByRef,
|
|
getQueryToCacheKey,
|
|
editAuditLog,
|
|
distributeUpdate,
|
|
newAuditLog,
|
|
distributeNew,
|
|
distributeChildUpdate,
|
|
distributeChildDelete,
|
|
distributeChildNew,
|
|
distributeStats,
|
|
editNotification,
|
|
deleteNotification,
|
|
flatternObjectIds,
|
|
} from '../utils.js';
|
|
import { getAllModels, getModelByName } from '../services/misc/model.js';
|
|
import { redisServer } from './redis.js';
|
|
import { auditLogModel } from './schemas/management/auditlog.schema.js';
|
|
import { convertObjectIdStringsInFilter } from './utils.js';
|
|
|
|
const BASIC_OBJECT_FIELDS = '_id _reference name color state';
|
|
|
|
const logger = log4js.getLogger('Database');
|
|
logger.level = config.server.logLevel;
|
|
|
|
const cacheLogger = log4js.getLogger('DatabaseCache');
|
|
cacheLogger.level = config.server.logLevel;
|
|
|
|
const CACHE_TTL_SECONDS = parseInt(config.database.redis.cacheTtl || '30', 10);
|
|
const NEIGHBORS_CACHE_TTL_SECONDS = 60;
|
|
const NEIGHBORS_CACHE_WINDOW = 25;
|
|
const NEIGHBORS_CACHE_PREFIX = 'neighbors';
|
|
|
|
const isMergeableObject = (value) => _.isPlainObject(value);
|
|
|
|
const mergeObjectUpdates = (target, source) => {
|
|
if (!isMergeableObject(source)) {
|
|
return source;
|
|
}
|
|
|
|
if (!isMergeableObject(target)) {
|
|
target = {};
|
|
}
|
|
|
|
for (const key of Object.keys(source)) {
|
|
const srcValue = source[key];
|
|
const objValue = target[key];
|
|
|
|
// Key exists on source (including explicit undefined) — use source.
|
|
// Keys omitted from source keep the target value.
|
|
if (srcValue === undefined) {
|
|
target[key] = undefined;
|
|
continue;
|
|
}
|
|
|
|
if (Array.isArray(objValue) || Array.isArray(srcValue)) {
|
|
target[key] = srcValue;
|
|
} else if (key === 'permissions') {
|
|
target[key] = srcValue;
|
|
} else if (isMergeableObject(objValue) && isMergeableObject(srcValue)) {
|
|
mergeObjectUpdates(objValue, srcValue);
|
|
} else {
|
|
target[key] = srcValue;
|
|
}
|
|
}
|
|
|
|
return target;
|
|
};
|
|
|
|
export const retrieveObjectCache = async ({ model, id, populate = [] }) => {
|
|
if (!model || !id) return undefined;
|
|
|
|
const cacheKey = getQueryToCacheKey({ model: model.modelName, id, populate });
|
|
|
|
cacheLogger.trace('Retrieving object from cache:', { model: model.modelName, id, populate });
|
|
|
|
try {
|
|
const cachedObject = await redisServer.getKey(cacheKey);
|
|
if (cachedObject == null) {
|
|
cacheLogger.trace('Cache miss:', { model: model.modelName, id });
|
|
return undefined;
|
|
}
|
|
|
|
cacheLogger.trace('Cache hit:', {
|
|
model: model.modelName,
|
|
id: id.toString(),
|
|
});
|
|
|
|
return cachedObject;
|
|
} catch (err) {
|
|
cacheLogger.error('Error retrieving object from Redis cache:', err);
|
|
return undefined;
|
|
}
|
|
};
|
|
|
|
export const updateObjectCache = async ({ model, id, object, populate = [] }) => {
|
|
if (!model || !id || !object) return object;
|
|
|
|
const cacheKeyFilter = `${model.modelName}:${id?.toString()}*`;
|
|
const cacheKey = getQueryToCacheKey({ model: model.modelName, id, populate });
|
|
|
|
cacheLogger.trace('Updating object cache:', cacheKeyFilter);
|
|
|
|
try {
|
|
// Get all keys matching the filter pattern
|
|
const matchingKeys = await redisServer.getKeysByPattern(cacheKeyFilter);
|
|
// Merge the object with each cached object and update
|
|
const mergedObjects = [];
|
|
for (const key of matchingKeys) {
|
|
logger.trace('Updating object cache:', key);
|
|
const cachedObject = (await redisServer.getKey(key)) || {};
|
|
const mergedObject = mergeObjectUpdates(cachedObject, object);
|
|
await redisServer.setKey(key, mergedObject, CACHE_TTL_SECONDS);
|
|
mergedObjects.push(mergedObject);
|
|
}
|
|
|
|
const cacheObject = (await redisServer.getKey(cacheKey)) || {};
|
|
const mergedObject = mergeObjectUpdates(cacheObject, object);
|
|
await redisServer.setKey(cacheKey, mergedObject, CACHE_TTL_SECONDS);
|
|
|
|
cacheLogger.trace('Updated object cache:', {
|
|
filter: cacheKeyFilter,
|
|
keysUpdated: matchingKeys.length,
|
|
});
|
|
|
|
// Return the merged object
|
|
return mergedObject;
|
|
} catch (err) {
|
|
cacheLogger.error('Error updating object in Redis cache:', err);
|
|
// Fallback to returning the provided object if cache fails
|
|
return object;
|
|
}
|
|
};
|
|
|
|
export const deleteObjectCache = async ({ model, id }) => {
|
|
if (!model || !id) return;
|
|
|
|
const cacheKeyFilter = `${model.modelName}:${id?.toString()}*`;
|
|
|
|
cacheLogger.trace('Deleting object cache:', cacheKeyFilter);
|
|
|
|
try {
|
|
// Get all keys matching the filter pattern and delete them
|
|
const matchingKeys = await redisServer.getKeysByPattern(cacheKeyFilter);
|
|
|
|
for (const cacheKey of matchingKeys) {
|
|
await redisServer.deleteKey(cacheKey);
|
|
}
|
|
|
|
cacheLogger.trace('Deleted object cache:', {
|
|
filter: cacheKeyFilter,
|
|
keysDeleted: matchingKeys.length,
|
|
});
|
|
} catch (err) {
|
|
cacheLogger.error('Error deleting object from Redis cache:', err);
|
|
}
|
|
};
|
|
|
|
const buildNeighborsQueryFilter = (filter = {}, search = '') => {
|
|
const queryFilter = { ...filter };
|
|
|
|
Object.keys(queryFilter).forEach((key) => {
|
|
if (key.endsWith('._id')) {
|
|
const baseKey = key.slice(0, -4);
|
|
queryFilter[baseKey] = queryFilter[key];
|
|
delete queryFilter[key];
|
|
}
|
|
});
|
|
|
|
if (search) {
|
|
const isRefOrIdSearch = search.match(/^.{3}:/);
|
|
if (isRefOrIdSearch) {
|
|
const lookupValue = search.split(':')[1];
|
|
if (lookupValue.length <= 12) {
|
|
queryFilter._reference = lookupValue;
|
|
} else {
|
|
queryFilter._id = lookupValue;
|
|
}
|
|
} else {
|
|
queryFilter.$text = { $search: search };
|
|
}
|
|
}
|
|
|
|
return queryFilter;
|
|
};
|
|
|
|
const getNeighborsCacheKey = ({
|
|
model,
|
|
id,
|
|
filter = {},
|
|
search = '',
|
|
sort = '',
|
|
order = 'ascend',
|
|
}) => {
|
|
const normalizedSort = sort || 'createdAt';
|
|
const normalizedOrder = order || 'ascend';
|
|
const queryHash = crypto
|
|
.createHash('sha256')
|
|
.update(
|
|
JSON.stringify({
|
|
filter,
|
|
search,
|
|
sort: normalizedSort,
|
|
order: normalizedOrder,
|
|
})
|
|
)
|
|
.digest('hex');
|
|
|
|
return `${NEIGHBORS_CACHE_PREFIX}:${model.modelName}:${id?.toString()}:${queryHash}`;
|
|
};
|
|
|
|
const neighborsCacheToResponse = (cached) =>
|
|
expandObjectIds({
|
|
next: cached.next[0] ? { _id: cached.next[0] } : null,
|
|
previous: cached.previous[0] ? { _id: cached.previous[0] } : null,
|
|
});
|
|
|
|
const buildNeighborsCacheValue = (ids, index) => {
|
|
const previous = ids
|
|
.slice(Math.max(0, index - NEIGHBORS_CACHE_WINDOW), index)
|
|
.map((neighborId) => String(neighborId))
|
|
.reverse();
|
|
const next = ids
|
|
.slice(index + 1, index + 1 + NEIGHBORS_CACHE_WINDOW)
|
|
.map((neighborId) => String(neighborId));
|
|
|
|
return { previous, next };
|
|
};
|
|
|
|
export const invalidateNeighborsCacheForObject = async ({ model, id }) => {
|
|
if (!model || !id) return;
|
|
|
|
const objectId = id?.toString();
|
|
const cacheKeyPrefix = `${NEIGHBORS_CACHE_PREFIX}:${model.modelName}:`;
|
|
const cacheKeyForObject = `${cacheKeyPrefix}${objectId}:`;
|
|
|
|
cacheLogger.trace('Invalidating neighbors cache for object:', {
|
|
model: model.modelName,
|
|
id: objectId,
|
|
});
|
|
|
|
try {
|
|
const matchingKeys = await redisServer.getKeysByPattern(`${cacheKeyPrefix}*`);
|
|
|
|
for (const cacheKey of matchingKeys) {
|
|
if (cacheKey.startsWith(cacheKeyForObject)) {
|
|
await redisServer.deleteKey(cacheKey);
|
|
continue;
|
|
}
|
|
|
|
const cached = await redisServer.getKey(cacheKey);
|
|
if (!cached) continue;
|
|
|
|
const containsId =
|
|
(Array.isArray(cached.previous) && cached.previous.includes(objectId)) ||
|
|
(Array.isArray(cached.next) && cached.next.includes(objectId));
|
|
|
|
if (containsId) {
|
|
await redisServer.deleteKey(cacheKey);
|
|
}
|
|
}
|
|
|
|
cacheLogger.trace('Invalidated neighbors cache for object:', {
|
|
model: model.modelName,
|
|
id: objectId,
|
|
});
|
|
} catch (err) {
|
|
cacheLogger.error('Error invalidating neighbors cache:', err);
|
|
}
|
|
};
|
|
|
|
// Utility to run one or many rollup aggregations in a single query via $facet.
|
|
export const aggregateRollups = async ({ model, baseFilter = {}, rollupConfigs = [] }) => {
|
|
if (!rollupConfigs.length) {
|
|
return {};
|
|
}
|
|
|
|
const facetStage = rollupConfigs.reduce((facets, definition, index) => {
|
|
const key = definition.name || `rollup${index}`;
|
|
const matchStage = { $match: { ...baseFilter, ...(definition.filter || {}) } };
|
|
const groupStage = { $group: { _id: null } };
|
|
|
|
(definition.rollups || []).forEach((rollup) => {
|
|
switch (rollup.operation) {
|
|
case 'sum':
|
|
groupStage.$group[rollup.name] = { $sum: `$${rollup.property}` };
|
|
break;
|
|
case 'count':
|
|
groupStage.$group[rollup.name] = { $sum: 1 };
|
|
break;
|
|
case 'avg':
|
|
groupStage.$group[rollup.name] = { $avg: `$${rollup.property}` };
|
|
break;
|
|
default:
|
|
throw new Error(`Unsupported rollup operation: ${rollup.operation}`);
|
|
}
|
|
});
|
|
|
|
facets[key] = [matchStage, groupStage];
|
|
return facets;
|
|
}, {});
|
|
|
|
const [results] = await model.aggregate([{ $facet: facetStage }]);
|
|
|
|
return rollupConfigs.reduce((acc, definition, index) => {
|
|
const key = definition.name || `rollup${index}`;
|
|
const rawResult = results?.[key]?.[0] || {};
|
|
|
|
// Transform the result to nest rollup values under operation type
|
|
const transformedResult = {};
|
|
(definition.rollups || []).forEach((rollup) => {
|
|
const value = rawResult[rollup.name] || 0;
|
|
// If there's only one rollup and its name matches the key, flatten the structure
|
|
if (definition.rollups.length === 1 && rollup.name === key) {
|
|
transformedResult[rollup.operation] = value;
|
|
} else {
|
|
transformedResult[rollup.name] = { [rollup.operation]: value };
|
|
}
|
|
});
|
|
|
|
acc[key] = transformedResult;
|
|
return acc;
|
|
}, {});
|
|
};
|
|
|
|
// Snapshot absolute rollup values at each point in time by reconstructing
|
|
// object state from the current documents plus audit logs.
|
|
export const aggregateRollupsHistory = async ({
|
|
model,
|
|
baseFilter = {},
|
|
rollupConfigs = [],
|
|
startDate,
|
|
endDate,
|
|
}) => {
|
|
if (!rollupConfigs.length) {
|
|
return [];
|
|
}
|
|
|
|
const end = endDate ? new Date(endDate) : new Date();
|
|
const start = startDate ? new Date(startDate) : new Date(end.getTime() - 24 * 60 * 60 * 1000);
|
|
const parentType = model.modelName ? model.modelName : 'unknown';
|
|
|
|
const matchesFilter = (obj, filter) => {
|
|
if (!filter || Object.keys(filter).length === 0) return true;
|
|
|
|
for (const [key, expectedValue] of Object.entries(filter)) {
|
|
const actualValue = _.get(obj, key);
|
|
if (actualValue != expectedValue) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
};
|
|
|
|
const existedAt = (obj, bucketDate) => {
|
|
if (!obj?.createdAt) return true;
|
|
return new Date(obj.createdAt) <= bucketDate;
|
|
};
|
|
|
|
const snapshotRollups = (objects, bucketDate) => {
|
|
const bucketResult = {
|
|
date: bucketDate.toISOString(),
|
|
};
|
|
|
|
rollupConfigs.forEach((config) => {
|
|
const matchingObjects = objects.filter(
|
|
(obj) =>
|
|
existedAt(obj, bucketDate) &&
|
|
matchesFilter(obj, baseFilter) &&
|
|
matchesFilter(obj, config.filter)
|
|
);
|
|
|
|
(config.rollups || []).forEach((rollup) => {
|
|
let value = 0;
|
|
if (rollup.operation === 'count') {
|
|
value = matchingObjects.length;
|
|
} else if (rollup.operation === 'sum') {
|
|
value = _.sumBy(matchingObjects, (obj) => _.get(obj, rollup.property) || 0);
|
|
} else if (rollup.operation === 'avg') {
|
|
const sum = _.sumBy(matchingObjects, (obj) => _.get(obj, rollup.property) || 0);
|
|
value = matchingObjects.length ? sum / matchingObjects.length : 0;
|
|
}
|
|
|
|
bucketResult[rollup.name] = { [rollup.operation]: value };
|
|
});
|
|
});
|
|
|
|
return bucketResult;
|
|
};
|
|
|
|
const auditLogs = await auditLogModel
|
|
.find({
|
|
parentType,
|
|
createdAt: { $gte: start },
|
|
})
|
|
.sort({ createdAt: -1 })
|
|
.lean();
|
|
|
|
const currentObjects = await model.find(baseFilter).lean();
|
|
const objectMap = new Map();
|
|
currentObjects.forEach((obj) => {
|
|
objectMap.set(obj._id.toString(), expandObjectIds(obj));
|
|
});
|
|
|
|
const extraIds = [
|
|
...new Set(
|
|
auditLogs.map((log) => log.parent?.toString()).filter((id) => id && !objectMap.has(id))
|
|
),
|
|
];
|
|
if (extraIds.length) {
|
|
const extraObjects = await model.find({ _id: { $in: extraIds } }).lean();
|
|
extraObjects.forEach((obj) => {
|
|
objectMap.set(obj._id.toString(), expandObjectIds(obj));
|
|
});
|
|
}
|
|
|
|
if (objectMap.size === 0 && auditLogs.length === 0) {
|
|
return [];
|
|
}
|
|
|
|
const buckets = [];
|
|
let currentTime = new Date(end);
|
|
currentTime.setSeconds(0, 0);
|
|
|
|
while (currentTime >= start) {
|
|
buckets.push(new Date(currentTime));
|
|
currentTime = new Date(currentTime.getTime() - 60000);
|
|
}
|
|
|
|
if (!buckets.length) {
|
|
return [];
|
|
}
|
|
|
|
const workingObjects = new Map();
|
|
objectMap.forEach((val, key) => workingObjects.set(key, _.cloneDeep(val)));
|
|
|
|
const results = [];
|
|
let logIndex = 0;
|
|
|
|
for (const bucketDate of buckets) {
|
|
while (logIndex < auditLogs.length) {
|
|
const log = auditLogs[logIndex];
|
|
const logDate = new Date(log.createdAt);
|
|
|
|
if (logDate <= bucketDate) {
|
|
break;
|
|
}
|
|
|
|
const objectId = log.parent.toString();
|
|
const object = workingObjects.get(objectId);
|
|
|
|
if (log.operation === 'new') {
|
|
workingObjects.delete(objectId);
|
|
} else if (log.operation === 'delete' && log.changes?.old) {
|
|
if (!workingObjects.has(objectId)) {
|
|
workingObjects.set(objectId, expandObjectIds({ ...log.changes.old, _id: log.parent }));
|
|
}
|
|
} else if (object && log.changes?.old) {
|
|
mergeObjectUpdates(object, log.changes.old);
|
|
}
|
|
|
|
logIndex++;
|
|
}
|
|
|
|
results.push(snapshotRollups(Array.from(workingObjects.values()), bucketDate));
|
|
}
|
|
|
|
return results.reverse();
|
|
};
|
|
|
|
// Reusable function to list objects with aggregation, filtering, search, sorting, and pagination
|
|
export const listObjects = async ({
|
|
model,
|
|
populate = [],
|
|
page = 1,
|
|
limit = 25,
|
|
filter = {},
|
|
sort = '',
|
|
order = 'ascend',
|
|
pagination = true,
|
|
project, // optional: override default projection
|
|
}) => {
|
|
try {
|
|
logger.trace('Listing object:', {
|
|
model,
|
|
populate,
|
|
pagination,
|
|
page,
|
|
limit,
|
|
filter,
|
|
sort,
|
|
order,
|
|
project,
|
|
});
|
|
// Calculate the skip value based on the page number and limit
|
|
const skip = pagination ? (page - 1) * limit : 0;
|
|
// Fix: descend should be -1, ascend should be 1
|
|
const sortOrder = order === 'descend' ? -1 : 1;
|
|
|
|
if (!sort || sort === '') {
|
|
sort = 'createdAt';
|
|
}
|
|
|
|
// Translate any key ending with ._id to remove the ._id suffix for Mongoose
|
|
Object.keys(filter).forEach((key) => {
|
|
if (key.endsWith('._id')) {
|
|
const baseKey = key.slice(0, -4); // Remove '._id' suffix
|
|
filter[baseKey] = filter[key];
|
|
delete filter[key];
|
|
}
|
|
});
|
|
|
|
// Use find with population and filter
|
|
let query = model
|
|
.find(filter)
|
|
.sort({ [sort]: sortOrder })
|
|
.skip(skip)
|
|
.limit(pagination ? Number(limit) : undefined);
|
|
|
|
// Handle populate (array or single value)
|
|
if (populate) {
|
|
if (Array.isArray(populate)) {
|
|
for (const pop of populate) {
|
|
query = query.populate(pop);
|
|
}
|
|
} else if (typeof populate === 'string' || typeof populate === 'object') {
|
|
query = query.populate(populate);
|
|
}
|
|
}
|
|
|
|
// Handle select (projection)
|
|
if (project) {
|
|
query = query.select(project);
|
|
}
|
|
|
|
query = query.lean();
|
|
|
|
const queryResult = await query;
|
|
return expandObjectIds(queryResult);
|
|
} catch (error) {
|
|
logger.error('Object list error:', error);
|
|
return { error: error, code: 500 };
|
|
}
|
|
};
|
|
|
|
export const getObjectNeighbors = async ({
|
|
model,
|
|
id,
|
|
filter = {},
|
|
search = '',
|
|
sort = '',
|
|
order = 'ascend',
|
|
}) => {
|
|
try {
|
|
const sortOrder = order === 'descend' ? -1 : 1;
|
|
const sortField = sort || 'createdAt';
|
|
const cacheKey = getNeighborsCacheKey({ model, id, filter, search, sort: sortField, order });
|
|
|
|
cacheLogger.trace('Retrieving neighbors from cache:', {
|
|
model: model.modelName,
|
|
id: id?.toString(),
|
|
});
|
|
|
|
const cached = await redisServer.getKey(cacheKey);
|
|
if (cached) {
|
|
cacheLogger.trace('Neighbors cache hit:', {
|
|
model: model.modelName,
|
|
id: id?.toString(),
|
|
});
|
|
return neighborsCacheToResponse(cached);
|
|
}
|
|
|
|
cacheLogger.trace('Neighbors cache miss:', {
|
|
model: model.modelName,
|
|
id: id?.toString(),
|
|
});
|
|
|
|
const queryFilter = buildNeighborsQueryFilter(filter, search);
|
|
|
|
const results = await model
|
|
.find(queryFilter)
|
|
.sort({ [sortField]: sortOrder })
|
|
.select('_id')
|
|
.lean();
|
|
|
|
const ids = results.map((doc) => doc._id);
|
|
const index = ids.findIndex((docId) => String(docId) === String(id));
|
|
|
|
if (index === -1) {
|
|
return { next: null, previous: null };
|
|
}
|
|
|
|
const cacheValue = buildNeighborsCacheValue(ids, index);
|
|
await redisServer.setKey(cacheKey, cacheValue, NEIGHBORS_CACHE_TTL_SECONDS);
|
|
|
|
return neighborsCacheToResponse(cacheValue);
|
|
} catch (error) {
|
|
logger.error('Object neighbors error:', error);
|
|
return { error: error, code: 500 };
|
|
}
|
|
};
|
|
|
|
export const searchObjects = async ({ model, search, populate = [] }) => {
|
|
try {
|
|
const isRefOrIdSearch = search.match(/^.{3}:/);
|
|
let query = null;
|
|
if (isRefOrIdSearch) {
|
|
const lookupValue = search.split(':')[1];
|
|
if (lookupValue.length <= 12) {
|
|
query = model.find({ _reference: lookupValue });
|
|
} else {
|
|
query = model.find({ _id: lookupValue });
|
|
}
|
|
} else {
|
|
query = model.find({ $text: { $search: search } });
|
|
}
|
|
if (populate) {
|
|
for (const pop of populate) {
|
|
query = query.populate(pop);
|
|
}
|
|
}
|
|
const queryResult = await query.lean();
|
|
const result = queryResult.map((item) => expandObjectIds(item));
|
|
return result;
|
|
} catch (error) {
|
|
logger.error('Object search error:', error);
|
|
return { error: error, code: 500 };
|
|
}
|
|
};
|
|
|
|
// New function to list unique property values
|
|
export const listPropertyValues = async ({ model, property, filter = {}, search = '' }) => {
|
|
let aggregateCommand = [];
|
|
|
|
if (search) {
|
|
aggregateCommand.push({
|
|
$match: {
|
|
$text: { $search: search },
|
|
},
|
|
});
|
|
}
|
|
|
|
if (filter && Object.keys(filter).length > 0) {
|
|
aggregateCommand.push({ $match: filter });
|
|
}
|
|
|
|
aggregateCommand.push({ $group: { _id: `$${property}` } });
|
|
aggregateCommand.push({ $project: { _id: 0, [property]: '$_id' } });
|
|
|
|
return await model.aggregate(aggregateCommand);
|
|
};
|
|
|
|
function getEmbeddedSchemaType(path) {
|
|
if (!path) return null;
|
|
return path.embeddedSchemaType ?? path.$embeddedSchemaType ?? path.caster ?? null;
|
|
}
|
|
|
|
function getObjectRefPathInfo(model, property) {
|
|
if (!model?.schema || !property) return null;
|
|
|
|
const path = model.schema.path(property);
|
|
if (!path) return null;
|
|
|
|
const schemaType = path.instance === 'Array' ? getEmbeddedSchemaType(path) : path;
|
|
if (!schemaType) return null;
|
|
|
|
const instance = schemaType.instance;
|
|
if (instance !== 'ObjectID' && instance !== 'ObjectId') return null;
|
|
|
|
return {
|
|
ref: schemaType.options?.ref ?? path.options?.ref,
|
|
refPath: schemaType.options?.refPath ?? path.options?.refPath,
|
|
};
|
|
}
|
|
|
|
function resolveMongooseModel(refName) {
|
|
if (!refName || typeof refName !== 'string') return null;
|
|
try {
|
|
return mongoose.model(refName);
|
|
} catch {
|
|
return getModelByName(refName)?.model ?? null;
|
|
}
|
|
}
|
|
|
|
function inferTypeFieldFromRefFunction(refFn) {
|
|
if (typeof refFn !== 'function') return null;
|
|
const matches = [...refFn.toString().matchAll(/this\.(\w+)/g)].map((m) => m[1]);
|
|
return matches.find((name) => name.endsWith('Type') || name === 'type') || matches[0] || null;
|
|
}
|
|
|
|
function getArrayUnwindPaths(model, property) {
|
|
if (!model?.schema || !property) return [];
|
|
|
|
const paths = [];
|
|
const parts = property.split('.');
|
|
for (let i = 1; i < parts.length; i++) {
|
|
const parentPath = parts.slice(0, i).join('.');
|
|
const parentSchemaPath = model.schema.path(parentPath);
|
|
if (parentSchemaPath?.instance === 'Array') {
|
|
paths.push(parentPath);
|
|
}
|
|
}
|
|
|
|
const path = model.schema.path(property);
|
|
if (path?.instance === 'Array') {
|
|
paths.push(property);
|
|
}
|
|
return paths;
|
|
}
|
|
|
|
function flattenRefIds(ids) {
|
|
const flattened = [];
|
|
const visit = (value) => {
|
|
if (value == null) return;
|
|
if (Array.isArray(value)) {
|
|
value.forEach(visit);
|
|
return;
|
|
}
|
|
if (
|
|
typeof value === 'object' &&
|
|
!(value instanceof mongoose.Types.ObjectId) &&
|
|
!(value instanceof Date) &&
|
|
value._id != null
|
|
) {
|
|
visit(value._id);
|
|
return;
|
|
}
|
|
flattened.push(value);
|
|
};
|
|
visit(ids);
|
|
return flattened;
|
|
}
|
|
|
|
async function fetchBasicObjectsByIds(refName, ids) {
|
|
const uniqueIds = [...new Map(flattenRefIds(ids).map((id) => [id.toString(), id])).values()];
|
|
if (uniqueIds.length === 0) return [];
|
|
|
|
const withObjectType = (value) => {
|
|
const expanded = expandObjectIds(value);
|
|
const base =
|
|
expanded && typeof expanded === 'object' && !Array.isArray(expanded)
|
|
? expanded
|
|
: { _id: expanded };
|
|
return { ...base, objectType: refName };
|
|
};
|
|
|
|
const refModel = resolveMongooseModel(refName);
|
|
if (!refModel) {
|
|
return uniqueIds.map((id) => withObjectType(id));
|
|
}
|
|
|
|
const objects = await refModel
|
|
.find({ _id: { $in: uniqueIds } })
|
|
.select(BASIC_OBJECT_FIELDS)
|
|
.lean();
|
|
|
|
const foundById = new Map(objects.map((obj) => [obj._id.toString(), obj]));
|
|
return uniqueIds.map((id) => withObjectType(foundById.get(id.toString()) || id));
|
|
}
|
|
|
|
export const getPropertyValues = async ({ model, property, filter = {} }) => {
|
|
if (!property) {
|
|
return [];
|
|
}
|
|
|
|
const convertedFilter =
|
|
filter && Object.keys(filter).length > 0 ? convertObjectIdStringsInFilter(filter) : {};
|
|
|
|
if (property === 'state') {
|
|
return (await model.distinct('state.type', convertedFilter)).map((type) => ({ type }));
|
|
}
|
|
|
|
const pathInfo = getObjectRefPathInfo(model, property);
|
|
const hasFixedRef = typeof pathInfo?.ref === 'string';
|
|
const hasDynamicRef = Boolean(pathInfo?.refPath) || typeof pathInfo?.ref === 'function';
|
|
|
|
if (hasDynamicRef) {
|
|
const typeField = pathInfo.refPath || inferTypeFieldFromRefFunction(pathInfo.ref);
|
|
if (typeField) {
|
|
const pipeline = [];
|
|
if (Object.keys(convertedFilter).length > 0) {
|
|
pipeline.push({ $match: convertedFilter });
|
|
}
|
|
|
|
for (const unwindPath of getArrayUnwindPaths(model, property)) {
|
|
pipeline.push({
|
|
$unwind: {
|
|
path: `$${unwindPath}`,
|
|
preserveNullAndEmptyArrays: false,
|
|
},
|
|
});
|
|
}
|
|
|
|
pipeline.push(
|
|
{ $match: { [property]: { $ne: null }, [typeField]: { $ne: null } } },
|
|
{ $group: { _id: { id: `$${property}`, type: `$${typeField}` } } }
|
|
);
|
|
|
|
const pairs = await model.aggregate(pipeline);
|
|
const idsByRef = new Map();
|
|
|
|
for (const pair of pairs) {
|
|
const id = pair?._id?.id;
|
|
const typeValue = pair?._id?.type;
|
|
if (id == null || typeValue == null) continue;
|
|
|
|
let refName = typeValue;
|
|
if (typeof pathInfo.ref === 'function') {
|
|
refName = pathInfo.ref.call({ [typeField]: typeValue });
|
|
}
|
|
if (!refName) continue;
|
|
|
|
if (!idsByRef.has(refName)) idsByRef.set(refName, []);
|
|
idsByRef.get(refName).push(id);
|
|
}
|
|
|
|
const results = [];
|
|
for (const [refName, ids] of idsByRef) {
|
|
results.push(...(await fetchBasicObjectsByIds(refName, ids)));
|
|
}
|
|
return results;
|
|
}
|
|
}
|
|
|
|
if (hasFixedRef) {
|
|
const unwindPaths = getArrayUnwindPaths(model, property);
|
|
if (unwindPaths.length > 0) {
|
|
const pipeline = [];
|
|
if (Object.keys(convertedFilter).length > 0) {
|
|
pipeline.push({ $match: convertedFilter });
|
|
}
|
|
for (const unwindPath of unwindPaths) {
|
|
pipeline.push({
|
|
$unwind: {
|
|
path: `$${unwindPath}`,
|
|
preserveNullAndEmptyArrays: false,
|
|
},
|
|
});
|
|
}
|
|
pipeline.push({ $match: { [property]: { $ne: null } } }, { $group: { _id: `$${property}` } });
|
|
const ids = (await model.aggregate(pipeline)).map((row) => row._id);
|
|
return fetchBasicObjectsByIds(pathInfo.ref, ids);
|
|
}
|
|
|
|
const ids = await model.distinct(property, convertedFilter);
|
|
return fetchBasicObjectsByIds(pathInfo.ref, ids);
|
|
}
|
|
|
|
const result = await model.distinct(property, convertedFilter);
|
|
return result.map((item) => expandObjectIds(item));
|
|
};
|
|
|
|
// Helper to build nested structure for listObjectsByProperty
|
|
function nestGroups(groups, props, filter, idx = 0) {
|
|
if (idx >= props.length) return groups;
|
|
const prop = props[idx];
|
|
const filterPresent = Object.prototype.hasOwnProperty.call(filter, prop);
|
|
|
|
// Helper to extract a display key and possible filter values from a value
|
|
function getKeyAndFilterVals(value) {
|
|
if (value && typeof value === 'object') {
|
|
if (value.name) return { key: value.name, filterVals: [value._id?.toString?.(), value.name] };
|
|
if (value._id) return { key: value._id.toString(), filterVals: [value._id.toString()] };
|
|
}
|
|
return { key: value, filterVals: [value] };
|
|
}
|
|
|
|
// Build a map of key -> groups for this property
|
|
const keyToGroups = {};
|
|
for (const group of groups) {
|
|
const val = group._id[prop];
|
|
const { key } = getKeyAndFilterVals(val);
|
|
if (!keyToGroups[key]) {
|
|
keyToGroups[key] = {
|
|
groups: [],
|
|
value: val,
|
|
};
|
|
}
|
|
keyToGroups[key].groups.push(group);
|
|
}
|
|
|
|
let result = [];
|
|
|
|
if (filterPresent) {
|
|
const filterValue = filter[prop]?.toString?.() ?? filter[prop];
|
|
for (const [key, data] of Object.entries(keyToGroups)) {
|
|
const { groups: groupList, value } = data;
|
|
// Check if any group in this key matches the filter (by _id or name)
|
|
const matches = groupList.filter((group) => {
|
|
const { filterVals } = getKeyAndFilterVals(group._id[prop]);
|
|
return filterVals.some((val) => val?.toString() === filterValue);
|
|
});
|
|
|
|
let children = [];
|
|
if (matches.length > 0) {
|
|
if (idx === props.length - 1) {
|
|
// Last property in filter, return items
|
|
for (const group of matches) {
|
|
children = children.concat(group.objects.map(expandObjectIds));
|
|
}
|
|
} else {
|
|
children = nestGroups(matches, props, filter, idx + 1);
|
|
}
|
|
}
|
|
|
|
result.push({
|
|
property: prop,
|
|
value: expandObjectIds(value),
|
|
children: children,
|
|
});
|
|
}
|
|
} else {
|
|
// No filter for this property, just show all keys at this level with empty objects
|
|
for (const [key, data] of Object.entries(keyToGroups)) {
|
|
result.push({
|
|
property: prop,
|
|
value: expandObjectIds(data.value),
|
|
children: [],
|
|
});
|
|
}
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Group objects by multiple properties and return nested groupings
|
|
export const listObjectsByProperties = async ({
|
|
model,
|
|
properties = [],
|
|
filter = {},
|
|
masterFilter = {},
|
|
populate,
|
|
}) => {
|
|
try {
|
|
const propertiesPresent = !(
|
|
!Array.isArray(properties) ||
|
|
properties.length === 0 ||
|
|
properties[0] == ''
|
|
);
|
|
|
|
// Build aggregation pipeline
|
|
const pipeline = [];
|
|
|
|
// Match before populate so reference fields (e.g. filament) are still ObjectIds
|
|
if (Object.keys(masterFilter).length > 0) {
|
|
const convertedFilter = convertObjectIdStringsInFilter(masterFilter);
|
|
pipeline.push({ $match: convertedFilter });
|
|
}
|
|
|
|
// Handle populate (array or single value)
|
|
if (populate) {
|
|
const populates = Array.isArray(populate) ? populate : [populate];
|
|
for (const pop of populates) {
|
|
// Support both string and object syntax for populate
|
|
if (typeof pop === 'string') {
|
|
pipeline.push({
|
|
$lookup: {
|
|
from: pop.toLowerCase() + 's', // crude pluralization, adjust if needed
|
|
localField: pop,
|
|
foreignField: '_id',
|
|
as: pop,
|
|
},
|
|
});
|
|
// Unwind if it's a single reference
|
|
pipeline.push({
|
|
$unwind: {
|
|
path: `$${pop}`,
|
|
preserveNullAndEmptyArrays: true,
|
|
},
|
|
});
|
|
} else if (typeof pop === 'object' && pop.path) {
|
|
pipeline.push({
|
|
$lookup: {
|
|
from: pop.from ? pop.from : pop.path.toLowerCase(),
|
|
localField: pop.path,
|
|
foreignField: '_id',
|
|
as: pop.path,
|
|
},
|
|
});
|
|
if (pop?.multiple == false || pop?.multiple == undefined) {
|
|
// default to unwind unless justOne is explicitly false
|
|
pipeline.push({
|
|
$unwind: {
|
|
path: `$${pop.path}`,
|
|
preserveNullAndEmptyArrays: true,
|
|
},
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (propertiesPresent) {
|
|
// Build the $group _id object for all properties
|
|
const groupId = {};
|
|
for (const prop of properties) {
|
|
groupId[prop] = `$${prop}`;
|
|
}
|
|
pipeline.push({
|
|
$group: {
|
|
_id: groupId,
|
|
objects: { $push: '$$ROOT' },
|
|
},
|
|
});
|
|
|
|
// Run aggregation
|
|
const results = await model.aggregate(pipeline);
|
|
return nestGroups(results, properties, filter);
|
|
} else {
|
|
// If no properties specified, just return all objects without grouping
|
|
// Ensure pipeline is not empty by adding a $match stage if needed
|
|
if (pipeline.length === 0 && Object.keys(masterFilter).length === 0) {
|
|
pipeline.push({ $match: {} });
|
|
}
|
|
const results = await model.aggregate(pipeline);
|
|
return results;
|
|
}
|
|
} catch (error) {
|
|
logger.error('listObjectsByProperty error:', error);
|
|
return { error: error.message, code: 500 };
|
|
}
|
|
};
|
|
|
|
// Reusable function to get a single object by ID
|
|
export const getObject = async ({ model, id, populate }) => {
|
|
try {
|
|
logger.trace('Getting object:', {
|
|
model,
|
|
id,
|
|
populate,
|
|
});
|
|
|
|
// Try cache
|
|
const cachedObject = await retrieveObjectCache({ model, id, populate });
|
|
|
|
let query = model.findById(id).lean();
|
|
|
|
// Auto-populate file references if the model has them
|
|
if (modelHasRef(model, 'file')) {
|
|
const fileFields = getFieldsByRef(model, 'file');
|
|
|
|
// Populate all file reference fields
|
|
for (const field of fileFields) {
|
|
query = query.populate(field);
|
|
}
|
|
}
|
|
|
|
// Handle populate (array or single value)
|
|
if (populate) {
|
|
if (Array.isArray(populate)) {
|
|
for (const pop of populate) {
|
|
query = query.populate(pop);
|
|
}
|
|
} else if (typeof populate === 'string' || typeof populate === 'object') {
|
|
query = query.populate(populate);
|
|
}
|
|
}
|
|
const result = await query;
|
|
|
|
if (!result) {
|
|
return { error: 'Object not found.', code: 404 };
|
|
}
|
|
|
|
const expanded = mergeObjectUpdates(cachedObject || {}, expandObjectIds(result));
|
|
|
|
// Update cache with the expanded object
|
|
await updateObjectCache({
|
|
model,
|
|
id: expanded._id,
|
|
object: expanded,
|
|
populate,
|
|
});
|
|
|
|
return expanded;
|
|
} catch (error) {
|
|
return { error: error, code: 500 };
|
|
}
|
|
};
|
|
|
|
export const getModelStats = async ({ model }) => {
|
|
if (!model.stats) {
|
|
logger.warn(`Model ${model.modelName} does not have a stats method.`);
|
|
return { error: 'Model does not have a stats method.', code: 500 };
|
|
}
|
|
return await model.stats();
|
|
};
|
|
|
|
export const getModelHistory = async ({ model, from, to }) => {
|
|
if (!model.history && !from && !to) {
|
|
logger.warn(`Model ${model.modelName} does not have a history method.`);
|
|
return { error: 'Model does not have a history method.', code: 500 };
|
|
}
|
|
return await model.history(from, to);
|
|
};
|
|
|
|
export const listObjectDependencies = async ({ model, id }) => {
|
|
try {
|
|
const dependencies = [];
|
|
const parentModelName = model?.modelName;
|
|
if (!parentModelName || !id) {
|
|
return [];
|
|
}
|
|
|
|
const allModelEntries = getAllModels();
|
|
|
|
for (const entry of allModelEntries) {
|
|
const targetModel = entry?.model;
|
|
if (!targetModel || !targetModel.schema) continue;
|
|
|
|
const referencingPaths = [];
|
|
|
|
targetModel.schema.eachPath((pathName, schemaType) => {
|
|
const directRef = schemaType?.options?.ref;
|
|
const arrayRef = schemaType?.caster?.options?.ref;
|
|
const refName = directRef || arrayRef;
|
|
if (refName === parentModelName) {
|
|
referencingPaths.push(pathName);
|
|
}
|
|
});
|
|
|
|
if (referencingPaths.length === 0) continue;
|
|
|
|
for (const pathName of referencingPaths) {
|
|
const filter = { [pathName]: id };
|
|
const results = await targetModel.find(filter).lean();
|
|
for (const doc of results) {
|
|
const object = expandObjectIds(doc);
|
|
dependencies.push({
|
|
objectType: targetModel.modelName,
|
|
_id: object._id,
|
|
name: object?.name,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
return dependencies;
|
|
} catch (error) {
|
|
logger.error('listObjectDependencies error:', error);
|
|
return { error: error.message, code: 500 };
|
|
}
|
|
};
|
|
|
|
export const checkStates = async ({ model, id, states }) => {
|
|
try {
|
|
const object = await getObject({ model, id, cached: true });
|
|
if (!object?.state?.type) {
|
|
logger.warn(`Object ${id} has no state type.`);
|
|
return false;
|
|
}
|
|
if (states.includes(object?.state?.type)) {
|
|
return true;
|
|
}
|
|
return false;
|
|
} catch (error) {
|
|
logger.error('checkStates error:', error);
|
|
return { error: error.message, code: 500 };
|
|
}
|
|
};
|
|
|
|
// Reusable function to edit an object by ID, with audit logging and distribution
|
|
export const editObject = async ({ model, id, updateData, user, populate, recalculate = true }) => {
|
|
try {
|
|
// Determine parentType from model name
|
|
const parentType = model.modelName ? model.modelName : 'unknown';
|
|
// Fetch the and update object
|
|
var query = model.findByIdAndUpdate(id, flatternObjectIds(updateData)).lean();
|
|
|
|
if (populate) {
|
|
if (Array.isArray(populate)) {
|
|
for (const pop of populate) {
|
|
query = query.populate(pop);
|
|
}
|
|
} else if (typeof populate === 'string' || typeof populate === 'object') {
|
|
query = query.populate(populate);
|
|
}
|
|
}
|
|
|
|
const previousObject = await query;
|
|
|
|
if (!previousObject) {
|
|
return { error: `${parentType} not found.`, code: 404 };
|
|
}
|
|
|
|
const previousExpandedObject = expandObjectIds(previousObject);
|
|
|
|
// Check if any model parameters have ref: 'file' and flush files if so
|
|
if (modelHasRef(model, 'file')) {
|
|
logger.debug(`Model ${model.modelName} has file references, checking for files to flush`);
|
|
const fileFields = getFieldsByRef(model, 'file');
|
|
|
|
for (const fieldName of fileFields) {
|
|
const fieldValue = previousExpandedObject[fieldName];
|
|
|
|
if (fieldValue) {
|
|
if (Array.isArray(fieldValue)) {
|
|
// Handle file arrays
|
|
for (const fileRef of fieldValue) {
|
|
if (fileRef && fileRef._id) {
|
|
logger.debug(`Flushing file from array field ${fieldName}: ${fileRef._id}`);
|
|
await flushFile({ id: fileRef._id, user });
|
|
}
|
|
}
|
|
} else if (fieldValue._id) {
|
|
// Handle single file reference
|
|
logger.debug(`Flushing file from field ${fieldName}: ${fieldValue._id}`);
|
|
await flushFile({ id: fieldValue._id, user });
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const updatedObject = mergeObjectUpdates(
|
|
_.cloneDeep(previousExpandedObject),
|
|
expandObjectIds(updateData)
|
|
);
|
|
|
|
// Audit log before update
|
|
await editAuditLog(previousExpandedObject, updatedObject, id, parentType, user);
|
|
|
|
if (
|
|
parentType !== 'notification' &&
|
|
parentType !== 'auditLog' &&
|
|
parentType !== 'userNotifier'
|
|
) {
|
|
await editNotification(previousExpandedObject, updatedObject, id, parentType, user);
|
|
}
|
|
// Distribute update
|
|
await distributeUpdate(updateData, id, parentType);
|
|
// Call childUpdate event for any child objects
|
|
await distributeChildUpdate(previousExpandedObject, updatedObject, id, model);
|
|
|
|
// Update cache with the new version
|
|
await updateObjectCache({
|
|
model,
|
|
id,
|
|
object: updatedObject,
|
|
populate,
|
|
});
|
|
await invalidateNeighborsCacheForObject({ model, id });
|
|
|
|
if (model.recalculate && recalculate == true) {
|
|
logger.debug(`Recalculating ${model.modelName}`);
|
|
await model.recalculate(updatedObject, user);
|
|
}
|
|
|
|
if (model.stats) {
|
|
logger.debug(`Getting stats for ${model.modelName}`);
|
|
const statsData = await model.stats();
|
|
await distributeStats(statsData, parentType);
|
|
}
|
|
|
|
return updatedObject;
|
|
} catch (error) {
|
|
logger.error('editObject error:', error);
|
|
return { error: error.message, code: 500 };
|
|
}
|
|
};
|
|
|
|
// Reusable function to edit multiple objects
|
|
export const editObjects = async ({ model, updates, user, populate, recalculate = true }) => {
|
|
try {
|
|
const results = [];
|
|
for (const update of updates) {
|
|
const id = update._id || update.id;
|
|
const updateData = { ...update };
|
|
delete updateData._id;
|
|
delete updateData.id;
|
|
|
|
const result = await editObject({
|
|
model,
|
|
id,
|
|
updateData,
|
|
user,
|
|
populate,
|
|
recalculate,
|
|
});
|
|
results.push(result);
|
|
}
|
|
return results;
|
|
} catch (error) {
|
|
logger.error('editObjects error:', error);
|
|
return { error: error.message, code: 500 };
|
|
}
|
|
};
|
|
|
|
// Reusable function to create a new object
|
|
export const newObject = async (
|
|
{ model, newData, user = null, recalculate = true },
|
|
distributeChanges = true
|
|
) => {
|
|
try {
|
|
const parentType = model.modelName ? model.modelName : 'unknown';
|
|
|
|
const result = await model.create(newData);
|
|
if (!result || result.length === 0) {
|
|
return { error: 'No object created.', code: 500 };
|
|
}
|
|
const created = expandObjectIds(result.toObject());
|
|
|
|
await newAuditLog(newData, created._id, parentType, user);
|
|
|
|
if (distributeChanges == true) {
|
|
await distributeNew(created, parentType);
|
|
}
|
|
await distributeChildNew(created, created._id, model);
|
|
|
|
// Cache the newly created object
|
|
await updateObjectCache({
|
|
model,
|
|
id: created._id,
|
|
object: created,
|
|
populate: [],
|
|
});
|
|
|
|
if (model.recalculate && recalculate == true) {
|
|
logger.debug(`Recalculating ${model.modelName}`);
|
|
await model.recalculate(created, user);
|
|
}
|
|
|
|
if (model.stats) {
|
|
logger.debug(`Getting stats for ${model.modelName}`);
|
|
const statsData = await model.stats();
|
|
await distributeStats(statsData, parentType);
|
|
}
|
|
|
|
return created;
|
|
} catch (error) {
|
|
logger.error('newObject error:', error);
|
|
return { error: error.message, code: 500 };
|
|
}
|
|
};
|
|
|
|
const FILE_THUMBNAIL_SIZES = [64, 128, 265];
|
|
const FILE_THUMBNAIL_CACHE_TTL_SECONDS = 5 * 60;
|
|
|
|
function getFileThumbnailCacheKey(fileId, size) {
|
|
return `thumbnails:${size}:${fileId}`;
|
|
}
|
|
|
|
function getFileThumbnailS3Key(fileId, size) {
|
|
return `thumbnails/${size}/${fileId}.jpg`;
|
|
}
|
|
|
|
async function streamToBuffer(body) {
|
|
if (Buffer.isBuffer(body)) {
|
|
return body;
|
|
}
|
|
|
|
if (body && typeof body.transformToByteArray === 'function') {
|
|
return Buffer.from(await body.transformToByteArray());
|
|
}
|
|
|
|
const chunks = [];
|
|
for await (const chunk of body) {
|
|
chunks.push(chunk);
|
|
}
|
|
|
|
return Buffer.concat(chunks);
|
|
}
|
|
|
|
export async function getThumbnail({ fileId, size, file }) {
|
|
const parsedSize = Number(size);
|
|
|
|
if (!FILE_THUMBNAIL_SIZES.includes(parsedSize)) {
|
|
return { error: 'Invalid thumbnail size.', code: 400 };
|
|
}
|
|
|
|
const normalizedFileId = fileId.toString();
|
|
const cacheKey = getFileThumbnailCacheKey(normalizedFileId, parsedSize);
|
|
|
|
try {
|
|
const cached = await redisServer.getKey(cacheKey);
|
|
if (cached) {
|
|
return {
|
|
buffer: Buffer.from(cached, 'base64'),
|
|
contentType: 'image/jpeg',
|
|
};
|
|
}
|
|
|
|
const s3Key =
|
|
file?.metaData?.thumbnails?.[parsedSize] ||
|
|
file?.metaData?.thumbnails?.[String(parsedSize)] ||
|
|
getFileThumbnailS3Key(normalizedFileId, parsedSize);
|
|
|
|
const body = await downloadFile(BUCKETS.FILES, s3Key);
|
|
const buffer = await streamToBuffer(body);
|
|
|
|
await redisServer.setKey(cacheKey, buffer.toString('base64'), FILE_THUMBNAIL_CACHE_TTL_SECONDS);
|
|
|
|
return {
|
|
buffer,
|
|
contentType: 'image/jpeg',
|
|
};
|
|
} catch (error) {
|
|
if (error.message === 'File not found') {
|
|
return { error: 'Thumbnail not found.', code: 404 };
|
|
}
|
|
|
|
logger.error(
|
|
`Error getting thumbnail for file ${normalizedFileId} (size ${parsedSize}):`,
|
|
error
|
|
);
|
|
return { error: error.message, code: 500 };
|
|
}
|
|
}
|
|
|
|
async function deleteFileThumbnails(fileObject) {
|
|
if (!fileObject?._id) {
|
|
return;
|
|
}
|
|
|
|
const hasStoredThumbnails =
|
|
fileObject.hasThumbnails ||
|
|
fileObject.metaData?.thumbnails ||
|
|
fileObject.type?.startsWith('image/');
|
|
|
|
if (!hasStoredThumbnails) {
|
|
return;
|
|
}
|
|
|
|
const fileId = fileObject._id.toString();
|
|
const s3Keys = new Set(FILE_THUMBNAIL_SIZES.map((size) => getFileThumbnailS3Key(fileId, size)));
|
|
|
|
if (fileObject.metaData?.thumbnails) {
|
|
for (const key of Object.values(fileObject.metaData.thumbnails)) {
|
|
if (typeof key === 'string') {
|
|
s3Keys.add(key);
|
|
}
|
|
}
|
|
}
|
|
|
|
for (const s3Key of s3Keys) {
|
|
try {
|
|
await deleteFile(BUCKETS.FILES, s3Key);
|
|
logger.debug(`Deleted thumbnail from S3: ${s3Key}`);
|
|
} catch (error) {
|
|
logger.warn(`Failed to delete thumbnail from S3 (${s3Key}):`, error.message);
|
|
}
|
|
}
|
|
|
|
for (const size of FILE_THUMBNAIL_SIZES) {
|
|
try {
|
|
await redisServer.deleteKey(getFileThumbnailCacheKey(fileId, size));
|
|
logger.debug(`Deleted thumbnail from Redis: ${getFileThumbnailCacheKey(fileId, size)}`);
|
|
} catch (error) {
|
|
logger.warn(
|
|
`Failed to delete thumbnail from Redis (${getFileThumbnailCacheKey(fileId, size)}):`,
|
|
error.message
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function createFileThumbnails(fileData, fileObject) {
|
|
try {
|
|
const extension = fileObject?.extension?.toLowerCase();
|
|
const isGCodeFileType = extension === '.gcode' || extension === '.g';
|
|
const gcodeThumbnail = fileObject?.metaData?.thumbnail;
|
|
const hasGCodeThumbnailMeta =
|
|
isGCodeFileType && gcodeThumbnail && typeof gcodeThumbnail.data === 'string';
|
|
const isImageFileType = fileObject?.type?.startsWith('image/');
|
|
|
|
const hasThumbnailSource =
|
|
(hasGCodeThumbnailMeta && gcodeThumbnail.data.length > 0) || (isImageFileType && fileData);
|
|
|
|
if (!hasThumbnailSource || !fileObject?._id) {
|
|
return;
|
|
}
|
|
|
|
let thumbnailData = fileData;
|
|
if (hasGCodeThumbnailMeta) {
|
|
const normalized = gcodeThumbnail.data
|
|
.replace(/^data:image\/\w+;base64,/, '')
|
|
.replace(/\s/g, '');
|
|
thumbnailData = Buffer.from(normalized, 'base64');
|
|
if (!thumbnailData.length) return;
|
|
}
|
|
|
|
const fileId = fileObject._id.toString();
|
|
const image = sharp(thumbnailData, { failOn: 'none' });
|
|
const imageMetadata = await image.metadata();
|
|
|
|
if (!imageMetadata.width || !imageMetadata.height) {
|
|
return;
|
|
}
|
|
|
|
const blurhashSource = await sharp(thumbnailData)
|
|
.rotate()
|
|
.resize(32, 32, { fit: 'inside' })
|
|
.ensureAlpha()
|
|
.raw()
|
|
.toBuffer({ resolveWithObject: true });
|
|
|
|
const blurHash = encodeBlurHash(
|
|
new Uint8ClampedArray(blurhashSource.data),
|
|
blurhashSource.info.width,
|
|
blurhashSource.info.height,
|
|
4,
|
|
4
|
|
);
|
|
|
|
const thumbnails = {};
|
|
|
|
for (const size of FILE_THUMBNAIL_SIZES) {
|
|
const thumbnailBuffer = await sharp(thumbnailData)
|
|
.rotate()
|
|
.resize(size, size, { fit: 'inside', withoutEnlargement: true })
|
|
.jpeg({ quality: 85 })
|
|
.toBuffer();
|
|
|
|
const s3Key = getFileThumbnailS3Key(fileId, size);
|
|
await uploadFile(BUCKETS.FILES, s3Key, thumbnailBuffer, 'image/jpeg', {
|
|
fileId,
|
|
thumbnailSize: String(size),
|
|
});
|
|
|
|
await redisServer.setKey(
|
|
getFileThumbnailCacheKey(fileId, size),
|
|
thumbnailBuffer.toString('base64'),
|
|
FILE_THUMBNAIL_CACHE_TTL_SECONDS
|
|
);
|
|
|
|
thumbnails[size] = s3Key;
|
|
}
|
|
|
|
await editObject({
|
|
model: fileModel,
|
|
id: fileObject._id,
|
|
updateData: {
|
|
hasThumbnails: true,
|
|
metaData: {
|
|
...(fileObject.metaData || {}),
|
|
blurHash,
|
|
thumbnails,
|
|
},
|
|
},
|
|
});
|
|
} catch (error) {
|
|
logger.error('Error creating file thumbnails:', error);
|
|
}
|
|
}
|
|
|
|
// Reusable function to delete an object by ID, with audit logging and distribution
|
|
export const deleteObject = async (
|
|
{ model, id, user = null, checkUnused = false },
|
|
distributeChanges = true
|
|
) => {
|
|
try {
|
|
const parentType = model.modelName ? model.modelName : 'unknown';
|
|
|
|
if (checkUnused) {
|
|
const dependencies = await listObjectDependencies({ model, id });
|
|
if (dependencies?.error) {
|
|
return { error: dependencies.error, code: dependencies.code || 500 };
|
|
}
|
|
if (dependencies?.length > 0) {
|
|
return {
|
|
error: 'Object is in use and cannot be deleted',
|
|
code: 409,
|
|
dependencies: dependencies.length,
|
|
dependencyDetails: dependencies,
|
|
};
|
|
}
|
|
}
|
|
|
|
// Delete the object
|
|
const result = await model.findByIdAndDelete(id);
|
|
|
|
if (!result) {
|
|
return { error: `${parentType} not found.`, code: 404 };
|
|
}
|
|
|
|
const deleted = expandObjectIds(result.toObject());
|
|
|
|
// Audit log the deletion
|
|
await deleteAuditLog(deleted, id.toString(), parentType, user);
|
|
|
|
if (
|
|
parentType !== 'notification' &&
|
|
parentType !== 'auditLog' &&
|
|
parentType !== 'userNotifier'
|
|
) {
|
|
await deleteNotification(deleted, id.toString(), parentType, user);
|
|
}
|
|
|
|
if (distributeChanges == true) {
|
|
await distributeDelete(deleted, parentType);
|
|
}
|
|
|
|
await distributeChildDelete(deleted, id, model);
|
|
|
|
// Invalidate cache for this object
|
|
await deleteObjectCache({ model, id });
|
|
await invalidateNeighborsCacheForObject({ model, id });
|
|
|
|
if (model.recalculate) {
|
|
logger.debug(`Recalculating ${model.modelName}`);
|
|
await model.recalculate(deleted, user);
|
|
}
|
|
|
|
if (model.stats) {
|
|
logger.debug(`Getting stats for ${model.modelName}`);
|
|
const statsData = await model.stats();
|
|
await distributeStats(statsData, parentType);
|
|
}
|
|
|
|
return { deleted: true, object: deleted };
|
|
} catch (error) {
|
|
logger.error('deleteObject error:', error);
|
|
return { error: error.message, code: 500 };
|
|
}
|
|
};
|
|
|
|
export const flushFile = async ({ id, user }) => {
|
|
try {
|
|
logger.info(`Starting file deletion process for file ID: ${id}`);
|
|
|
|
// First, check if the file exists
|
|
const file = await fileModel.findById(id).lean();
|
|
if (!file) {
|
|
logger.warn(`File with ID ${id} not found`);
|
|
return {
|
|
error: 'File not found',
|
|
code: 404,
|
|
};
|
|
}
|
|
|
|
logger.info(`Found file: ${file.name} (${file._id})`);
|
|
|
|
// Check if this file has any dependencies
|
|
const dependencies = await listObjectDependencies({
|
|
model: fileModel,
|
|
id: file._id,
|
|
});
|
|
|
|
if (dependencies.length > 0) {
|
|
logger.info(
|
|
`File ${file._id} (${file.name}) has ${dependencies.length} dependencies, cannot delete`
|
|
);
|
|
return {
|
|
error: 'File has dependencies and cannot be deleted',
|
|
code: 409,
|
|
dependencies: dependencies.length,
|
|
dependencyDetails: dependencies,
|
|
};
|
|
}
|
|
|
|
logger.debug(`File ${file._id} (${file.name}) has no dependencies, proceeding with deletion`);
|
|
|
|
// Delete from database first
|
|
const deleteResult = await deleteObject({
|
|
model: fileModel,
|
|
id: file._id,
|
|
user,
|
|
});
|
|
|
|
if (deleteResult.error) {
|
|
logger.error(`Failed to delete file ${file._id} from database:`, deleteResult.error);
|
|
return {
|
|
error: deleteResult.error,
|
|
code: deleteResult.code || 500,
|
|
};
|
|
}
|
|
|
|
// Try to delete from Ceph storage if it exists
|
|
if (file.extension) {
|
|
try {
|
|
const cephKey = `files/${file._id}${file.extension}`;
|
|
|
|
await deleteFile(BUCKETS.FILES, cephKey);
|
|
logger.debug(`Deleted file from Ceph storage: ${cephKey}`);
|
|
} catch (cephError) {
|
|
logger.warn(`Failed to delete file ${file._id} from Ceph storage:`, cephError.message);
|
|
// Don't treat Ceph deletion failure as a critical error since DB record is already deleted
|
|
}
|
|
}
|
|
|
|
await deleteFileThumbnails(file);
|
|
|
|
const result = {
|
|
success: true,
|
|
deletedFile: {
|
|
fileId: file._id,
|
|
fileName: file.name,
|
|
deletedAt: new Date(),
|
|
},
|
|
};
|
|
|
|
logger.info(`Successfully deleted file: ${file.name} (${file._id})`);
|
|
return result;
|
|
} catch (error) {
|
|
logger.error('Error in flushFile:', error);
|
|
return {
|
|
error: error.message,
|
|
code: 500,
|
|
};
|
|
}
|
|
};
|
|
|
|
// Helper function to recursively delete objects and their children
|
|
export const recursivelyDeleteChildObjects = async (
|
|
{ model, id, user = null },
|
|
distributeChanges = true
|
|
) => {
|
|
const deletedIds = [];
|
|
|
|
// Find all objects that have this object as their parent
|
|
const childObjects = await model.find({ parent: id });
|
|
|
|
// Recursively delete all children first
|
|
for (const childObject of childObjects) {
|
|
const childDeletedIds = await recursivelyDeleteChildObjects(
|
|
{ model, id: childObject._id, user },
|
|
false
|
|
);
|
|
deletedIds.push(...childDeletedIds);
|
|
}
|
|
|
|
// Delete the current object
|
|
|
|
await deleteObject({ model, id, user }, distributeChanges);
|
|
|
|
deletedIds.push(id);
|
|
|
|
return deletedIds;
|
|
};
|