From e4c790e7cc7055f99d06ee05d790f89be3308505 Mon Sep 17 00:00:00 2001 From: Tom Butcher Date: Fri, 5 Sep 2025 23:28:23 +0100 Subject: [PATCH] Refactor caching mechanism in database.js - Replaced the previous model cache implementation with a unified object cache using NodeCache for improved performance and simplicity. - Updated cache retrieval and update functions to support additional parameters such as `populate`, `filter`, `sort`, and `project`. - Enhanced logging for cache operations to provide better traceability of cache hits and misses. - Removed deprecated functions and streamlined the caching logic for object and list retrieval. --- src/database/database.js | 242 ++++++++++++++++++--------------------- 1 file changed, 109 insertions(+), 133 deletions(-) diff --git a/src/database/database.js b/src/database/database.js index 6c49582..c7a138e 100644 --- a/src/database/database.js +++ b/src/database/database.js @@ -11,6 +11,7 @@ import { import log4js from 'log4js'; import { loadConfig } from '../config.js'; import { userModel } from './schemas/management/user.schema.js'; +import { jsonToCacheKey } from '../utils.js'; const config = loadConfig(); @@ -19,44 +20,31 @@ const cacheLogger = log4js.getLogger('Local Cache'); logger.level = config.server.logLevel; cacheLogger.level = config.server.logLevel; -const modelCaches = new Map(); +const objectCache = new NodeCache({ + stdTTL: 30, // 30 sec expiration + checkperiod: 600, // 30 sec periodic cleanup + useClones: false // Don't clone objects for better performance +}); const listCache = new NodeCache({ stdTTL: 30, // 30 sec expiration checkperiod: 600, // 30 sec periodic cleanup useClones: false // Don't clone objects for better performance }); -function getModelCache(model) { - const modelName = model.modelName; - const modelCache = modelCaches.get(modelName); - if (modelCache == undefined) { - logger.trace('Creating new model cache...'); - const newModelCache = new NodeCache({ - stdTTL: 30, // 30 sec expiration - checkperiod: 30, // 30 sec periodic cleanup - useClones: false // Don't clone objects for better performance - }); - modelCaches.set(modelName, newModelCache); - return newModelCache; - } - logger.trace('Getting model cache...'); - return modelCache; -} - -export const retrieveObjectCache = ({ model, id }) => { - cacheLogger.trace('Retrieving:', { +export const retrieveObjectCache = ({ model, id, populate = [] }) => { + const cacheKeyObject = { model: model.modelName, - id - }); - const modelCache = getModelCache(model); + id, + populate + }; - const cachedObject = modelCache.get(id); + const cacheKey = jsonToCacheKey(cacheKeyObject); + + cacheLogger.trace('Retrieving:'); + const cachedObject = objectCache.get(cacheKey); if (cachedObject == undefined) { - cacheLogger.trace('Miss:', { - model: model.modelName, - id - }); + cacheLogger.trace('Miss:', cacheKeyObject); return undefined; } @@ -68,25 +56,36 @@ export const retrieveObjectCache = ({ model, id }) => { return cachedObject; }; -export const retrieveObjectsCache = ({ model }) => { - cacheLogger.trace('Retrieving:', { - model: model.modelName - }); - const modelCache = getModelCache(model); +export const retrieveListCache = ({ + model, + populate = [], + filter = {}, + sort = '', + order = 'ascend', + project = {} +}) => { + const cacheKeyObject = { + model: model.modelName, + id, + populate, + filter, + sort, + project, + order + }; - const modelCacheKeys = modelCache.keys(); + cacheLogger.trace('Retrieving:', cacheKeyObject); - const cachedList = listCache.get(model.modelName); + const cacheKey = jsonToCacheKey(cacheKeyObject); - if (cachedList == true) { - const cachedObjects = modelCacheKeys.map(key => modelCache.get(key)); + const cachedList = listCache.get(cacheKey); + if (cachedList != undefined) { cacheLogger.trace('Hit:', { - model: model.modelName, - length: cachedObjects.length + ...cacheKeyObject, + length: cachedList.length }); - - return cachedObjects; + return cachedList; } cacheLogger.trace('Miss:', { @@ -95,21 +94,23 @@ export const retrieveObjectsCache = ({ model }) => { return undefined; }; -export const updateObjectCache = ({ model, id, object }) => { - cacheLogger.trace('Updating:', { +export const updateObjectCache = ({ model, id, object, populate = [] }) => { + const cacheKeyObject = { model: model.modelName, - id - }); - const modelCache = getModelCache(model); - const cachedObject = modelCache.get(id) || {}; + id, + populate + }; + + const cacheKey = jsonToCacheKey(cacheKeyObject); + + cacheLogger.trace('Updating:', cacheKeyObject); + + const cachedObject = objectCache.get(cacheKey) || {}; const mergedObject = _.merge(cachedObject, object); - modelCache.set(id, mergedObject); + objectCache.set(cacheKey, mergedObject); - cacheLogger.trace('Updated:', { - model: model.modelName, - id - }); + cacheLogger.trace('Updated:', { ...cacheKeyObject }); return mergedObject; }; @@ -130,29 +131,39 @@ export const deleteObjectCache = ({ model, id }) => { return mergedObject; }; -export const updateObjectsCache = ({ model, objects }) => { - cacheLogger.trace('Updating:', { +export const updateListCache = ({ + model, + objects, + populate = [], + filter = {}, + sort = '', + order = 'ascend', + project = {} +}) => { + const cacheKeyObject = { model: model.modelName, + populate, + filter, + sort, + project, + order + }; + + cacheLogger.trace('Updating:', { + ...cacheKeyObject, length: objects.length }); - const modelCache = getModelCache(model); - objects.forEach(object => { - const cachedObject = modelCache.get(object._id) || {}; + const cacheKey = jsonToCacheKey(cacheKeyObject); - const mergedObject = _.merge(cachedObject, object); - - modelCache.set(object._id, mergedObject); - }); - - listCache.set(model.modelName, true); + listCache.set(cacheKey, objects); cacheLogger.trace('Updated:', { - model: model.modelName, + ...cacheKeyObject, length: objects.length }); - return mergedObject; + return objects; }; // Reusable function to list objects with aggregation, filtering, search, sorting, and pagination @@ -162,27 +173,29 @@ export const listObjects = async ({ filter = {}, sort = '', order = 'ascend', - project, // optional: override default projection + project = {}, // optional: override default projection cached = false }) => { try { logger.trace('Listing objects:', { model, populate, - page, - limit, filter, sort, order, project, - cache + cached }); - var cacheKey = undefined; - var modelCache = getModelCache(model); - if (cached == true) { - const objectsCache = retrieveObjectsCache({ model }); + const objectsCache = retrieveObjectsCache({ + model, + populate, + filter, + sort, + order, + project + }); if (objectsCache != undefined) { return objectsCache; } @@ -210,7 +223,7 @@ export const listObjects = async ({ let query = model.find(filter).sort({ [sort]: sortOrder }); // Handle populate (array or single value) - if (populate) { + if (populate.length > 0) { if (Array.isArray(populate)) { for (const pop of populate) { query = query.populate(pop); @@ -221,7 +234,7 @@ export const listObjects = async ({ } // Handle select (projection) - if (project) { + if (project != {}) { query = query.select(project); } @@ -231,18 +244,25 @@ export const listObjects = async ({ const finalResult = expandObjectIds(queryResult); - updateObjectsCache({ model, objects }); + updateListCache({ + model, + objects: finalResult, + populate, + filter, + sort, + order, + project + }); logger.trace('Retreived from database:', { model, populate, - page, - limit, filter, sort, order, project, - cache + cached, + length: finalResult.length }); return finalResult; } catch (error) { @@ -252,7 +272,12 @@ export const listObjects = async ({ }; // Reusable function to get a single object by ID -export const getObject = async ({ model, id, populate, cached = false }) => { +export const getObject = async ({ + model, + id, + populate = [], + cached = false +}) => { try { logger.trace('Getting object:', { model, @@ -261,7 +286,7 @@ export const getObject = async ({ model, id, populate, cached = false }) => { }); if (cached == true) { - const cachedObject = retrieveObjectCache({ model, id }); + const cachedObject = retrieveObjectCache({ model, id, populate }); if (cachedObject != undefined) { return cachedObject; } @@ -299,63 +324,14 @@ export const getObject = async ({ model, id, populate, cached = false }) => { updateObjectCache({ model: model, id: finalResult._id.toString(), + populate, object: finalResult }); return finalResult; } catch (error) { logger.error('An error retreiving object:', error.message); - return undefined; - } -}; - -// Reusable function to get a single object by ID -export const getObjectByFilter = async ({ model, filter, populate }) => { - try { - logger.trace('Getting object:', { - model, - filter, - populate - }); - - let query = model.findOne(filter).lean(); - - // Handle populate (array or single value) - if (populate) { - if (Array.isArray(populate)) { - for (const pop of populate) { - query = query.populate(pop); - } - } else if (typeof populate === 'string' || typeof populate === 'object') { - query = query.populate(populate); - } - } - const finalResult = await query; - - if (!finalResult) { - logger.warn('Object not found in database:', { - model, - filter, - populate - }); - return undefined; - } - - logger.trace('Retreived object from database:', { - model, - filter, - populate - }); - - updateObjectCache({ - model: model, - id: finalResult._id.toString(), - object: finalResult - }); - - return finalResult; - } catch (error) { - logger.error('An error retreiving object:', error.message); + throw error; return undefined; } }; @@ -367,7 +343,7 @@ export const editObject = async ({ updateData, owner = undefined, ownerType = undefined, - populate + populate = [] }) => { try { // Determine parentType from model name