Refactor caching mechanism in database.js

- Replaced the previous model cache implementation with a unified object cache using NodeCache for improved performance and simplicity.
- Updated cache retrieval and update functions to support additional parameters such as `populate`, `filter`, `sort`, and `project`.
- Enhanced logging for cache operations to provide better traceability of cache hits and misses.
- Removed deprecated functions and streamlined the caching logic for object and list retrieval.

src/database/database.js

@@ -11,6 +11,7 @@ import {
 import log4js from 'log4js';
 import { loadConfig } from '../config.js';
 import { userModel } from './schemas/management/user.schema.js';
+import { jsonToCacheKey } from '../utils.js';
 
 const config = loadConfig();
 
@@ -19,44 +20,31 @@ const cacheLogger = log4js.getLogger('Local Cache');
 logger.level = config.server.logLevel;
 cacheLogger.level = config.server.logLevel;
 
-const modelCaches = new Map();
+const objectCache = new NodeCache({
+  stdTTL: 30, // 30 sec expiration
+  checkperiod: 600, // 30 sec periodic cleanup
+  useClones: false // Don't clone objects for better performance
+});
 const listCache = new NodeCache({
   stdTTL: 30, // 30 sec expiration
   checkperiod: 600, // 30 sec periodic cleanup
   useClones: false // Don't clone objects for better performance
 });
 
-function getModelCache(model) {
+export const retrieveObjectCache = ({ model, id, populate = [] }) => {
-  const modelName = model.modelName;
+  const cacheKeyObject = {
-  const modelCache = modelCaches.get(modelName);
-  if (modelCache == undefined) {
-    logger.trace('Creating new model cache...');
-    const newModelCache = new NodeCache({
-      stdTTL: 30, // 30 sec expiration
-      checkperiod: 30, // 30 sec periodic cleanup
-      useClones: false // Don't clone objects for better performance
-    });
-    modelCaches.set(modelName, newModelCache);
-    return newModelCache;
-  }
-  logger.trace('Getting model cache...');
-  return modelCache;
-}
-
-export const retrieveObjectCache = ({ model, id }) => {
-  cacheLogger.trace('Retrieving:', {
     model: model.modelName,
-    id
+    id,
-  });
+    populate
-  const modelCache = getModelCache(model);
+  };
 
-  const cachedObject = modelCache.get(id);
+  const cacheKey = jsonToCacheKey(cacheKeyObject);
+
+  cacheLogger.trace('Retrieving:');
+  const cachedObject = objectCache.get(cacheKey);
 
   if (cachedObject == undefined) {
-    cacheLogger.trace('Miss:', {
+    cacheLogger.trace('Miss:', cacheKeyObject);
-      model: model.modelName,
-      id
-    });
     return undefined;
   }
 
@@ -68,25 +56,36 @@ export const retrieveObjectCache = ({ model, id }) => {
   return cachedObject;
 };
 
-export const retrieveObjectsCache = ({ model }) => {
+export const retrieveListCache = ({
-  cacheLogger.trace('Retrieving:', {
+  model,
-    model: model.modelName
+  populate = [],
-  });
+  filter = {},
-  const modelCache = getModelCache(model);
+  sort = '',
-
+  order = 'ascend',
-  const modelCacheKeys = modelCache.keys();
+  project = {}
-
+}) => {
-  const cachedList = listCache.get(model.modelName);
+  const cacheKeyObject = {
-
-  if (cachedList == true) {
-    const cachedObjects = modelCacheKeys.map(key => modelCache.get(key));
-
-    cacheLogger.trace('Hit:', {
     model: model.modelName,
-      length: cachedObjects.length
+    id,
-    });
+    populate,
+    filter,
+    sort,
+    project,
+    order
+  };
 
-    return cachedObjects;
+  cacheLogger.trace('Retrieving:', cacheKeyObject);
+
+  const cacheKey = jsonToCacheKey(cacheKeyObject);
+
+  const cachedList = listCache.get(cacheKey);
+
+  if (cachedList != undefined) {
+    cacheLogger.trace('Hit:', {
+      ...cacheKeyObject,
+      length: cachedList.length
+    });
+    return cachedList;
   }
 
   cacheLogger.trace('Miss:', {
@@ -95,21 +94,23 @@ export const retrieveObjectsCache = ({ model }) => {
   return undefined;
 };
 
-export const updateObjectCache = ({ model, id, object }) => {
+export const updateObjectCache = ({ model, id, object, populate = [] }) => {
-  cacheLogger.trace('Updating:', {
+  const cacheKeyObject = {
     model: model.modelName,
-    id
+    id,
-  });
+    populate
-  const modelCache = getModelCache(model);
+  };
-  const cachedObject = modelCache.get(id) || {};
+
+  const cacheKey = jsonToCacheKey(cacheKeyObject);
+
+  cacheLogger.trace('Updating:', cacheKeyObject);
+
+  const cachedObject = objectCache.get(cacheKey) || {};
   const mergedObject = _.merge(cachedObject, object);
 
-  modelCache.set(id, mergedObject);
+  objectCache.set(cacheKey, mergedObject);
 
-  cacheLogger.trace('Updated:', {
+  cacheLogger.trace('Updated:', { ...cacheKeyObject });
-    model: model.modelName,
-    id
-  });
 
   return mergedObject;
 };
@@ -130,29 +131,39 @@ export const deleteObjectCache = ({ model, id }) => {
   return mergedObject;
 };
 
-export const updateObjectsCache = ({ model, objects }) => {
+export const updateListCache = ({
-  cacheLogger.trace('Updating:', {
+  model,
+  objects,
+  populate = [],
+  filter = {},
+  sort = '',
+  order = 'ascend',
+  project = {}
+}) => {
+  const cacheKeyObject = {
     model: model.modelName,
+    populate,
+    filter,
+    sort,
+    project,
+    order
+  };
+
+  cacheLogger.trace('Updating:', {
+    ...cacheKeyObject,
     length: objects.length
   });
-  const modelCache = getModelCache(model);
 
-  objects.forEach(object => {
+  const cacheKey = jsonToCacheKey(cacheKeyObject);
-    const cachedObject = modelCache.get(object._id) || {};
 
-    const mergedObject = _.merge(cachedObject, object);
+  listCache.set(cacheKey, objects);
-
-    modelCache.set(object._id, mergedObject);
-  });
-
-  listCache.set(model.modelName, true);
 
   cacheLogger.trace('Updated:', {
-    model: model.modelName,
+    ...cacheKeyObject,
     length: objects.length
   });
 
-  return mergedObject;
+  return objects;
 };
 
 // Reusable function to list objects with aggregation, filtering, search, sorting, and pagination
@@ -162,27 +173,29 @@ export const listObjects = async ({
   filter = {},
   sort = '',
   order = 'ascend',
-  project, // optional: override default projection
+  project = {}, // optional: override default projection
   cached = false
 }) => {
   try {
     logger.trace('Listing objects:', {
       model,
       populate,
-      page,
-      limit,
       filter,
       sort,
       order,
       project,
-      cache
+      cached
     });
 
-    var cacheKey = undefined;
-    var modelCache = getModelCache(model);
-
     if (cached == true) {
-      const objectsCache = retrieveObjectsCache({ model });
+      const objectsCache = retrieveObjectsCache({
+        model,
+        populate,
+        filter,
+        sort,
+        order,
+        project
+      });
       if (objectsCache != undefined) {
         return objectsCache;
       }
@@ -210,7 +223,7 @@ export const listObjects = async ({
     let query = model.find(filter).sort({ [sort]: sortOrder });
 
     // Handle populate (array or single value)
-    if (populate) {
+    if (populate.length > 0) {
       if (Array.isArray(populate)) {
         for (const pop of populate) {
           query = query.populate(pop);
@@ -221,7 +234,7 @@ export const listObjects = async ({
     }
 
     // Handle select (projection)
-    if (project) {
+    if (project != {}) {
       query = query.select(project);
     }
 
@@ -231,18 +244,25 @@ export const listObjects = async ({
 
     const finalResult = expandObjectIds(queryResult);
 
-    updateObjectsCache({ model, objects });
+    updateListCache({
+      model,
+      objects: finalResult,
+      populate,
+      filter,
+      sort,
+      order,
+      project
+    });
 
     logger.trace('Retreived from database:', {
       model,
       populate,
-      page,
-      limit,
       filter,
       sort,
       order,
       project,
-      cache
+      cached,
+      length: finalResult.length
     });
     return finalResult;
   } catch (error) {
@@ -252,7 +272,12 @@ export const listObjects = async ({
 };
 
 // Reusable function to get a single object by ID
-export const getObject = async ({ model, id, populate, cached = false }) => {
+export const getObject = async ({
+  model,
+  id,
+  populate = [],
+  cached = false
+}) => {
   try {
     logger.trace('Getting object:', {
       model,
@@ -261,7 +286,7 @@ export const getObject = async ({ model, id, populate, cached = false }) => {
     });
 
     if (cached == true) {
-      const cachedObject = retrieveObjectCache({ model, id });
+      const cachedObject = retrieveObjectCache({ model, id, populate });
       if (cachedObject != undefined) {
         return cachedObject;
       }
@@ -299,63 +324,14 @@ export const getObject = async ({ model, id, populate, cached = false }) => {
     updateObjectCache({
       model: model,
       id: finalResult._id.toString(),
+      populate,
       object: finalResult
     });
 
     return finalResult;
   } catch (error) {
     logger.error('An error retreiving object:', error.message);
-    return undefined;
+    throw error;
-  }
-};
-
-// Reusable function to get a single object by ID
-export const getObjectByFilter = async ({ model, filter, populate }) => {
-  try {
-    logger.trace('Getting object:', {
-      model,
-      filter,
-      populate
-    });
-
-    let query = model.findOne(filter).lean();
-
-    // Handle populate (array or single value)
-    if (populate) {
-      if (Array.isArray(populate)) {
-        for (const pop of populate) {
-          query = query.populate(pop);
-        }
-      } else if (typeof populate === 'string' || typeof populate === 'object') {
-        query = query.populate(populate);
-      }
-    }
-    const finalResult = await query;
-
-    if (!finalResult) {
-      logger.warn('Object not found in database:', {
-        model,
-        filter,
-        populate
-      });
-      return undefined;
-    }
-
-    logger.trace('Retreived object from database:', {
-      model,
-      filter,
-      populate
-    });
-
-    updateObjectCache({
-      model: model,
-      id: finalResult._id.toString(),
-      object: finalResult
-    });
-
-    return finalResult;
-  } catch (error) {
-    logger.error('An error retreiving object:', error.message);
     return undefined;
   }
 };
@@ -367,7 +343,7 @@ export const editObject = async ({
   updateData,
   owner = undefined,
   ownerType = undefined,
-  populate
+  populate = []
 }) => {
   try {
     // Determine parentType from model name