Refactor caching mechanism in database.js
- Replaced the previous model cache implementation with a unified object cache using NodeCache for improved performance and simplicity. - Updated cache retrieval and update functions to support additional parameters such as `populate`, `filter`, `sort`, and `project`. - Enhanced logging for cache operations to provide better traceability of cache hits and misses. - Removed deprecated functions and streamlined the caching logic for object and list retrieval.
This commit is contained in:
parent
4ac87f0141
commit
e4c790e7cc
@ -11,6 +11,7 @@ import {
|
||||
import log4js from 'log4js';
|
||||
import { loadConfig } from '../config.js';
|
||||
import { userModel } from './schemas/management/user.schema.js';
|
||||
import { jsonToCacheKey } from '../utils.js';
|
||||
|
||||
const config = loadConfig();
|
||||
|
||||
@ -19,44 +20,31 @@ const cacheLogger = log4js.getLogger('Local Cache');
|
||||
logger.level = config.server.logLevel;
|
||||
cacheLogger.level = config.server.logLevel;
|
||||
|
||||
const modelCaches = new Map();
|
||||
const objectCache = new NodeCache({
|
||||
stdTTL: 30, // 30 sec expiration
|
||||
checkperiod: 600, // 30 sec periodic cleanup
|
||||
useClones: false // Don't clone objects for better performance
|
||||
});
|
||||
const listCache = new NodeCache({
|
||||
stdTTL: 30, // 30 sec expiration
|
||||
checkperiod: 600, // 30 sec periodic cleanup
|
||||
useClones: false // Don't clone objects for better performance
|
||||
});
|
||||
|
||||
function getModelCache(model) {
|
||||
const modelName = model.modelName;
|
||||
const modelCache = modelCaches.get(modelName);
|
||||
if (modelCache == undefined) {
|
||||
logger.trace('Creating new model cache...');
|
||||
const newModelCache = new NodeCache({
|
||||
stdTTL: 30, // 30 sec expiration
|
||||
checkperiod: 30, // 30 sec periodic cleanup
|
||||
useClones: false // Don't clone objects for better performance
|
||||
});
|
||||
modelCaches.set(modelName, newModelCache);
|
||||
return newModelCache;
|
||||
}
|
||||
logger.trace('Getting model cache...');
|
||||
return modelCache;
|
||||
}
|
||||
|
||||
export const retrieveObjectCache = ({ model, id }) => {
|
||||
cacheLogger.trace('Retrieving:', {
|
||||
export const retrieveObjectCache = ({ model, id, populate = [] }) => {
|
||||
const cacheKeyObject = {
|
||||
model: model.modelName,
|
||||
id
|
||||
});
|
||||
const modelCache = getModelCache(model);
|
||||
id,
|
||||
populate
|
||||
};
|
||||
|
||||
const cachedObject = modelCache.get(id);
|
||||
const cacheKey = jsonToCacheKey(cacheKeyObject);
|
||||
|
||||
cacheLogger.trace('Retrieving:');
|
||||
const cachedObject = objectCache.get(cacheKey);
|
||||
|
||||
if (cachedObject == undefined) {
|
||||
cacheLogger.trace('Miss:', {
|
||||
model: model.modelName,
|
||||
id
|
||||
});
|
||||
cacheLogger.trace('Miss:', cacheKeyObject);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@ -68,25 +56,36 @@ export const retrieveObjectCache = ({ model, id }) => {
|
||||
return cachedObject;
|
||||
};
|
||||
|
||||
export const retrieveObjectsCache = ({ model }) => {
|
||||
cacheLogger.trace('Retrieving:', {
|
||||
model: model.modelName
|
||||
});
|
||||
const modelCache = getModelCache(model);
|
||||
export const retrieveListCache = ({
|
||||
model,
|
||||
populate = [],
|
||||
filter = {},
|
||||
sort = '',
|
||||
order = 'ascend',
|
||||
project = {}
|
||||
}) => {
|
||||
const cacheKeyObject = {
|
||||
model: model.modelName,
|
||||
id,
|
||||
populate,
|
||||
filter,
|
||||
sort,
|
||||
project,
|
||||
order
|
||||
};
|
||||
|
||||
const modelCacheKeys = modelCache.keys();
|
||||
cacheLogger.trace('Retrieving:', cacheKeyObject);
|
||||
|
||||
const cachedList = listCache.get(model.modelName);
|
||||
const cacheKey = jsonToCacheKey(cacheKeyObject);
|
||||
|
||||
if (cachedList == true) {
|
||||
const cachedObjects = modelCacheKeys.map(key => modelCache.get(key));
|
||||
const cachedList = listCache.get(cacheKey);
|
||||
|
||||
if (cachedList != undefined) {
|
||||
cacheLogger.trace('Hit:', {
|
||||
model: model.modelName,
|
||||
length: cachedObjects.length
|
||||
...cacheKeyObject,
|
||||
length: cachedList.length
|
||||
});
|
||||
|
||||
return cachedObjects;
|
||||
return cachedList;
|
||||
}
|
||||
|
||||
cacheLogger.trace('Miss:', {
|
||||
@ -95,21 +94,23 @@ export const retrieveObjectsCache = ({ model }) => {
|
||||
return undefined;
|
||||
};
|
||||
|
||||
export const updateObjectCache = ({ model, id, object }) => {
|
||||
cacheLogger.trace('Updating:', {
|
||||
export const updateObjectCache = ({ model, id, object, populate = [] }) => {
|
||||
const cacheKeyObject = {
|
||||
model: model.modelName,
|
||||
id
|
||||
});
|
||||
const modelCache = getModelCache(model);
|
||||
const cachedObject = modelCache.get(id) || {};
|
||||
id,
|
||||
populate
|
||||
};
|
||||
|
||||
const cacheKey = jsonToCacheKey(cacheKeyObject);
|
||||
|
||||
cacheLogger.trace('Updating:', cacheKeyObject);
|
||||
|
||||
const cachedObject = objectCache.get(cacheKey) || {};
|
||||
const mergedObject = _.merge(cachedObject, object);
|
||||
|
||||
modelCache.set(id, mergedObject);
|
||||
objectCache.set(cacheKey, mergedObject);
|
||||
|
||||
cacheLogger.trace('Updated:', {
|
||||
model: model.modelName,
|
||||
id
|
||||
});
|
||||
cacheLogger.trace('Updated:', { ...cacheKeyObject });
|
||||
|
||||
return mergedObject;
|
||||
};
|
||||
@ -130,29 +131,39 @@ export const deleteObjectCache = ({ model, id }) => {
|
||||
return mergedObject;
|
||||
};
|
||||
|
||||
export const updateObjectsCache = ({ model, objects }) => {
|
||||
cacheLogger.trace('Updating:', {
|
||||
export const updateListCache = ({
|
||||
model,
|
||||
objects,
|
||||
populate = [],
|
||||
filter = {},
|
||||
sort = '',
|
||||
order = 'ascend',
|
||||
project = {}
|
||||
}) => {
|
||||
const cacheKeyObject = {
|
||||
model: model.modelName,
|
||||
populate,
|
||||
filter,
|
||||
sort,
|
||||
project,
|
||||
order
|
||||
};
|
||||
|
||||
cacheLogger.trace('Updating:', {
|
||||
...cacheKeyObject,
|
||||
length: objects.length
|
||||
});
|
||||
const modelCache = getModelCache(model);
|
||||
|
||||
objects.forEach(object => {
|
||||
const cachedObject = modelCache.get(object._id) || {};
|
||||
const cacheKey = jsonToCacheKey(cacheKeyObject);
|
||||
|
||||
const mergedObject = _.merge(cachedObject, object);
|
||||
|
||||
modelCache.set(object._id, mergedObject);
|
||||
});
|
||||
|
||||
listCache.set(model.modelName, true);
|
||||
listCache.set(cacheKey, objects);
|
||||
|
||||
cacheLogger.trace('Updated:', {
|
||||
model: model.modelName,
|
||||
...cacheKeyObject,
|
||||
length: objects.length
|
||||
});
|
||||
|
||||
return mergedObject;
|
||||
return objects;
|
||||
};
|
||||
|
||||
// Reusable function to list objects with aggregation, filtering, search, sorting, and pagination
|
||||
@ -162,27 +173,29 @@ export const listObjects = async ({
|
||||
filter = {},
|
||||
sort = '',
|
||||
order = 'ascend',
|
||||
project, // optional: override default projection
|
||||
project = {}, // optional: override default projection
|
||||
cached = false
|
||||
}) => {
|
||||
try {
|
||||
logger.trace('Listing objects:', {
|
||||
model,
|
||||
populate,
|
||||
page,
|
||||
limit,
|
||||
filter,
|
||||
sort,
|
||||
order,
|
||||
project,
|
||||
cache
|
||||
cached
|
||||
});
|
||||
|
||||
var cacheKey = undefined;
|
||||
var modelCache = getModelCache(model);
|
||||
|
||||
if (cached == true) {
|
||||
const objectsCache = retrieveObjectsCache({ model });
|
||||
const objectsCache = retrieveObjectsCache({
|
||||
model,
|
||||
populate,
|
||||
filter,
|
||||
sort,
|
||||
order,
|
||||
project
|
||||
});
|
||||
if (objectsCache != undefined) {
|
||||
return objectsCache;
|
||||
}
|
||||
@ -210,7 +223,7 @@ export const listObjects = async ({
|
||||
let query = model.find(filter).sort({ [sort]: sortOrder });
|
||||
|
||||
// Handle populate (array or single value)
|
||||
if (populate) {
|
||||
if (populate.length > 0) {
|
||||
if (Array.isArray(populate)) {
|
||||
for (const pop of populate) {
|
||||
query = query.populate(pop);
|
||||
@ -221,7 +234,7 @@ export const listObjects = async ({
|
||||
}
|
||||
|
||||
// Handle select (projection)
|
||||
if (project) {
|
||||
if (project != {}) {
|
||||
query = query.select(project);
|
||||
}
|
||||
|
||||
@ -231,18 +244,25 @@ export const listObjects = async ({
|
||||
|
||||
const finalResult = expandObjectIds(queryResult);
|
||||
|
||||
updateObjectsCache({ model, objects });
|
||||
updateListCache({
|
||||
model,
|
||||
objects: finalResult,
|
||||
populate,
|
||||
filter,
|
||||
sort,
|
||||
order,
|
||||
project
|
||||
});
|
||||
|
||||
logger.trace('Retreived from database:', {
|
||||
model,
|
||||
populate,
|
||||
page,
|
||||
limit,
|
||||
filter,
|
||||
sort,
|
||||
order,
|
||||
project,
|
||||
cache
|
||||
cached,
|
||||
length: finalResult.length
|
||||
});
|
||||
return finalResult;
|
||||
} catch (error) {
|
||||
@ -252,7 +272,12 @@ export const listObjects = async ({
|
||||
};
|
||||
|
||||
// Reusable function to get a single object by ID
|
||||
export const getObject = async ({ model, id, populate, cached = false }) => {
|
||||
export const getObject = async ({
|
||||
model,
|
||||
id,
|
||||
populate = [],
|
||||
cached = false
|
||||
}) => {
|
||||
try {
|
||||
logger.trace('Getting object:', {
|
||||
model,
|
||||
@ -261,7 +286,7 @@ export const getObject = async ({ model, id, populate, cached = false }) => {
|
||||
});
|
||||
|
||||
if (cached == true) {
|
||||
const cachedObject = retrieveObjectCache({ model, id });
|
||||
const cachedObject = retrieveObjectCache({ model, id, populate });
|
||||
if (cachedObject != undefined) {
|
||||
return cachedObject;
|
||||
}
|
||||
@ -299,63 +324,14 @@ export const getObject = async ({ model, id, populate, cached = false }) => {
|
||||
updateObjectCache({
|
||||
model: model,
|
||||
id: finalResult._id.toString(),
|
||||
populate,
|
||||
object: finalResult
|
||||
});
|
||||
|
||||
return finalResult;
|
||||
} catch (error) {
|
||||
logger.error('An error retreiving object:', error.message);
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
|
||||
// Reusable function to get a single object by ID
|
||||
export const getObjectByFilter = async ({ model, filter, populate }) => {
|
||||
try {
|
||||
logger.trace('Getting object:', {
|
||||
model,
|
||||
filter,
|
||||
populate
|
||||
});
|
||||
|
||||
let query = model.findOne(filter).lean();
|
||||
|
||||
// Handle populate (array or single value)
|
||||
if (populate) {
|
||||
if (Array.isArray(populate)) {
|
||||
for (const pop of populate) {
|
||||
query = query.populate(pop);
|
||||
}
|
||||
} else if (typeof populate === 'string' || typeof populate === 'object') {
|
||||
query = query.populate(populate);
|
||||
}
|
||||
}
|
||||
const finalResult = await query;
|
||||
|
||||
if (!finalResult) {
|
||||
logger.warn('Object not found in database:', {
|
||||
model,
|
||||
filter,
|
||||
populate
|
||||
});
|
||||
return undefined;
|
||||
}
|
||||
|
||||
logger.trace('Retreived object from database:', {
|
||||
model,
|
||||
filter,
|
||||
populate
|
||||
});
|
||||
|
||||
updateObjectCache({
|
||||
model: model,
|
||||
id: finalResult._id.toString(),
|
||||
object: finalResult
|
||||
});
|
||||
|
||||
return finalResult;
|
||||
} catch (error) {
|
||||
logger.error('An error retreiving object:', error.message);
|
||||
throw error;
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
@ -367,7 +343,7 @@ export const editObject = async ({
|
||||
updateData,
|
||||
owner = undefined,
|
||||
ownerType = undefined,
|
||||
populate
|
||||
populate = []
|
||||
}) => {
|
||||
try {
|
||||
// Determine parentType from model name
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user