diff --git a/src/__tests__/utils.getFieldsByRef.test.js b/src/__tests__/utils.getFieldsByRef.test.js new file mode 100644 index 0000000..efa22be --- /dev/null +++ b/src/__tests__/utils.getFieldsByRef.test.js @@ -0,0 +1,42 @@ +import { describe, expect, it } from '@jest/globals'; +import mongoose from 'mongoose'; +import { + collectIdsAtPath, + getFieldsByRef, + modelHasRef, +} from '../utils.js'; +import { emailMessageModel } from '../database/schemas/management/emailmessage.schema.js'; + +describe('getFieldsByRef', () => { + it('finds nested array file refs on email message recipients', () => { + expect(modelHasRef(emailMessageModel, 'file')).toBe(true); + expect(getFieldsByRef(emailMessageModel, 'file')).toEqual(['recipients.attachments']); + }); + + it('finds top-level array refs using embeddedSchemaType', () => { + const schema = new mongoose.Schema({ + files: [{ type: mongoose.Schema.Types.ObjectId, ref: 'file' }], + }); + const model = { schema }; + + expect(modelHasRef(model, 'file')).toBe(true); + expect(getFieldsByRef(model, 'file')).toEqual(['files']); + }); +}); + +describe('collectIdsAtPath', () => { + it('collects file ids from nested recipient attachment lists', () => { + const object = { + recipients: [ + { attachments: [{ _id: 'file-1' }, 'file-2'] }, + { attachments: ['file-3'] }, + ], + }; + + expect(collectIdsAtPath(object, 'recipients.attachments')).toEqual([ + 'file-1', + 'file-2', + 'file-3', + ]); + }); +}); diff --git a/src/database/__tests__/database.test.js b/src/database/__tests__/database.test.js index 5265bf4..185efbd 100644 --- a/src/database/__tests__/database.test.js +++ b/src/database/__tests__/database.test.js @@ -32,6 +32,7 @@ jest.unstable_mockModule('../../utils.js', () => ({ return result; }), getFieldsByRef: jest.fn(() => []), + collectIdsAtPath: jest.fn(() => []), getQueryToCacheKey: jest.fn(({ model, id }) => `${model}:${id}`), modelHasRef: jest.fn(() => false), newAuditLog: jest.fn(), diff --git a/src/database/database.js b/src/database/database.js index 3e305ed..9432135 100644 --- a/src/database/database.js +++ b/src/database/database.js @@ -13,6 +13,7 @@ import { expandObjectIds, modelHasRef, getFieldsByRef, + collectIdsAtPath, getQueryToCacheKey, editAuditLog, distributeUpdate, @@ -1128,16 +1129,7 @@ export const listObjectDependencies = async ({ model, id }) => { const targetModel = entry?.model; if (!targetModel || !targetModel.schema) continue; - const referencingPaths = []; - - targetModel.schema.eachPath((pathName, schemaType) => { - const directRef = schemaType?.options?.ref; - const arrayRef = schemaType?.caster?.options?.ref; - const refName = directRef || arrayRef; - if (refName === parentModelName) { - referencingPaths.push(pathName); - } - }); + const referencingPaths = getFieldsByRef(targetModel, parentModelName); if (referencingPaths.length === 0) continue; @@ -1205,37 +1197,30 @@ export const editObject = async ({ model, id, updateData, user, populate, recalc const previousExpandedObject = expandObjectIds(previousObject); - // Check if any model parameters have ref: 'file' and flush files if so - if (modelHasRef(model, 'file')) { - logger.debug(`Model ${model.modelName} has file references, checking for files to flush`); - const fileFields = getFieldsByRef(model, 'file'); - - for (const fieldName of fileFields) { - const fieldValue = previousExpandedObject[fieldName]; - - if (fieldValue) { - if (Array.isArray(fieldValue)) { - // Handle file arrays - for (const fileRef of fieldValue) { - if (fileRef && fileRef._id) { - logger.debug(`Flushing file from array field ${fieldName}: ${fileRef._id}`); - await flushFile({ id: fileRef._id, user }); - } - } - } else if (fieldValue._id) { - // Handle single file reference - logger.debug(`Flushing file from field ${fieldName}: ${fieldValue._id}`); - await flushFile({ id: fieldValue._id, user }); - } - } - } - } - const updatedObject = mergeObjectUpdates( _.cloneDeep(previousExpandedObject), expandObjectIds(updateData) ); + // Flush files that this update actually dropped, not files still referenced + // on nested paths such as recipients.attachments. + if (modelHasRef(model, 'file')) { + logger.debug(`Model ${model.modelName} has file references, checking for files to flush`); + const fileFields = getFieldsByRef(model, 'file'); + const nextFileIds = new Set( + fileFields.flatMap((fieldName) => collectIdsAtPath(updatedObject, fieldName)) + ); + + for (const fieldName of fileFields) { + const previousIds = collectIdsAtPath(previousExpandedObject, fieldName); + for (const fileId of previousIds) { + if (nextFileIds.has(fileId)) continue; + logger.debug(`Flushing removed file from field ${fieldName}: ${fileId}`); + await flushFile({ id: fileId, user }); + } + } + } + // Audit log before update await editAuditLog(previousExpandedObject, updatedObject, id, parentType, user); diff --git a/src/database/schemas/management/emailmessage.schema.js b/src/database/schemas/management/emailmessage.schema.js index 5c4f066..955377d 100644 --- a/src/database/schemas/management/emailmessage.schema.js +++ b/src/database/schemas/management/emailmessage.schema.js @@ -3,6 +3,32 @@ import { generateId } from '../../utils.js'; const { Schema } = mongoose; +const emailRecipientSchema = new Schema( + { + recipientEmail: { type: String, required: true }, + recipientType: { type: String, enum: ['client', 'vendor'], required: false }, + recipient: { + type: Schema.Types.ObjectId, + refPath: 'recipients.recipientType', + required: false, + }, + objectType: { type: String, required: false }, + object: { + type: Schema.Types.ObjectId, + refPath: 'recipients.objectType', + required: false, + }, + pixelId: { type: String, required: true }, + messageId: { type: String, required: false }, + attachments: [{ type: Schema.Types.ObjectId, ref: 'file', required: false }], + read: { type: Boolean, required: true, default: false }, + sent: { type: Boolean, required: true, default: false }, + sentAt: { type: Date, required: false }, + readAt: { type: Date, required: false }, + }, + { _id: true } +); + const emailMessageSchema = new Schema( { _reference: { type: String, default: () => generateId()() }, @@ -20,36 +46,29 @@ const emailMessageSchema = new Schema( ref: 'emailAccount', required: true, }, - recipientEmail: { type: String, required: true }, - recipientType: { type: String, enum: ['client', 'vendor'], required: false }, - recipient: { - type: Schema.Types.ObjectId, - refPath: 'recipientType', - required: false, - }, + recipients: { type: [emailRecipientSchema], required: true, default: [] }, fromEmail: { type: String, required: true, immutable: true }, - messageId: { type: String, required: false }, subject: { type: String, required: false, default: '' }, content: { type: String, required: false, default: '' }, - attachments: [{ type: Schema.Types.ObjectId, ref: 'file', required: false }], state: { type: { type: String, required: true, default: 'queued' }, progress: { type: Number, required: false, default: 0 }, message: { type: String, required: false }, }, - read: { type: Boolean, required: true, default: false }, sentAt: { type: Date, required: false }, - readAt: { type: Date, required: false }, + firstReadAt: { type: Date, required: false }, + lastReadAt: { type: Date, required: false }, }, { timestamps: true, suppressReservedKeysWarning: true } ); emailMessageSchema.index({ name: 'text', - recipientEmail: 'text', fromEmail: 'text', objectType: 'text', + 'recipients.recipientEmail': 'text', }); +emailMessageSchema.index({ 'recipients.pixelId': 1 }, { unique: true, sparse: true }); emailMessageSchema.virtual('id').get(function () { return this._id; }); diff --git a/src/routes/management/emailmessages.js b/src/routes/management/emailmessages.js index 46f4ad4..c14fa86 100644 --- a/src/routes/management/emailmessages.js +++ b/src/routes/management/emailmessages.js @@ -22,28 +22,22 @@ const filters = [ 'emailAccount', 'objectType', 'objects', - 'recipientEmail', - 'recipientType', - 'recipient', 'fromEmail', - 'messageId', 'state', - 'read', 'sentAt', - 'readAt', + 'firstReadAt', + 'lastReadAt', 'createdAt', 'updatedAt', '_reference', ]; const sorters = [ 'name', - 'recipientEmail', 'fromEmail', - 'messageId', 'state', - 'read', 'sentAt', - 'readAt', + 'firstReadAt', + 'lastReadAt', 'createdAt', 'updatedAt', ]; @@ -51,9 +45,7 @@ const propertyFilters = [ 'emailTemplate', 'emailAccount', 'objectType', - 'recipientType', 'state', - 'read', ]; router.get('/', isAuthenticated, checkPermissions('emailMessage', 'list'), async (req, res) => { diff --git a/src/services/management/__tests__/emailmessages.test.js b/src/services/management/__tests__/emailmessages.test.js index 5fad49b..23cd75f 100644 --- a/src/services/management/__tests__/emailmessages.test.js +++ b/src/services/management/__tests__/emailmessages.test.js @@ -14,6 +14,7 @@ jest.unstable_mockModule('../../../database/schemas/management/emailmessage.sche emailMessageModel: { modelName: 'emailMessage', findById: jest.fn(), + findOne: jest.fn(), findByIdAndUpdate: jest.fn(), }, })); @@ -175,7 +176,15 @@ describe('email message creation', () => { expect.objectContaining({ properties: ['state', 'emailAccount'], filter: { read: false }, - populate: ['emailTemplate', 'emailAccount', 'objects', 'object', 'recipient', 'attachments'], + populate: [ + 'emailTemplate', + 'emailAccount', + 'objects', + 'object', + 'recipients.recipient', + 'recipients.object', + 'recipients.attachments', + ], }) ); }); @@ -187,7 +196,7 @@ describe('email message creation', () => { _id: 'message-1', emailTemplate: 'template-1', object: 'object-1', - recipientEmail: 'client@example.com', + recipients: [{ recipientEmail: 'client@example.com' }], }), }); templateManager.renderEmailTemplate.mockResolvedValue({ @@ -208,13 +217,101 @@ describe('email message creation', () => { user, updateData: expect.objectContaining({ state: { type: 'sent', progress: 1 }, - messageId: '', sentAt: expect.any(Date), + recipients: [ + expect.objectContaining({ + recipientEmail: 'client@example.com', + sent: true, + messageId: '', + }), + ], }), }) ); }); + it('sends recipients one at a time with a 500ms gap and live progress', async () => { + const user = { _id: 'user-1' }; + const originalSetTimeout = global.setTimeout; + const delays = []; + jest.spyOn(global, 'setTimeout').mockImplementation((fn, ms, ...args) => { + if (ms === 500) { + delays.push(ms); + fn(); + return 0; + } + return originalSetTimeout(fn, ms, ...args); + }); + emailMessageModel.findById.mockReturnValue({ + lean: jest.fn().mockResolvedValue({ + _id: 'message-1', + emailTemplate: 'template-1', + objectType: 'invoice', + recipients: [ + { recipientEmail: 'one@example.com', pixelId: 'pixel-1' }, + { recipientEmail: 'two@example.com', pixelId: 'pixel-2' }, + ], + }), + }); + templateManager.renderEmailTemplate.mockResolvedValue({ + subject: 'Invoice', + html: '

Hi

', + htmlWithPixel: '

Hi

', + }); + editObject.mockResolvedValue({ _id: 'message-1' }); + + try { + await processEmailMessage( + 'message-1', + { fromEmail: 'farm@example.com', host: 'smtp.example.com' }, + user + ); + + expect(templateManager.renderEmailTemplate).toHaveBeenCalledTimes(2); + expect(workerMessages).toHaveLength(2); + expect(workerMessages.map((payload) => payload.recipientEmail)).toEqual([ + 'one@example.com', + 'two@example.com', + ]); + expect(delays).toEqual([500]); + expect(editObject).toHaveBeenCalledWith( + expect.objectContaining({ + updateData: expect.objectContaining({ + state: { + type: 'sending', + progress: 0.25, + message: 'Sending email 1 of 2...', + }, + }), + }) + ); + expect(editObject).toHaveBeenCalledWith( + expect.objectContaining({ + updateData: expect.objectContaining({ + state: { + type: 'sending', + progress: 0.75, + message: 'Sending email 2 of 2...', + }, + }), + }) + ); + expect(editObject).toHaveBeenCalledWith( + expect.objectContaining({ + updateData: expect.objectContaining({ + state: { type: 'sent', progress: 1 }, + recipients: [ + expect.objectContaining({ recipientEmail: 'one@example.com', sent: true }), + expect.objectContaining({ recipientEmail: 'two@example.com', sent: true }), + ], + }), + }) + ); + } finally { + global.setTimeout.mockRestore(); + } + }); + it('renders document template attachments into files before sending', async () => { const user = { _id: 'user-1' }; const pdfBytes = Buffer.from('pdf-bytes'); @@ -225,7 +322,7 @@ describe('email message creation', () => { emailTemplate: 'template-1', objectType: 'invoice', object: 'object-1', - recipientEmail: 'client@example.com', + recipients: [{ recipientEmail: 'client@example.com' }], }), }); templateLean.mockResolvedValue({ @@ -291,8 +388,14 @@ describe('email message creation', () => { expect(editObject).toHaveBeenCalledWith( expect.objectContaining({ updateData: expect.objectContaining({ - state: { type: 'sending', progress: 0.7, message: 'Sending email...' }, - attachments: ['file-new'], + state: { type: 'sent', progress: 1 }, + recipients: [ + expect.objectContaining({ + recipientEmail: 'client@example.com', + attachments: ['file-new'], + messageId: '', + }), + ], }), }) ); @@ -313,7 +416,7 @@ describe('email message creation', () => { _id: 'message-1', emailTemplate: 'template-1', object: 'object-1', - recipientEmail: 'client@example.com', + recipients: [{ recipientEmail: 'client@example.com' }], }), }); templateLean.mockResolvedValue({ @@ -350,7 +453,12 @@ describe('email message creation', () => { expect(editObject).toHaveBeenCalledWith( expect.objectContaining({ updateData: expect.objectContaining({ - attachments: ['file-1'], + recipients: [ + expect.objectContaining({ + recipientEmail: 'client@example.com', + attachments: ['file-1'], + }), + ], }), }) ); @@ -364,8 +472,20 @@ describe('email message creation', () => { }); it('uses a system audit actor for automatic read updates', async () => { - req.params = { id: '507f1f77bcf86cd799439011' }; - editObject.mockResolvedValue({ _id: '507f1f77bcf86cd799439011', read: true }); + req.params = { id: 'pixel-recipient-1' }; + emailMessageModel.findOne.mockReturnValue({ + lean: jest.fn().mockResolvedValue({ + _id: '507f1f77bcf86cd799439011', + recipients: [ + { + recipientEmail: 'client@example.com', + pixelId: 'pixel-recipient-1', + read: false, + }, + ], + }), + }); + editObject.mockResolvedValue({ _id: '507f1f77bcf86cd799439011' }); await markEmailMessageAsReadRouteHandler(req, res); @@ -373,9 +493,16 @@ describe('email message creation', () => { expect.objectContaining({ id: expect.anything(), updateData: expect.objectContaining({ - read: true, - readAt: expect.any(Date), + firstReadAt: expect.any(Date), + lastReadAt: expect.any(Date), state: { type: 'read' }, + recipients: [ + expect.objectContaining({ + recipientEmail: 'client@example.com', + read: true, + readAt: expect.any(Date), + }), + ], }), user: 'system', }) diff --git a/src/services/management/emailmessages.js b/src/services/management/emailmessages.js index effaa65..b76a8f5 100644 --- a/src/services/management/emailmessages.js +++ b/src/services/management/emailmessages.js @@ -6,6 +6,7 @@ import { fileModel } from '../../database/schemas/management/file.schema.js'; import { userNotifierModel } from '../../database/schemas/misc/usernotifier.schema.js'; import log4js from 'log4js'; import mongoose from 'mongoose'; +import { nanoid } from 'nanoid'; import { Worker } from 'worker_threads'; import { randomUUID } from 'crypto'; import { fileURLToPath } from 'url'; @@ -37,7 +38,77 @@ import { const logger = log4js.getLogger('Email Messages'); logger.level = config.server.logLevel; const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const populate = ['emailTemplate', 'emailAccount', 'objects', 'object', 'recipient', 'attachments']; +const populate = [ + 'emailTemplate', + 'emailAccount', + 'objects', + 'object', + 'recipients.recipient', + 'recipients.object', + 'recipients.attachments', +]; + +const PIXEL_ID_LENGTH = 48; +const SEND_GAP_MS = 500; + +const roundProgress = (value) => Math.min(1, Math.round(value * 1000) / 1000); + +const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +const lastDate = (values) => { + const times = values + .filter((value) => value != null) + .map((value) => new Date(value).getTime()) + .filter((value) => !Number.isNaN(value)); + if (!times.length) return undefined; + return new Date(Math.max(...times)); +}; + +const firstDate = (values) => { + const times = values + .filter((value) => value != null) + .map((value) => new Date(value).getTime()) + .filter((value) => !Number.isNaN(value)); + if (!times.length) return undefined; + return new Date(Math.min(...times)); +}; + +const normalizeRecipients = (body = {}) => { + const source = Array.isArray(body.recipients) ? body.recipients : []; + const recipients = source + .map((recipient) => ({ + recipientEmail: recipient?.recipientEmail, + recipientType: recipient?.recipientType, + recipient: recipient?.recipient?._id ?? recipient?.recipient, + objectType: recipient?.objectType || body.objectType, + object: recipient?.object?._id ?? recipient?.object, + pixelId: nanoid(PIXEL_ID_LENGTH), + attachments: [], + read: false, + sent: false, + })) + .filter((recipient) => recipient.recipientEmail); + + if (recipients.length) return recipients; + + if (body.recipientEmail) { + return [ + { + recipientEmail: body.recipientEmail, + recipientType: body.recipientType, + recipient: body.recipient?._id ?? body.recipient, + objectType: body.objectType, + object: body.object?._id ?? body.object, + pixelId: nanoid(PIXEL_ID_LENGTH), + attachments: [], + read: false, + sent: false, + }, + ]; + } + + return []; +}; let worker; const pending = new Map(); @@ -155,7 +226,30 @@ async function loadFileRecord(file) { return record; } -async function resolveEmailAttachments(emailMessage, user) { +function recipientObjectRefs(emailMessage, recipient) { + if (recipient?.object != null) return [recipient.object]; + return relatedObjectRefs(emailMessage); +} + +async function loadRecipientObjects(emailMessage, recipient) { + const refs = recipientObjectRefs(emailMessage, recipient); + if (!refs.length) return []; + const modelEntry = getModelByName(emailMessage.objectType); + if (!modelEntry?.model) { + return []; + } + const objects = []; + for (const ref of refs) { + const object = await getObject({ + model: modelEntry.model, + id: ref?._id || ref, + }); + objects.push(object); + } + return objects; +} + +async function resolveEmailAttachments(emailMessage, recipient, user) { const template = await emailTemplateModel .findById(emailMessage.emailTemplate) .populate({ path: 'attachments.file', strictPopulate: false }) @@ -163,24 +257,16 @@ async function resolveEmailAttachments(emailMessage, user) { if (!template) throw new Error('Email template not found.'); const mailAttachments = []; - const fileIds = []; + const attachments = []; for (const attachment of template.attachments || []) { if (attachment.type === 'documentTemplate') { const templateId = attachment.file?._id ?? attachment.file; if (!templateId) throw new Error('Document template attachment is missing a file.'); - const modelEntry = getModelByName(emailMessage.objectType); - if (!modelEntry?.model) { + if (!getModelByName(emailMessage.objectType)?.model) { throw new Error(`Unknown object type for email attachment: ${emailMessage.objectType}`); } - const objects = []; - for (const ref of relatedObjectRefs(emailMessage)) { - const object = await getObject({ - model: modelEntry.model, - id: ref?._id || ref, - }); - objects.push(object); - } + const objects = await loadRecipientObjects(emailMessage, recipient); const rendered = await templateManager.renderDownload( templateId, undefined, @@ -209,10 +295,11 @@ async function resolveEmailAttachments(emailMessage, user) { metaData: { source: 'emailTemplateAttachment', emailMessage: String(emailMessage._id), + pixelId: recipient?.pixelId, documentTemplate: String(templateId), }, }); - fileIds.push(created._id); + attachments.push(created._id); mailAttachments.push({ filename: names.filename, content: buffer, @@ -228,7 +315,7 @@ async function resolveEmailAttachments(emailMessage, user) { const body = await downloadFile(BUCKETS.FILES, `files/${fileRecord._id}${extension}`); const buffer = await streamToBuffer(body); const names = splitAttachmentName(attachment.fileName || fileRecord.name, extension, 0, 1); - fileIds.push(fileRecord._id); + attachments.push(fileRecord._id); mailAttachments.push({ filename: names.filename, content: buffer, @@ -237,7 +324,7 @@ async function resolveEmailAttachments(emailMessage, user) { } } - return { fileIds, mailAttachments }; + return { attachments, mailAttachments }; } const updateEmailMessage = async (id, updateData, user) => { @@ -263,58 +350,112 @@ const report = async (id, success, message) => { export const processEmailMessage = async (id, account, user) => { try { - await updateEmailMessage( - id, - { state: { type: 'rendering', progress: 0.2, message: 'Rendering email...' } }, - user - ); const emailMessage = await emailMessageModel.findById(id).lean(); if (!emailMessage) throw new Error('Email message not found.'); - const rendered = await templateManager.renderEmailTemplate( - emailMessage.emailTemplate, - undefined, - undefined, - templateDataFromObjects(relatedObjectRefs(emailMessage)), - {}, - false, - String(id) - ); - if (rendered?.error) throw new Error(rendered.error); + const recipients = Array.isArray(emailMessage.recipients) ? [...emailMessage.recipients] : []; + if (!recipients.length) throw new Error('Email message has no recipients.'); - const { fileIds, mailAttachments } = await resolveEmailAttachments(emailMessage, user); + const total = recipients.length; + let subject = emailMessage.subject; + let content = emailMessage.content; - await updateEmailMessage( - id, - { - state: { type: 'sending', progress: 0.7, message: 'Sending email...' }, + for (let index = 0; index < total; index += 1) { + const recipient = recipients[index]; + const suffix = total > 1 ? ` ${index + 1} of ${total}` : ''; + await updateEmailMessage( + id, + { + state: { + type: 'rendering', + progress: roundProgress(index / total), + message: `Rendering email${suffix}...`, + }, + }, + user + ); + + const objects = await loadRecipientObjects(emailMessage, recipient); + const rendered = await templateManager.renderEmailTemplate( + emailMessage.emailTemplate, + undefined, + undefined, + templateDataFromObjects(objects), + {}, + false, + recipient.pixelId + ); + if (rendered?.error) throw new Error(rendered.error); + subject = rendered.subject; + content = rendered.html; + + const { attachments, mailAttachments } = await resolveEmailAttachments( + emailMessage, + recipient, + user + ); + await updateEmailMessage( + id, + { + state: { + type: 'sending', + progress: roundProgress((index + 0.5) / total), + message: `Sending email${suffix}...`, + }, + }, + user + ); + const result = await deliver({ + kind: 'emailMessage', + recipientEmail: recipient.recipientEmail, + from: account.fromName ? `${account.fromName} <${account.fromEmail}>` : account.fromEmail, subject: rendered.subject, - content: rendered.html, - attachments: fileIds, - }, - user - ); - const result = await deliver({ - kind: 'emailMessage', - recipientEmail: emailMessage.recipientEmail, - from: account.fromName ? `${account.fromName} <${account.fromEmail}>` : account.fromEmail, - subject: rendered.subject, - html: rendered.htmlWithPixel, - attachments: mailAttachments, - smtpConfig: { - host: account.host, - port: account.port, - secure: account.secure, - username: account.username, - password: account.password, - }, - }); + html: rendered.htmlWithPixel, + attachments: mailAttachments, + smtpConfig: { + host: account.host, + port: account.port, + secure: account.secure, + username: account.username, + password: account.password, + }, + }); + const sentAt = new Date(); + recipients[index] = { + ...recipient, + sent: true, + sentAt, + messageId: result.messageId || undefined, + attachments, + }; + await updateEmailMessage( + id, + { + state: { + type: 'sending', + progress: roundProgress((index + 1) / total), + message: + total > 1 ? `Sent ${index + 1} of ${total}` : 'Sending email...', + }, + subject, + content, + recipients: recipients.map((entry) => ({ ...entry })), + sentAt: lastDate(recipients.map((entry) => entry.sentAt)), + }, + user + ); + if (index < total - 1) { + await wait(SEND_GAP_MS); + } + } await updateEmailMessage( id, { state: { type: 'sent', progress: 1 }, - messageId: result.messageId || undefined, - sentAt: new Date(), + subject, + content, + recipients: recipients.map((entry) => ({ ...entry })), + sentAt: lastDate(recipients.map((recipient) => recipient.sentAt)), }, user ); @@ -445,6 +586,11 @@ export const newEmailMessageRouteHandler = async (req, res) => { }); } + const recipients = normalizeRecipients(req.body); + if (!recipients.length) { + return res.status(400).send({ error: 'At least one recipient is required.', code: 400 }); + } + const newData = { createdAt: new Date(), updatedAt: new Date(), @@ -453,12 +599,9 @@ export const newEmailMessageRouteHandler = async (req, res) => { objectType: req.body.objectType, objects: relatedObjectIdsFromBody(req.body), emailAccount: account._id, - recipientEmail: req.body.recipientEmail, - recipientType: req.body.recipientType, - recipient: req.body.recipient?._id ?? req.body.recipient, + recipients, fromEmail: account.fromEmail, state: { type: 'queued', progress: 0, message: 'Queued' }, - read: false, }; const result = await newObject({ @@ -483,7 +626,11 @@ export const newEmailMessageRouteHandler = async (req, res) => { logger.debug(`New email message with ID: ${result._id}`); res.send(withLegacyObjects(result)); - setImmediate(() => processEmailMessage(result._id, account, req.user)); + setImmediate(() => { + processEmailMessage(result._id, account, req.user).catch((error) => { + logger.error(`Email message ${result._id} failed:`, error); + }); + }); }; export const getEmailMessageStatsRouteHandler = async (req, res) => { @@ -541,19 +688,36 @@ export const getEmailMessageNeighborsRouteHandler = async ( }; export const markEmailMessageAsReadRouteHandler = async (req, res) => { - const id = new mongoose.Types.ObjectId(req.params.id); + const pixelId = req.params.id; - logger.trace(`Email message with ID: ${id}`); + logger.trace(`Email message pixel: ${pixelId}`); + const emailMessage = await emailMessageModel.findOne({ 'recipients.pixelId': pixelId }).lean(); + if (!emailMessage) { + return res.status(404).send({ error: 'Email message not found.', code: 404 }); + } + + const now = new Date(); + const recipients = (Array.isArray(emailMessage.recipients) ? emailMessage.recipients : []).map( + (recipient) => + recipient?.pixelId === pixelId + ? { + ...recipient, + read: true, + readAt: recipient?.readAt || now, + } + : recipient + ); const updateData = { - updatedAt: new Date(), - read: true, - readAt: new Date(), + updatedAt: now, + recipients, + firstReadAt: emailMessage.firstReadAt || firstDate(recipients.map((recipient) => recipient.readAt)) || now, + lastReadAt: now, state: { type: 'read' }, }; const result = await editObject({ model: emailMessageModel, - id, + id: emailMessage._id, updateData, user: 'system', populate, @@ -565,7 +729,7 @@ export const markEmailMessageAsReadRouteHandler = async (req, res) => { return; } - logger.debug(`Marked email message as read with ID: ${id}`); + logger.debug(`Marked email message as read with pixel: ${pixelId}`); res.send(withLegacyObjects(result)); }; diff --git a/src/services/misc/templateresources.js b/src/services/misc/templateresources.js index 9c281d4..9b5ffa3 100644 --- a/src/services/misc/templateresources.js +++ b/src/services/misc/templateresources.js @@ -15,7 +15,7 @@ export async function getTemplateResourceRouteHandler(req, res) { if (!TOKEN_PATTERN.test(token || '')) return notFound(res); try { - const payload = await redisServer.getAndDeleteKey(`${TEMPLATE_RESOURCE_KEY_PREFIX}${token}`); + const payload = await redisServer.getKey(`${TEMPLATE_RESOURCE_KEY_PREFIX}${token}`); if (!payload?.fileId || !mongoose.Types.ObjectId.isValid(payload.fileId)) { return notFound(res); } diff --git a/src/templates/__tests__/templatemanager.test.js b/src/templates/__tests__/templatemanager.test.js index 367f20b..754b6ee 100644 --- a/src/templates/__tests__/templatemanager.test.js +++ b/src/templates/__tests__/templatemanager.test.js @@ -73,6 +73,7 @@ jest.unstable_mockModule('../../database/redis.js', () => ({ redisServer: { setKey: jest.fn().mockResolvedValue(undefined), getKey: jest.fn().mockResolvedValue(null), + deleteKey: jest.fn().mockResolvedValue(undefined), }, })); @@ -123,6 +124,7 @@ const ejs = (await import('ejs')).default; const { generatePDF } = await import('../pdffactory.js'); const { eventManager } = await import('../../events/eventmanager.js'); const { redisServer } = await import('../../database/redis.js'); +const { TEMPLATE_RESOURCE_TTL_SECONDS } = await import('../templateresources.js'); describe('stripDocumentTemplateReference', () => { it('strips a DTP: prefix', () => { @@ -248,27 +250,36 @@ describe('TemplateManager', () => { describe('renderTemplate', () => { it('replaces each valid resource occurrence before rendering', async () => { - getObject.mockResolvedValue({ - documentSize: { width: 100, height: 100, infiniteHeight: false }, - global: true, - resources: [{ name: 'logo', file: { _id: 'file-1' } }], - }); + jest.useFakeTimers(); + try { + getObject.mockResolvedValue({ + documentSize: { width: 100, height: 100, infiniteHeight: false }, + global: true, + resources: [{ name: 'logo', file: { _id: 'file-1' } }], + }); - await templateManager.renderTemplate( - 'temp-id', - '' - ); + await templateManager.renderTemplate( + 'temp-id', + '' + ); - const renderedSource = ejs.render.mock.calls[0][0]; - const tokens = [...renderedSource.matchAll(/template-resources\/([a-f0-9]{64})/g)]; - expect(tokens).toHaveLength(2); - expect(tokens[0][1]).not.toBe(tokens[1][1]); - expect(redisServer.setKey).toHaveBeenCalledTimes(2); - expect(redisServer.setKey).toHaveBeenCalledWith( - expect.stringMatching(/^templateresources:[a-f0-9]{64}$/), - { fileId: 'file-1' }, - 3 - ); + const renderedSource = ejs.render.mock.calls[0][0]; + const tokens = [...renderedSource.matchAll(/template-resources\/([a-f0-9]{64})/g)]; + expect(tokens).toHaveLength(2); + expect(tokens[0][1]).not.toBe(tokens[1][1]); + expect(redisServer.setKey).toHaveBeenCalledTimes(2); + expect(redisServer.setKey).toHaveBeenCalledWith( + expect.stringMatching(/^templateresources:[a-f0-9]{64}$/), + { fileId: 'file-1' }, + TEMPLATE_RESOURCE_TTL_SECONDS + ); + expect(redisServer.deleteKey).not.toHaveBeenCalled(); + + await jest.advanceTimersByTimeAsync(10000); + expect(redisServer.deleteKey).toHaveBeenCalledTimes(2); + } finally { + jest.useRealTimers(); + } }); it('resolves parent resources from the parent owner only', async () => { @@ -290,12 +301,12 @@ describe('TemplateManager', () => { expect(redisServer.setKey).toHaveBeenCalledWith( expect.any(String), { fileId: 'child-file' }, - 3 + TEMPLATE_RESOURCE_TTL_SECONDS ); expect(redisServer.setKey).toHaveBeenCalledWith( expect.any(String), { fileId: 'parent-file' }, - 3 + TEMPLATE_RESOURCE_TTL_SECONDS ); }); @@ -789,6 +800,22 @@ describe('TemplateManager', () => { expect(result.pdf).toBeDefined(); }); + it('clears resource keys immediately after rendering a PDF', async () => { + getObject.mockResolvedValue({ + documentSize: { width: 100, height: 100, infiniteHeight: false }, + global: true, + resources: [{ name: 'logo', file: { _id: 'file-1' } }], + }); + + await templateManager.renderPDF('temp-id', ''); + + expect(generatePDF).toHaveBeenCalled(); + expect(redisServer.deleteKey).toHaveBeenCalledTimes(1); + expect(redisServer.deleteKey).toHaveBeenCalledWith( + expect.stringMatching(/^templateresources:[a-f0-9]{64}$/) + ); + }); + it('adds padding to page size when padding is requested', async () => { getObject.mockResolvedValue({ documentSize: { diff --git a/src/templates/__tests__/templateresources.test.js b/src/templates/__tests__/templateresources.test.js index 786e774..7b08357 100644 --- a/src/templates/__tests__/templateresources.test.js +++ b/src/templates/__tests__/templateresources.test.js @@ -1,20 +1,24 @@ import { jest } from '@jest/globals'; const setKey = jest.fn(); +const deleteKey = jest.fn(); jest.unstable_mockModule('../../database/redis.js', () => ({ - redisServer: { setKey }, + redisServer: { setKey, deleteKey }, })); jest.unstable_mockModule('../../config.js', () => ({ default: { app: { urlApi: 'https://api.example.test' } }, })); const { + clearTemplateResourceKeys, createTemplateResourceUrl, normalizeTemplateResourceName, normalizeTemplateResources, preprocessTemplateResources, + runWithTemplateResourceSession, validateTemplateResources, + TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS, TEMPLATE_RESOURCE_KEY_PREFIX, TEMPLATE_RESOURCE_TTL_SECONDS, } = await import('../templateresources.js'); @@ -64,7 +68,7 @@ describe('template resources', () => { expect(result).toContain(''); }); - it('creates a crypto-random one-time token with an exact three-second TTL', async () => { + it('creates a crypto-random one-time token with a fallback TTL', async () => { const first = await createTemplateResourceUrl('507f1f77bcf86cd799439011'); const second = await createTemplateResourceUrl('507f1f77bcf86cd799439011'); @@ -76,6 +80,34 @@ describe('template resources', () => { { fileId: '507f1f77bcf86cd799439011' }, TEMPLATE_RESOURCE_TTL_SECONDS ); - expect(TEMPLATE_RESOURCE_TTL_SECONDS).toBe(3); + expect(TEMPLATE_RESOURCE_TTL_SECONDS).toBeGreaterThan( + TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS / 1000 + ); + }); + + it('clears session keys immediately or after the HTML delay', async () => { + jest.useFakeTimers(); + try { + await runWithTemplateResourceSession(async () => { + const url = await createTemplateResourceUrl('507f1f77bcf86cd799439011'); + const key = `${TEMPLATE_RESOURCE_KEY_PREFIX}${url.split('/').at(-1)}`; + + await clearTemplateResourceKeys(TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS); + expect(deleteKey).not.toHaveBeenCalled(); + + await jest.advanceTimersByTimeAsync(TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS); + expect(deleteKey).toHaveBeenCalledWith(key); + }); + + deleteKey.mockClear(); + await runWithTemplateResourceSession(async () => { + const url = await createTemplateResourceUrl('507f1f77bcf86cd799439012'); + const key = `${TEMPLATE_RESOURCE_KEY_PREFIX}${url.split('/').at(-1)}`; + await clearTemplateResourceKeys(); + expect(deleteKey).toHaveBeenCalledWith(key); + }); + } finally { + jest.useRealTimers(); + } }); }); diff --git a/src/templates/assets/previewpagination.js b/src/templates/assets/previewpagination.js index 670f927..50e9d89 100644 --- a/src/templates/assets/previewpagination.js +++ b/src/templates/assets/previewpagination.js @@ -4,6 +4,7 @@ var UNSPLITTABLE = 'tr, thead, tfoot, img, svg, hr, canvas, video, picture, col, colgroup'; var IGNORED = 'script, style, link, meta, .documentPageBreak'; var PAGE_BREAK_SELECTOR = '.documentPageBreak, pagebreak'; + var REPEATING = '.documentHeader, .documentFooter'; function markPaginated() { document.documentElement.setAttribute('data-paginated', 'true'); @@ -72,7 +73,7 @@ function findFirstOverflowingLeaf(page, clipBottom, locked) { function walk(el) { - if (locked.has(el) || isIgnored(el)) { + if (locked.has(el) || isIgnored(el) || isRepeating(el)) { return null; } if (!isOverflowing(el, clipBottom)) { @@ -126,11 +127,64 @@ return clone; } + function isRepeating(el) { + return el.matches(REPEATING); + } + + function containsRepeating(el) { + return isRepeating(el) || !!el.querySelector(REPEATING); + } + + function snapshotChildren(el) { + var children = []; + var child = el.firstElementChild; + while (child) { + children.push(child); + child = child.nextElementSibling; + } + return children; + } + + function cloneRepeating(el) { + var clone = el.cloneNode(true); + clone.removeAttribute('id'); + var nested = clone.querySelectorAll('[id]'); + for (var i = 0; i < nested.length; i += 1) { + nested[i].removeAttribute('id'); + } + return clone; + } + + function copyRepeatingInPlace(dst, originalChildren, continuation) { + var before = []; + var after = []; + var bucket = before; + for (var i = 0; i < originalChildren.length; i += 1) { + var child = originalChildren[i]; + if (child === continuation) { + bucket = after; + continue; + } + if (containsRepeating(child)) { + bucket.push(cloneRepeating(child)); + } + } + var ref = dst.firstElementChild; + for (var i = 0; i < before.length; i += 1) { + dst.insertBefore(before[i], ref); + } + for (var i = 0; i < after.length; i += 1) { + dst.appendChild(after[i]); + } + } + function moveFrom(startNode, destination) { var node = startNode; while (node) { var next = node.nextElementSibling; - destination.appendChild(node); + if (!containsRepeating(node)) { + destination.appendChild(node); + } node = next; } } @@ -141,6 +195,10 @@ for (var i = 0; i < breaks.length; i += 1) { breaks[i].remove(); } + var repeating = clone.querySelectorAll(REPEATING); + for (var i = 0; i < repeating.length; i += 1) { + repeating[i].remove(); + } if (clone.querySelector('img, svg, canvas, video, picture, hr, table, .documentBarcode')) { return false; } @@ -158,6 +216,12 @@ parent = parent.parentElement; } + var originalPageChildren = snapshotChildren(page); + var originalAncestorChildren = []; + for (var i = 0; i < ancestors.length; i += 1) { + originalAncestorChildren.push(snapshotChildren(ancestors[i])); + } + var clonedAncestors = []; for (var i = 0; i < ancestors.length; i += 1) { clonedAncestors.push(cloneEmpty(ancestors[i])); @@ -182,6 +246,16 @@ moveFrom(ancestors[i].nextElementSibling, destination); } + for (var i = 0; i < ancestors.length; i += 1) { + var continuation = i === 0 ? splitNode : ancestors[i - 1]; + copyRepeatingInPlace(clonedAncestors[i], originalAncestorChildren[i], continuation); + } + copyRepeatingInPlace( + newPage, + originalPageChildren, + ancestors.length > 0 ? ancestors[ancestors.length - 1] : splitNode + ); + if (isEffectivelyEmpty(newPage)) { return null; } @@ -225,7 +299,7 @@ } var nextPage = moveOverflowToNewPage(page, splitNode, { - discardSplitNode: discardSplitNode + discardSplitNode: discardSplitNode, }); if (!nextPage && pageBreak && pageBreak.isConnected) { pageBreak.remove(); @@ -323,10 +397,28 @@ page = nextPage; } - document.documentElement.setAttribute( - 'data-page-count', - String(paginationApi.getPageCount()) - ); + document.documentElement.setAttribute('data-page-count', String(paginationApi.getPageCount())); + + applyPagePlaceholders(); + } + + function applyPagePlaceholders() { + var pages = getPageElements(); + var totalPages = pages.length || 1; + for (var i = 0; i < pages.length; i += 1) { + var pageNumber = String(i + 1); + var total = String(totalPages); + var nodes = pages[i].querySelectorAll('*'); + for (var j = 0; j < nodes.length; j += 1) { + var html = String(nodes[j].innerHTML || '').trim(); + if (html.includes('!pageNumber')) { + nodes[j].innerHTML = nodes[j].innerHTML.replaceAll('!pageNumber', pageNumber); + } + if (html.includes('!totalPages')) { + nodes[j].innerHTML = nodes[j].innerHTML.replaceAll('!totalPages', total); + } + } + } } function waitForImages() { diff --git a/src/templates/templatemanager.js b/src/templates/templatemanager.js index 5fac0eb..6c334ec 100644 --- a/src/templates/templatemanager.js +++ b/src/templates/templatemanager.js @@ -24,7 +24,12 @@ import { } from './templateintellisense.js'; import { eventManager } from '../events/eventmanager.js'; import { redisServer } from '../database/redis.js'; -import { preprocessTemplateResources } from './templateresources.js'; +import { + clearTemplateResourceKeys, + preprocessTemplateResources, + runWithTemplateResourceSession, + TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS, +} from './templateresources.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -60,11 +65,25 @@ async function loadTemplates() { loadTemplates(); +function getNodeClass(existingClasses = '', attributes) { + var classes = existingClasses; + if (attributes?.header == 'true') { + classes += ' documentHeader'; + } + if (attributes?.footer == 'true') { + classes += ' documentFooter'; + } + return classes; +} + function getNodeStyles(attributes) { var styles = ''; if (attributes?.padding) { styles += `padding: ${attributes.padding};`; } + if (attributes?.margin) { + styles += `margin: ${attributes.margin};`; + } if (attributes?.width) { styles += `width: ${attributes.width};`; } @@ -137,38 +156,41 @@ async function transformCustomElements(content) { tree.match({ tag: 'Title1' }, (node) => ({ ...node, tag: 'h1', - attrs: { class: 'documentTitle' }, + attrs: { class: getNodeClass('documentTitle', node.attrs) }, })), (tree) => tree.match({ tag: 'Title2' }, (node) => ({ ...node, tag: 'h2', - attrs: { class: 'documentTitle' }, + attrs: { class: getNodeClass('documentTitle', node.attrs) }, })), (tree) => tree.match({ tag: 'Title3' }, (node) => ({ ...node, tag: 'h3', - attrs: { class: 'documentText' }, + attrs: { class: getNodeClass('documentText', node.attrs) }, })), (tree) => tree.match({ tag: 'Title4' }, (node) => ({ ...node, tag: 'h4', - attrs: { class: 'documentText' }, + attrs: { class: getNodeClass('documentText', node.attrs) }, })), (tree) => tree.match({ tag: 'Text' }, (node) => ({ ...node, tag: 'p', - attrs: { class: 'documentText', style: getNodeStyles(node.attrs) }, + attrs: { + class: getNodeClass('documentText', node.attrs), + style: getNodeStyles(node.attrs), + }, })), (tree) => tree.match({ tag: 'Bold' }, (node) => ({ ...node, tag: 'strong', attrs: { - class: 'documentBoldText', + class: getNodeClass('documentBoldText', node.attrs), style: getNodeStyles(node.attrs), }, })), @@ -180,7 +202,7 @@ async function transformCustomElements(content) { { tag: 'svg', attrs: { - class: 'documentBarcode', + class: getNodeClass('documentBarcode', node.attrs), 'jsbarcode-displayValue': 'false', 'jsbarcode-value': node.content.at(0) || '', 'jsbarcode-format': node.attrs.format || 'code128', @@ -190,7 +212,7 @@ async function transformCustomElements(content) { }, ], attrs: { - class: 'documentBarcodeContainer', + class: getNodeClass('documentBarcodeContainer', node.attrs), style: getNodeStyles(node.attrs), }, }; @@ -200,7 +222,7 @@ async function transformCustomElements(content) { ...node, tag: 'div', attrs: { - class: 'documentContainer', + class: getNodeClass('documentContainer', node.attrs), style: getNodeStyles(node.attrs), }, })), @@ -210,7 +232,7 @@ async function transformCustomElements(content) { ...node, tag: 'div', attrs: { - class: 'documentFlex', + class: getNodeClass('documentFlex', node.attrs), style: getNodeStyles(node.attrs), }, }; @@ -221,7 +243,7 @@ async function transformCustomElements(content) { ...node, tag: 'hr', attrs: { - class: 'documentDivider', + class: getNodeClass('documentDivider', node.attrs), style: getNodeStyles(node.attrs), }, }; @@ -239,7 +261,7 @@ async function transformCustomElements(content) { ...node, tag: 'div', attrs: { - class: 'documentProgressBar', + class: getNodeClass('documentProgressBar', node.attrs), style: getNodeStyles(node.attrs), }, content: [ @@ -260,7 +282,7 @@ async function transformCustomElements(content) { content: [dateTime.format('YYYY-MM-DD hh:mm:ss')], tag: 'span', attrs: { - class: 'documentDateTime', + class: getNodeClass('documentDateTime', node.attrs), style: getNodeStyles(node.attrs), }, }; @@ -271,7 +293,7 @@ async function transformCustomElements(content) { ...node, tag: 'table', attrs: { - class: 'documentTable', + class: getNodeClass('documentTable', node.attrs), style: getNodeStyles(node.attrs), }, }; @@ -319,7 +341,7 @@ async function transformCustomElements(content) { tag: 'tr', content, attrs: { - class: 'documentTableRowHeader', + class: getNodeClass('documentTableRowHeader', node.attrs), style: getNodeStyles(node.attrs), }, }; @@ -331,7 +353,7 @@ async function transformCustomElements(content) { tag: 'tr', content, attrs: { - class: 'documentTableRowFooter', + class: getNodeClass('documentTableRowFooter', node.attrs), style: getNodeStyles(node.attrs), }, }; @@ -342,7 +364,7 @@ async function transformCustomElements(content) { tag: 'tr', content, attrs: { - class: 'documentTableRow', + class: getNodeClass('documentTableRow', node.attrs), style: getNodeStyles(node.attrs), }, }; @@ -552,6 +574,17 @@ function getRenderObjects(resolvedData) { return [{}]; } +function flattenResolvedPreviewData(resolvedData) { + const objects = getRenderObjects(resolvedData).filter( + (item) => item != null && typeof item === 'object' && !Array.isArray(item) + ); + const first = objects[0]; + if (first == null || typeof first.toHexString === 'function') { + return resolvedData; + } + return { ...first, objects }; +} + const OBJECT_PAGE_BREAK = '
'; function buildTemplateData(documentTemplate, data = {}, fc) { @@ -814,7 +847,7 @@ export class TemplateManager { const visited = new Set(template._reference ? [String(template._reference)] : []); let templateData; const fc = this.createEmailTemplateFc(() => templateData, renderOptions, visited); - templateData = buildTemplateData(template, resolvedData, fc); + templateData = buildTemplateData(template, flattenResolvedPreviewData(resolvedData), fc); const subjectSource = typeof subject === 'string' ? subject : template.subject || ''; const contentSource = await preprocessTemplateResources( typeof content === 'string' ? content : template.content || '', @@ -951,203 +984,222 @@ export class TemplateManager { } async renderTemplate(id, content, data = {}, _scale = 1, options = {}, preview = true) { - try { - const { - padding: paddingRequested = false, - ejsOptions, - onProgress, - } = splitRenderOptions(options); - const defaultOptions = { - async: true, - ...ejsOptions, - }; - const report = preview === true ? async () => {} : onProgress || (async () => {}); - logger.debug('Rendering template:', id); + return runWithTemplateResourceSession(async () => { + const result = await (async () => { + try { + const { + padding: paddingRequested = false, + ejsOptions, + onProgress, + } = splitRenderOptions(options); + const defaultOptions = { + async: true, + ...ejsOptions, + }; + const report = preview === true ? async () => {} : onProgress || (async () => {}); + logger.debug('Rendering template:', id); - await report(0.05, 'Loading template...'); - const documentTemplate = await getObject({ - model: documentTemplateModel, - id, - populate: [ - { path: 'documentSize' }, - { path: 'parent', strictPopulate: false }, - { path: 'resources.file', strictPopulate: false }, - ], - }); - - if (documentTemplate == null || documentTemplate.error) { - return { error: documentTemplate?.error || 'Document template not found.', code: 404 }; - } - - const documentSize = documentTemplate.documentSize; - if (documentSize == null) { - return { error: 'Document template size not found.', code: 400 }; - } - - const templateContentSource = await preprocessTemplateResources( - content != null && typeof content === 'string' ? content : documentTemplate.content, - documentTemplate.resources - ); - - if (templateContentSource == null || typeof templateContentSource !== 'string') { - return { error: 'Template content is required and must be a string.', code: 400 }; - } - - const visited = new Set(); - if (documentTemplate._reference) { - visited.add(String(documentTemplate._reference)); - } - - await report(0.1, 'Loading document data...'); - const resolvedData = await this.hydratePreviewTestObject(documentTemplate, data, preview); - if (resolvedData?.error) { - return resolvedData; - } - - await report(0.16, 'Rendering template content...'); - let templateContent; - if (documentTemplate.global == true) { - let templateData = { content: contentPlaceholder }; - templateData.fc = this.createTemplateFc(() => templateData, defaultOptions, visited); - templateContent = await ejs.render(templateContentSource, templateData, defaultOptions); - } else { - const objectType = documentTemplate?.objectType; - const modelEntry = getModelByName(objectType); - const model = modelEntry?.model || modelEntry; - if (model == null || model.schema == null) { - return { error: `Unknown object type: ${objectType}`, code: 400 }; - } - const objects = getRenderObjects(resolvedData); - const renderedParts = []; - for (const object of objects) { - const objectVisited = new Set(visited); - let objectData; - const fc = this.createTemplateFc(() => objectData, defaultOptions, objectVisited); - objectData = buildTemplateData(documentTemplate, object, fc); - renderedParts.push( - await ejs.render(templateContentSource, objectData, defaultOptions) - ); - } - templateContent = renderedParts.join(OBJECT_PAGE_BREAK); - } - - var templateWithParentContent; - var parentTemplate = documentTemplate.parent; - - if (documentTemplate.parent != undefined) { - if (typeof parentTemplate === 'string') { - parentTemplate = await getObject({ + await report(0.05, 'Loading template...'); + const documentTemplate = await getObject({ model: documentTemplateModel, - id: parentTemplate, + id, populate: [ { path: 'documentSize' }, { path: 'parent', strictPopulate: false }, { path: 'resources.file', strictPopulate: false }, ], }); - } - if ( - parentTemplate?.error || - parentTemplate.content == null || - typeof parentTemplate.content !== 'string' - ) { - logger.error( - 'Parent template content is required and must be a string.', - parentTemplate?.content + + if (documentTemplate == null || documentTemplate.error) { + return { error: documentTemplate?.error || 'Document template not found.', code: 404 }; + } + + const documentSize = documentTemplate.documentSize; + if (documentSize == null) { + return { error: 'Document template size not found.', code: 400 }; + } + + const templateContentSource = await preprocessTemplateResources( + content != null && typeof content === 'string' ? content : documentTemplate.content, + documentTemplate.resources ); + + if (templateContentSource == null || typeof templateContentSource !== 'string') { + return { error: 'Template content is required and must be a string.', code: 400 }; + } + + const visited = new Set(); + if (documentTemplate._reference) { + visited.add(String(documentTemplate._reference)); + } + + await report(0.1, 'Loading document data...'); + const resolvedData = await this.hydratePreviewTestObject(documentTemplate, data, preview); + if (resolvedData?.error) { + return resolvedData; + } + + await report(0.16, 'Rendering template content...'); + let templateContent; + if (documentTemplate.global == true) { + let templateData = { content: contentPlaceholder }; + templateData.fc = this.createTemplateFc(() => templateData, defaultOptions, visited); + templateContent = await ejs.render(templateContentSource, templateData, defaultOptions); + } else { + const objectType = documentTemplate?.objectType; + const modelEntry = getModelByName(objectType); + const model = modelEntry?.model || modelEntry; + if (model == null || model.schema == null) { + return { error: `Unknown object type: ${objectType}`, code: 400 }; + } + const objects = getRenderObjects(resolvedData); + const renderedParts = []; + for (const object of objects) { + const objectVisited = new Set(visited); + let objectData; + const fc = this.createTemplateFc(() => objectData, defaultOptions, objectVisited); + objectData = buildTemplateData(documentTemplate, object, fc); + renderedParts.push( + await ejs.render(templateContentSource, objectData, defaultOptions) + ); + } + templateContent = renderedParts.join(OBJECT_PAGE_BREAK); + } + + var templateWithParentContent; + var parentTemplate = documentTemplate.parent; + + if (documentTemplate.parent != undefined) { + if (typeof parentTemplate === 'string') { + parentTemplate = await getObject({ + model: documentTemplateModel, + id: parentTemplate, + populate: [ + { path: 'documentSize' }, + { path: 'parent', strictPopulate: false }, + { path: 'resources.file', strictPopulate: false }, + ], + }); + } + if ( + parentTemplate?.error || + parentTemplate.content == null || + typeof parentTemplate.content !== 'string' + ) { + logger.error( + 'Parent template content is required and must be a string.', + parentTemplate?.content + ); + return { + error: + 'Parent template content is required and must be a string.' + + (parentTemplate?.content || ''), + code: 400, + }; + } + if (parentTemplate._reference) { + visited.add(String(parentTemplate._reference)); + } + const parentData = { content: templateContent }; + parentData.fc = this.createTemplateFc(() => parentData, defaultOptions, visited); + await report(0.22, 'Rendering parent template...'); + const parentSource = await preprocessTemplateResources( + parentTemplate.content, + parentTemplate.resources + ); + templateWithParentContent = await ejs.render(parentSource, parentData, defaultOptions); + } else { + templateWithParentContent = templateContent; + } + + if (templateWithParentContent == null || typeof templateWithParentContent !== 'string') { + return { error: 'Failed to render template content.', code: 500 }; + } + + await report(0.28, 'Transforming template elements...'); + const templateHtml = await transformCustomElements(templateWithParentContent); + + if (templateHtml == null || typeof templateHtml !== 'string') { + return { error: 'Failed to transform template content.', code: 500 }; + } + + var innerHtml = null; + + if (preview == true) { + innerHtml = await ejs.render( + previewTemplate, + { content: templateHtml }, + defaultOptions + ); + } else { + innerHtml = await ejs.render( + renderTemplateEjs, + { content: templateHtml }, + defaultOptions + ); + } + + if (innerHtml == null || typeof innerHtml !== 'string') { + return { error: 'Failed to render inner template content.', code: 500 }; + } + + const infiniteHeight = documentSize.infiniteHeight == true; + const padding = resolveDocumentPadding(documentSize, paddingRequested == true); + const pageWidth = (documentSize.width || 0) + padding.paddingLeft + padding.paddingRight; + const pageHeight = infiniteHeight + ? null + : (documentSize.height || 0) + padding.paddingTop + padding.paddingBottom; + + await report(0.32, 'Building document...'); + const baseHtml = await ejs.render( + baseTemplate, + { + content: innerHtml, + width: `${documentSize.width}mm`, + height: infiniteHeight ? 'fit-content' : `${documentSize.height}mm`, + scaledWidth: `${documentSize.width}mm`, + scaledHeight: infiniteHeight ? 'auto' : `${documentSize.height}mm`, + scaledPageWidth: `${pageWidth}mm`, + scaledPageHeight: infiniteHeight ? 'auto' : `${pageHeight}mm`, + paddingTop: `${padding.paddingTop}mm`, + paddingRight: `${padding.paddingRight}mm`, + paddingBottom: `${padding.paddingBottom}mm`, + paddingLeft: `${padding.paddingLeft}mm`, + scale: '1', + baseCSS: baseCSS, + infiniteHeight: infiniteHeight, + pageWidth: `${pageWidth}mm`, + pageHeight: infiniteHeight ? 'auto' : `${pageHeight}mm`, + previewPaginationScript: previewPaginationScript, + }, + defaultOptions + ); + return { - error: - 'Parent template content is required and must be a string.' + - (parentTemplate?.content || ''), - code: 400, + html: baseHtml, + width: pageWidth, + height: infiniteHeight ? 'auto' : pageHeight, + infiniteHeight: infiniteHeight, + printPadding: documentSize.printPadding == true, + paddingLeft: padding.paddingLeft, + paddingRight: padding.paddingRight, + paddingTop: padding.paddingTop, + paddingBottom: padding.paddingBottom, }; + } catch (error) { + console.error(error); + logger.warn('Error whilst previewing template:', error.message); + return { error: error.message, code: 500 }; } - if (parentTemplate._reference) { - visited.add(String(parentTemplate._reference)); - } - const parentData = { content: templateContent }; - parentData.fc = this.createTemplateFc(() => parentData, defaultOptions, visited); - await report(0.22, 'Rendering parent template...'); - const parentSource = await preprocessTemplateResources( - parentTemplate.content, - parentTemplate.resources - ); - templateWithParentContent = await ejs.render(parentSource, parentData, defaultOptions); - } else { - templateWithParentContent = templateContent; + })(); + + if (result?.error) { + await clearTemplateResourceKeys(); + } else if (preview == true) { + await clearTemplateResourceKeys(TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS); } - - if (templateWithParentContent == null || typeof templateWithParentContent !== 'string') { - return { error: 'Failed to render template content.', code: 500 }; - } - - await report(0.28, 'Transforming template elements...'); - const templateHtml = await transformCustomElements(templateWithParentContent); - - if (templateHtml == null || typeof templateHtml !== 'string') { - return { error: 'Failed to transform template content.', code: 500 }; - } - - var innerHtml = null; - - if (preview == true) { - innerHtml = await ejs.render(previewTemplate, { content: templateHtml }, defaultOptions); - } else { - innerHtml = await ejs.render(renderTemplateEjs, { content: templateHtml }, defaultOptions); - } - - if (innerHtml == null || typeof innerHtml !== 'string') { - return { error: 'Failed to render inner template content.', code: 500 }; - } - - const infiniteHeight = documentSize.infiniteHeight == true; - const padding = resolveDocumentPadding(documentSize, paddingRequested == true); - const pageWidth = (documentSize.width || 0) + padding.paddingLeft + padding.paddingRight; - const pageHeight = infiniteHeight - ? null - : (documentSize.height || 0) + padding.paddingTop + padding.paddingBottom; - - await report(0.32, 'Building document...'); - const baseHtml = await ejs.render( - baseTemplate, - { - content: innerHtml, - width: `${documentSize.width}mm`, - height: infiniteHeight ? 'fit-content' : `${documentSize.height}mm`, - scaledWidth: `${documentSize.width}mm`, - scaledHeight: infiniteHeight ? 'auto' : `${documentSize.height}mm`, - scaledPageWidth: `${pageWidth}mm`, - scaledPageHeight: infiniteHeight ? 'auto' : `${pageHeight}mm`, - paddingTop: `${padding.paddingTop}mm`, - paddingRight: `${padding.paddingRight}mm`, - paddingBottom: `${padding.paddingBottom}mm`, - paddingLeft: `${padding.paddingLeft}mm`, - scale: '1', - baseCSS: baseCSS, - infiniteHeight: infiniteHeight, - pageWidth: `${pageWidth}mm`, - pageHeight: infiniteHeight ? 'auto' : `${pageHeight}mm`, - previewPaginationScript: previewPaginationScript, - }, - defaultOptions - ); - - return { - html: baseHtml, - width: pageWidth, - height: infiniteHeight ? 'auto' : pageHeight, - infiniteHeight: infiniteHeight, - printPadding: documentSize.printPadding == true, - paddingLeft: padding.paddingLeft, - paddingRight: padding.paddingRight, - paddingTop: padding.paddingTop, - paddingBottom: padding.paddingBottom, - }; - } catch (error) { - console.error(error); - logger.warn('Error whilst previewing template:', error.message); - return { error: error.message, code: 500 }; - } + return result; + }); } validateTemplate(templateString) { @@ -1160,46 +1212,51 @@ export class TemplateManager { } async renderPDF(id, content, data = {}, options = {}) { - try { - logger.debug('Rendering PDF for template:', id); - const { onProgress, padding, ejsOptions, renderRequestId } = splitRenderOptions(options); - const report = typeof onProgress === 'function' ? onProgress : async () => {}; - const templateOptions = { padding, ...ejsOptions, onProgress: report }; + return runWithTemplateResourceSession(async () => { + try { + logger.debug('Rendering PDF for template:', id); + const { onProgress, padding, ejsOptions, renderRequestId } = splitRenderOptions(options); + const report = typeof onProgress === 'function' ? onProgress : async () => {}; + const templateOptions = { padding, ...ejsOptions, onProgress: report }; - const renderedTemplate = await this.renderTemplate( - id, - content, - data, - 1, - templateOptions, - false - ); + const renderedTemplate = await this.renderTemplate( + id, + content, + data, + 1, + templateOptions, + false + ); - if (renderedTemplate.error != undefined) { - return { error: renderedTemplate.error, code: renderedTemplate.code }; + if (renderedTemplate.error != undefined) { + await clearTemplateResourceKeys(); + return { error: renderedTemplate.error, code: renderedTemplate.code }; + } + const baseHtml = renderedTemplate.html; + + await report(0.35, 'Generating PDF...'); + const pdfBuffer = await generatePDF(baseHtml, { + width: renderedTemplate.width, + height: renderedTemplate.height, + infiniteHeight: renderedTemplate.infiniteHeight, + renderRequestId, + progressStart: 0.35, + progressEnd: 0.9, + }); + + await clearTemplateResourceKeys(); + return { + pdf: pdfBuffer, + width: renderedTemplate.width, + height: renderedTemplate.height, + infiniteHeight: renderedTemplate.infiniteHeight, + }; + } catch (error) { + logger.warn('Error whilst rendering PDF:', error.message); + await clearTemplateResourceKeys(); + return { error: error.message, code: 500 }; } - const baseHtml = renderedTemplate.html; - - await report(0.35, 'Generating PDF...'); - const pdfBuffer = await generatePDF(baseHtml, { - width: renderedTemplate.width, - height: renderedTemplate.height, - infiniteHeight: renderedTemplate.infiniteHeight, - renderRequestId, - progressStart: 0.35, - progressEnd: 0.9, - }); - - return { - pdf: pdfBuffer, - width: renderedTemplate.width, - height: renderedTemplate.height, - infiniteHeight: renderedTemplate.infiniteHeight, - }; - } catch (error) { - logger.warn('Error whilst rendering PDF:', error.message); - return { error: error.message, code: 500 }; - } + }); } async renderImages(id, content, data = {}, options = {}, format = 'jpeg') { @@ -1401,7 +1458,7 @@ export class TemplateManager { const fc = isEmail ? this.createEmailTemplateFc(() => templateData, renderOptions, visited) : this.createTemplateFc(() => templateData, renderOptions, visited); - templateData = buildTemplateData(template, resolvedData, fc); + templateData = buildTemplateData(template, flattenResolvedPreviewData(resolvedData), fc); const session = createCursorIntellisenseRender( source, diff --git a/src/templates/templateresources.js b/src/templates/templateresources.js index cea5ffc..ad14073 100644 --- a/src/templates/templateresources.js +++ b/src/templates/templateresources.js @@ -1,11 +1,14 @@ +import { AsyncLocalStorage } from 'async_hooks'; import { randomBytes } from 'crypto'; import config from '../config.js'; import { redisServer } from '../database/redis.js'; const RESOURCE_NAME_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; const TEMPLATE_RESOURCE_KEY_PREFIX = 'templateresources:'; -const TEMPLATE_RESOURCE_TTL_SECONDS = 3; +const TEMPLATE_RESOURCE_TTL_SECONDS = 6000; +const TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS = 10000; const RESOURCE_REFERENCE_PATTERN = /@\[([a-z0-9]+(?:-[a-z0-9]+)*)\]/g; +const templateResourceSession = new AsyncLocalStorage(); export function normalizeTemplateResourceName(value) { return String(value ?? '') @@ -43,15 +46,44 @@ export function validateTemplateResources(resources) { return true; } +export function runWithTemplateResourceSession(fn) { + const existing = templateResourceSession.getStore(); + if (existing) return fn(); + return templateResourceSession.run({ keys: [] }, fn); +} + +function takeTemplateResourceKeys() { + const store = templateResourceSession.getStore(); + if (!store?.keys?.length) return []; + return store.keys.splice(0, store.keys.length); +} + +export async function clearTemplateResourceKeys(delayMs = 0) { + const keys = takeTemplateResourceKeys(); + if (keys.length === 0) return; + + const deleteKeys = async () => { + await Promise.all(keys.map((key) => redisServer.deleteKey(key))); + }; + + if (delayMs > 0) { + const timeout = setTimeout(() => { + deleteKeys().catch(() => {}); + }, delayMs); + timeout.unref?.(); + return; + } + + await deleteKeys(); +} + export async function createTemplateResourceUrl(file) { const fileId = file?._id ?? file; if (!fileId) return null; const token = randomBytes(32).toString('hex'); - await redisServer.setKey( - `${TEMPLATE_RESOURCE_KEY_PREFIX}${token}`, - { fileId: String(fileId) }, - TEMPLATE_RESOURCE_TTL_SECONDS - ); + const key = `${TEMPLATE_RESOURCE_KEY_PREFIX}${token}`; + await redisServer.setKey(key, { fileId: String(fileId) }, TEMPLATE_RESOURCE_TTL_SECONDS); + templateResourceSession.getStore()?.keys?.push(key); return `${config.app.urlApi}/template-resources/${token}`; } @@ -208,4 +240,9 @@ export async function preprocessTemplateResources( return output + source.slice(cursor); } -export { RESOURCE_NAME_PATTERN, TEMPLATE_RESOURCE_KEY_PREFIX, TEMPLATE_RESOURCE_TTL_SECONDS }; +export { + RESOURCE_NAME_PATTERN, + TEMPLATE_RESOURCE_KEY_PREFIX, + TEMPLATE_RESOURCE_TTL_SECONDS, + TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS, +}; diff --git a/src/utils.js b/src/utils.js index 5f4dfbc..01d8bb4 100644 --- a/src/utils.js +++ b/src/utils.js @@ -2174,17 +2174,37 @@ async function getFileMeta(file) { } } +function getSchemaTypeRef(schemaType) { + if (!schemaType) return undefined; + return ( + schemaType.options?.ref || + schemaType.caster?.options?.ref || + schemaType.embeddedSchemaType?.options?.ref + ); +} + +function forEachSchemaPath(schema, callback) { + if (!schema || typeof callback !== 'function') return; + + schema.eachPath((pathName, schemaType) => { + callback(pathName, schemaType); + }); + + const subpaths = schema.subpaths || {}; + for (const [pathName, schemaType] of Object.entries(subpaths)) { + if (pathName.endsWith('.$')) continue; + callback(pathName, schemaType); + } +} + function modelHasRef(model, refName) { if (!model || !model.schema) { return false; } let hasRef = false; - model.schema.eachPath((pathName, schemaType) => { - const directRef = schemaType?.options?.ref; - const arrayRef = schemaType?.caster?.options?.ref; - const ref = directRef || arrayRef; - if (ref === refName) { + forEachSchemaPath(model.schema, (_pathName, schemaType) => { + if (getSchemaTypeRef(schemaType) === refName) { hasRef = true; } }); @@ -2198,18 +2218,39 @@ function getFieldsByRef(model, refName) { } const fields = []; - model.schema.eachPath((pathName, schemaType) => { - const directRef = schemaType?.options?.ref; - const arrayRef = schemaType?.caster?.options?.ref; - const ref = directRef || arrayRef; - if (ref === refName) { - fields.push(pathName); - } + const seen = new Set(); + forEachSchemaPath(model.schema, (pathName, schemaType) => { + if (getSchemaTypeRef(schemaType) !== refName || seen.has(pathName)) return; + seen.add(pathName); + fields.push(pathName); }); return fields; } +function collectIdsAtPath(object, pathName) { + if (object == null || !pathName) return []; + + const parts = String(pathName).split('.').filter(Boolean); + let nodes = [object]; + for (const part of parts) { + nodes = nodes.flatMap((node) => { + if (node == null || typeof node !== 'object') return []; + const next = node[part]; + if (next == null || next === '') return []; + return Array.isArray(next) ? next : [next]; + }); + } + + return nodes + .map((value) => { + if (value == null || value === '') return null; + if (typeof value === 'object') return value._id != null ? String(value._id) : null; + return String(value); + }) + .filter(Boolean); +} + // Build a nested populate specification by walking the schema graph, // instead of recursing over already-populated documents. function buildDeepPopulateSpec(object, model, populated = new Set()) { @@ -2221,9 +2262,7 @@ function buildDeepPopulateSpec(object, model, populated = new Set()) { const populateSpec = []; schema.eachPath((pathname, schemaType) => { - const directRef = schemaType.options?.ref; - const arrayRef = schemaType.caster?.options?.ref; - const ref = directRef || arrayRef; + const ref = getSchemaTypeRef(schemaType); if (!ref) return; const refName = typeof ref === 'function' ? ref.call(object) : ref; @@ -2305,6 +2344,7 @@ export { getFileMeta, modelHasRef, getFieldsByRef, + collectIdsAtPath, jsonToCacheKey, subscribeAuditLog, unsubscribeAuditLog,