Implement schema utility functions and enhance email message handling

This commit introduces several utility functions in the `utils.js` file, including `getSchemaTypeRef`, `forEachSchemaPath`, and `collectIdsAtPath`, to streamline schema reference handling. Additionally, it refactors the `modelHasRef` and `getFieldsByRef` functions to utilize these new utilities, improving code readability and maintainability. The email message schema is updated to support multiple recipients with enhanced attachment handling, ensuring better management of email content and references. Tests are added to validate the new functionality, reinforcing the integrity of email message processing.

src/__tests__/utils.getFieldsByRef.test.js

@@ -0,0 +1,42 @@
+import { describe, expect, it } from '@jest/globals';
+import mongoose from 'mongoose';
+import {
+  collectIdsAtPath,
+  getFieldsByRef,
+  modelHasRef,
+} from '../utils.js';
+import { emailMessageModel } from '../database/schemas/management/emailmessage.schema.js';
+
+describe('getFieldsByRef', () => {
+  it('finds nested array file refs on email message recipients', () => {
+    expect(modelHasRef(emailMessageModel, 'file')).toBe(true);
+    expect(getFieldsByRef(emailMessageModel, 'file')).toEqual(['recipients.attachments']);
+  });
+
+  it('finds top-level array refs using embeddedSchemaType', () => {
+    const schema = new mongoose.Schema({
+      files: [{ type: mongoose.Schema.Types.ObjectId, ref: 'file' }],
+    });
+    const model = { schema };
+
+    expect(modelHasRef(model, 'file')).toBe(true);
+    expect(getFieldsByRef(model, 'file')).toEqual(['files']);
+  });
+});
+
+describe('collectIdsAtPath', () => {
+  it('collects file ids from nested recipient attachment lists', () => {
+    const object = {
+      recipients: [
+        { attachments: [{ _id: 'file-1' }, 'file-2'] },
+        { attachments: ['file-3'] },
+      ],
+    };
+
+    expect(collectIdsAtPath(object, 'recipients.attachments')).toEqual([
+      'file-1',
+      'file-2',
+      'file-3',
+    ]);
+  });
+});

src/database/__tests__/database.test.js

@@ -32,6 +32,7 @@ jest.unstable_mockModule('../../utils.js', () => ({
     return result;
   }),
   getFieldsByRef: jest.fn(() => []),
+  collectIdsAtPath: jest.fn(() => []),
   getQueryToCacheKey: jest.fn(({ model, id }) => `${model}:${id}`),
   modelHasRef: jest.fn(() => false),
   newAuditLog: jest.fn(),

src/database/database.js

@@ -13,6 +13,7 @@ import {
   expandObjectIds,
   modelHasRef,
   getFieldsByRef,
+  collectIdsAtPath,
   getQueryToCacheKey,
   editAuditLog,
   distributeUpdate,
@@ -1128,16 +1129,7 @@ export const listObjectDependencies = async ({ model, id }) => {
       const targetModel = entry?.model;
       if (!targetModel || !targetModel.schema) continue;
 
-      const referencingPaths = [];
+      const referencingPaths = getFieldsByRef(targetModel, parentModelName);
-
-      targetModel.schema.eachPath((pathName, schemaType) => {
-        const directRef = schemaType?.options?.ref;
-        const arrayRef = schemaType?.caster?.options?.ref;
-        const refName = directRef || arrayRef;
-        if (refName === parentModelName) {
-          referencingPaths.push(pathName);
-        }
-      });
 
       if (referencingPaths.length === 0) continue;
 
@@ -1205,37 +1197,30 @@ export const editObject = async ({ model, id, updateData, user, populate, recalc
 
     const previousExpandedObject = expandObjectIds(previousObject);
 
-    // Check if any model parameters have ref: 'file' and flush files if so
-    if (modelHasRef(model, 'file')) {
-      logger.debug(`Model ${model.modelName} has file references, checking for files to flush`);
-      const fileFields = getFieldsByRef(model, 'file');
-
-      for (const fieldName of fileFields) {
-        const fieldValue = previousExpandedObject[fieldName];
-
-        if (fieldValue) {
-          if (Array.isArray(fieldValue)) {
-            // Handle file arrays
-            for (const fileRef of fieldValue) {
-              if (fileRef && fileRef._id) {
-                logger.debug(`Flushing file from array field ${fieldName}: ${fileRef._id}`);
-                await flushFile({ id: fileRef._id, user });
-              }
-            }
-          } else if (fieldValue._id) {
-            // Handle single file reference
-            logger.debug(`Flushing file from field ${fieldName}: ${fieldValue._id}`);
-            await flushFile({ id: fieldValue._id, user });
-          }
-        }
-      }
-    }
-
     const updatedObject = mergeObjectUpdates(
       _.cloneDeep(previousExpandedObject),
       expandObjectIds(updateData)
     );
 
+    // Flush files that this update actually dropped, not files still referenced
+    // on nested paths such as recipients.attachments.
+    if (modelHasRef(model, 'file')) {
+      logger.debug(`Model ${model.modelName} has file references, checking for files to flush`);
+      const fileFields = getFieldsByRef(model, 'file');
+      const nextFileIds = new Set(
+        fileFields.flatMap((fieldName) => collectIdsAtPath(updatedObject, fieldName))
+      );
+
+      for (const fieldName of fileFields) {
+        const previousIds = collectIdsAtPath(previousExpandedObject, fieldName);
+        for (const fileId of previousIds) {
+          if (nextFileIds.has(fileId)) continue;
+          logger.debug(`Flushing removed file from field ${fieldName}: ${fileId}`);
+          await flushFile({ id: fileId, user });
+        }
+      }
+    }
+
     // Audit log before update
     await editAuditLog(previousExpandedObject, updatedObject, id, parentType, user);
 

src/database/schemas/management/emailmessage.schema.js

@@ -3,6 +3,32 @@ import { generateId } from '../../utils.js';
 
 const { Schema } = mongoose;
 
+const emailRecipientSchema = new Schema(
+  {
+    recipientEmail: { type: String, required: true },
+    recipientType: { type: String, enum: ['client', 'vendor'], required: false },
+    recipient: {
+      type: Schema.Types.ObjectId,
+      refPath: 'recipients.recipientType',
+      required: false,
+    },
+    objectType: { type: String, required: false },
+    object: {
+      type: Schema.Types.ObjectId,
+      refPath: 'recipients.objectType',
+      required: false,
+    },
+    pixelId: { type: String, required: true },
+    messageId: { type: String, required: false },
+    attachments: [{ type: Schema.Types.ObjectId, ref: 'file', required: false }],
+    read: { type: Boolean, required: true, default: false },
+    sent: { type: Boolean, required: true, default: false },
+    sentAt: { type: Date, required: false },
+    readAt: { type: Date, required: false },
+  },
+  { _id: true }
+);
+
 const emailMessageSchema = new Schema(
   {
     _reference: { type: String, default: () => generateId()() },
@@ -20,36 +46,29 @@ const emailMessageSchema = new Schema(
       ref: 'emailAccount',
       required: true,
     },
-    recipientEmail: { type: String, required: true },
+    recipients: { type: [emailRecipientSchema], required: true, default: [] },
-    recipientType: { type: String, enum: ['client', 'vendor'], required: false },
-    recipient: {
-      type: Schema.Types.ObjectId,
-      refPath: 'recipientType',
-      required: false,
-    },
     fromEmail: { type: String, required: true, immutable: true },
-    messageId: { type: String, required: false },
     subject: { type: String, required: false, default: '' },
     content: { type: String, required: false, default: '' },
-    attachments: [{ type: Schema.Types.ObjectId, ref: 'file', required: false }],
     state: {
       type: { type: String, required: true, default: 'queued' },
       progress: { type: Number, required: false, default: 0 },
       message: { type: String, required: false },
     },
-    read: { type: Boolean, required: true, default: false },
     sentAt: { type: Date, required: false },
-    readAt: { type: Date, required: false },
+    firstReadAt: { type: Date, required: false },
+    lastReadAt: { type: Date, required: false },
   },
   { timestamps: true, suppressReservedKeysWarning: true }
 );
 
 emailMessageSchema.index({
   name: 'text',
-  recipientEmail: 'text',
   fromEmail: 'text',
   objectType: 'text',
+  'recipients.recipientEmail': 'text',
 });
+emailMessageSchema.index({ 'recipients.pixelId': 1 }, { unique: true, sparse: true });
 emailMessageSchema.virtual('id').get(function () {
   return this._id;
 });

src/routes/management/emailmessages.js

@@ -22,28 +22,22 @@ const filters = [
   'emailAccount',
   'objectType',
   'objects',
-  'recipientEmail',
-  'recipientType',
-  'recipient',
   'fromEmail',
-  'messageId',
   'state',
-  'read',
   'sentAt',
-  'readAt',
+  'firstReadAt',
+  'lastReadAt',
   'createdAt',
   'updatedAt',
   '_reference',
 ];
 const sorters = [
   'name',
-  'recipientEmail',
   'fromEmail',
-  'messageId',
   'state',
-  'read',
   'sentAt',
-  'readAt',
+  'firstReadAt',
+  'lastReadAt',
   'createdAt',
   'updatedAt',
 ];
@@ -51,9 +45,7 @@ const propertyFilters = [
   'emailTemplate',
   'emailAccount',
   'objectType',
-  'recipientType',
   'state',
-  'read',
 ];
 
 router.get('/', isAuthenticated, checkPermissions('emailMessage', 'list'), async (req, res) => {

src/services/management/__tests__/emailmessages.test.js

@@ -14,6 +14,7 @@ jest.unstable_mockModule('../../../database/schemas/management/emailmessage.sche
   emailMessageModel: {
     modelName: 'emailMessage',
     findById: jest.fn(),
+    findOne: jest.fn(),
     findByIdAndUpdate: jest.fn(),
   },
 }));
@@ -175,7 +176,15 @@ describe('email message creation', () => {
       expect.objectContaining({
         properties: ['state', 'emailAccount'],
         filter: { read: false },
-        populate: ['emailTemplate', 'emailAccount', 'objects', 'object', 'recipient', 'attachments'],
+        populate: [
+          'emailTemplate',
+          'emailAccount',
+          'objects',
+          'object',
+          'recipients.recipient',
+          'recipients.object',
+          'recipients.attachments',
+        ],
       })
     );
   });
@@ -187,7 +196,7 @@ describe('email message creation', () => {
         _id: 'message-1',
         emailTemplate: 'template-1',
         object: 'object-1',
-        recipientEmail: 'client@example.com',
+        recipients: [{ recipientEmail: 'client@example.com' }],
       }),
     });
     templateManager.renderEmailTemplate.mockResolvedValue({
@@ -208,13 +217,101 @@ describe('email message creation', () => {
         user,
         updateData: expect.objectContaining({
           state: { type: 'sent', progress: 1 },
-          messageId: '<abc@smtp>',
           sentAt: expect.any(Date),
+          recipients: [
+            expect.objectContaining({
+              recipientEmail: 'client@example.com',
+              sent: true,
+              messageId: '<abc@smtp>',
+            }),
+          ],
         }),
       })
     );
   });
 
+  it('sends recipients one at a time with a 500ms gap and live progress', async () => {
+    const user = { _id: 'user-1' };
+    const originalSetTimeout = global.setTimeout;
+    const delays = [];
+    jest.spyOn(global, 'setTimeout').mockImplementation((fn, ms, ...args) => {
+      if (ms === 500) {
+        delays.push(ms);
+        fn();
+        return 0;
+      }
+      return originalSetTimeout(fn, ms, ...args);
+    });
+    emailMessageModel.findById.mockReturnValue({
+      lean: jest.fn().mockResolvedValue({
+        _id: 'message-1',
+        emailTemplate: 'template-1',
+        objectType: 'invoice',
+        recipients: [
+          { recipientEmail: 'one@example.com', pixelId: 'pixel-1' },
+          { recipientEmail: 'two@example.com', pixelId: 'pixel-2' },
+        ],
+      }),
+    });
+    templateManager.renderEmailTemplate.mockResolvedValue({
+      subject: 'Invoice',
+      html: '<p>Hi</p>',
+      htmlWithPixel: '<p>Hi</p><img>',
+    });
+    editObject.mockResolvedValue({ _id: 'message-1' });
+
+    try {
+      await processEmailMessage(
+        'message-1',
+        { fromEmail: 'farm@example.com', host: 'smtp.example.com' },
+        user
+      );
+
+      expect(templateManager.renderEmailTemplate).toHaveBeenCalledTimes(2);
+      expect(workerMessages).toHaveLength(2);
+      expect(workerMessages.map((payload) => payload.recipientEmail)).toEqual([
+        'one@example.com',
+        'two@example.com',
+      ]);
+      expect(delays).toEqual([500]);
+      expect(editObject).toHaveBeenCalledWith(
+        expect.objectContaining({
+          updateData: expect.objectContaining({
+            state: {
+              type: 'sending',
+              progress: 0.25,
+              message: 'Sending email 1 of 2...',
+            },
+          }),
+        })
+      );
+      expect(editObject).toHaveBeenCalledWith(
+        expect.objectContaining({
+          updateData: expect.objectContaining({
+            state: {
+              type: 'sending',
+              progress: 0.75,
+              message: 'Sending email 2 of 2...',
+            },
+          }),
+        })
+      );
+      expect(editObject).toHaveBeenCalledWith(
+        expect.objectContaining({
+          updateData: expect.objectContaining({
+            state: { type: 'sent', progress: 1 },
+            recipients: [
+              expect.objectContaining({ recipientEmail: 'one@example.com', sent: true }),
+              expect.objectContaining({ recipientEmail: 'two@example.com', sent: true }),
+            ],
+          }),
+        })
+      );
+    } finally {
+      global.setTimeout.mockRestore();
+    }
+  });
+
   it('renders document template attachments into files before sending', async () => {
     const user = { _id: 'user-1' };
     const pdfBytes = Buffer.from('pdf-bytes');
@@ -225,7 +322,7 @@ describe('email message creation', () => {
         emailTemplate: 'template-1',
         objectType: 'invoice',
         object: 'object-1',
-        recipientEmail: 'client@example.com',
+        recipients: [{ recipientEmail: 'client@example.com' }],
       }),
     });
     templateLean.mockResolvedValue({
@@ -291,8 +388,14 @@ describe('email message creation', () => {
     expect(editObject).toHaveBeenCalledWith(
       expect.objectContaining({
         updateData: expect.objectContaining({
-          state: { type: 'sending', progress: 0.7, message: 'Sending email...' },
+          state: { type: 'sent', progress: 1 },
+          recipients: [
+            expect.objectContaining({
+              recipientEmail: 'client@example.com',
               attachments: ['file-new'],
+              messageId: '<abc@smtp>',
+            }),
+          ],
         }),
       })
     );
@@ -313,7 +416,7 @@ describe('email message creation', () => {
         _id: 'message-1',
         emailTemplate: 'template-1',
         object: 'object-1',
-        recipientEmail: 'client@example.com',
+        recipients: [{ recipientEmail: 'client@example.com' }],
       }),
     });
     templateLean.mockResolvedValue({
@@ -350,8 +453,13 @@ describe('email message creation', () => {
     expect(editObject).toHaveBeenCalledWith(
       expect.objectContaining({
         updateData: expect.objectContaining({
+          recipients: [
+            expect.objectContaining({
+              recipientEmail: 'client@example.com',
               attachments: ['file-1'],
             }),
+          ],
+        }),
       })
     );
     expect(workerMessages[0].attachments).toEqual([
@@ -364,8 +472,20 @@ describe('email message creation', () => {
   });
 
   it('uses a system audit actor for automatic read updates', async () => {
-    req.params = { id: '507f1f77bcf86cd799439011' };
+    req.params = { id: 'pixel-recipient-1' };
-    editObject.mockResolvedValue({ _id: '507f1f77bcf86cd799439011', read: true });
+    emailMessageModel.findOne.mockReturnValue({
+      lean: jest.fn().mockResolvedValue({
+        _id: '507f1f77bcf86cd799439011',
+        recipients: [
+          {
+            recipientEmail: 'client@example.com',
+            pixelId: 'pixel-recipient-1',
+            read: false,
+          },
+        ],
+      }),
+    });
+    editObject.mockResolvedValue({ _id: '507f1f77bcf86cd799439011' });
 
     await markEmailMessageAsReadRouteHandler(req, res);
 
@@ -373,9 +493,16 @@ describe('email message creation', () => {
       expect.objectContaining({
         id: expect.anything(),
         updateData: expect.objectContaining({
+          firstReadAt: expect.any(Date),
+          lastReadAt: expect.any(Date),
+          state: { type: 'read' },
+          recipients: [
+            expect.objectContaining({
+              recipientEmail: 'client@example.com',
               read: true,
               readAt: expect.any(Date),
-          state: { type: 'read' },
+            }),
+          ],
         }),
         user: 'system',
       })

src/services/management/emailmessages.js

@@ -6,6 +6,7 @@ import { fileModel } from '../../database/schemas/management/file.schema.js';
 import { userNotifierModel } from '../../database/schemas/misc/usernotifier.schema.js';
 import log4js from 'log4js';
 import mongoose from 'mongoose';
+import { nanoid } from 'nanoid';
 import { Worker } from 'worker_threads';
 import { randomUUID } from 'crypto';
 import { fileURLToPath } from 'url';
@@ -37,7 +38,77 @@ import {
 const logger = log4js.getLogger('Email Messages');
 logger.level = config.server.logLevel;
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const populate = ['emailTemplate', 'emailAccount', 'objects', 'object', 'recipient', 'attachments'];
+const populate = [
+  'emailTemplate',
+  'emailAccount',
+  'objects',
+  'object',
+  'recipients.recipient',
+  'recipients.object',
+  'recipients.attachments',
+];
+
+const PIXEL_ID_LENGTH = 48;
+const SEND_GAP_MS = 500;
+
+const roundProgress = (value) => Math.min(1, Math.round(value * 1000) / 1000);
+
+const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const lastDate = (values) => {
+  const times = values
+    .filter((value) => value != null)
+    .map((value) => new Date(value).getTime())
+    .filter((value) => !Number.isNaN(value));
+  if (!times.length) return undefined;
+  return new Date(Math.max(...times));
+};
+
+const firstDate = (values) => {
+  const times = values
+    .filter((value) => value != null)
+    .map((value) => new Date(value).getTime())
+    .filter((value) => !Number.isNaN(value));
+  if (!times.length) return undefined;
+  return new Date(Math.min(...times));
+};
+
+const normalizeRecipients = (body = {}) => {
+  const source = Array.isArray(body.recipients) ? body.recipients : [];
+  const recipients = source
+    .map((recipient) => ({
+      recipientEmail: recipient?.recipientEmail,
+      recipientType: recipient?.recipientType,
+      recipient: recipient?.recipient?._id ?? recipient?.recipient,
+      objectType: recipient?.objectType || body.objectType,
+      object: recipient?.object?._id ?? recipient?.object,
+      pixelId: nanoid(PIXEL_ID_LENGTH),
+      attachments: [],
+      read: false,
+      sent: false,
+    }))
+    .filter((recipient) => recipient.recipientEmail);
+
+  if (recipients.length) return recipients;
+
+  if (body.recipientEmail) {
+    return [
+      {
+        recipientEmail: body.recipientEmail,
+        recipientType: body.recipientType,
+        recipient: body.recipient?._id ?? body.recipient,
+        objectType: body.objectType,
+        object: body.object?._id ?? body.object,
+        pixelId: nanoid(PIXEL_ID_LENGTH),
+        attachments: [],
+        read: false,
+        sent: false,
+      },
+    ];
+  }
+
+  return [];
+};
 let worker;
 const pending = new Map();
 
@@ -155,7 +226,30 @@ async function loadFileRecord(file) {
   return record;
 }
 
-async function resolveEmailAttachments(emailMessage, user) {
+function recipientObjectRefs(emailMessage, recipient) {
+  if (recipient?.object != null) return [recipient.object];
+  return relatedObjectRefs(emailMessage);
+}
+
+async function loadRecipientObjects(emailMessage, recipient) {
+  const refs = recipientObjectRefs(emailMessage, recipient);
+  if (!refs.length) return [];
+  const modelEntry = getModelByName(emailMessage.objectType);
+  if (!modelEntry?.model) {
+    return [];
+  }
+  const objects = [];
+  for (const ref of refs) {
+    const object = await getObject({
+      model: modelEntry.model,
+      id: ref?._id || ref,
+    });
+    objects.push(object);
+  }
+  return objects;
+}
+
+async function resolveEmailAttachments(emailMessage, recipient, user) {
   const template = await emailTemplateModel
     .findById(emailMessage.emailTemplate)
     .populate({ path: 'attachments.file', strictPopulate: false })
@@ -163,24 +257,16 @@ async function resolveEmailAttachments(emailMessage, user) {
   if (!template) throw new Error('Email template not found.');
 
   const mailAttachments = [];
-  const fileIds = [];
+  const attachments = [];
 
   for (const attachment of template.attachments || []) {
     if (attachment.type === 'documentTemplate') {
       const templateId = attachment.file?._id ?? attachment.file;
       if (!templateId) throw new Error('Document template attachment is missing a file.');
-      const modelEntry = getModelByName(emailMessage.objectType);
+      if (!getModelByName(emailMessage.objectType)?.model) {
-      if (!modelEntry?.model) {
         throw new Error(`Unknown object type for email attachment: ${emailMessage.objectType}`);
       }
-      const objects = [];
+      const objects = await loadRecipientObjects(emailMessage, recipient);
-      for (const ref of relatedObjectRefs(emailMessage)) {
-        const object = await getObject({
-          model: modelEntry.model,
-          id: ref?._id || ref,
-        });
-        objects.push(object);
-      }
       const rendered = await templateManager.renderDownload(
         templateId,
         undefined,
@@ -209,10 +295,11 @@ async function resolveEmailAttachments(emailMessage, user) {
           metaData: {
             source: 'emailTemplateAttachment',
             emailMessage: String(emailMessage._id),
+            pixelId: recipient?.pixelId,
             documentTemplate: String(templateId),
           },
         });
-        fileIds.push(created._id);
+        attachments.push(created._id);
         mailAttachments.push({
           filename: names.filename,
           content: buffer,
@@ -228,7 +315,7 @@ async function resolveEmailAttachments(emailMessage, user) {
       const body = await downloadFile(BUCKETS.FILES, `files/${fileRecord._id}${extension}`);
       const buffer = await streamToBuffer(body);
       const names = splitAttachmentName(attachment.fileName || fileRecord.name, extension, 0, 1);
-      fileIds.push(fileRecord._id);
+      attachments.push(fileRecord._id);
       mailAttachments.push({
         filename: names.filename,
         content: buffer,
@@ -237,7 +324,7 @@ async function resolveEmailAttachments(emailMessage, user) {
     }
   }
 
-  return { fileIds, mailAttachments };
+  return { attachments, mailAttachments };
 }
 
 const updateEmailMessage = async (id, updateData, user) => {
@@ -263,40 +350,64 @@ const report = async (id, success, message) => {
 
 export const processEmailMessage = async (id, account, user) => {
   try {
-    await updateEmailMessage(
-      id,
-      { state: { type: 'rendering', progress: 0.2, message: 'Rendering email...' } },
-      user
-    );
     const emailMessage = await emailMessageModel.findById(id).lean();
     if (!emailMessage) throw new Error('Email message not found.');
 
+    const recipients = Array.isArray(emailMessage.recipients) ? [...emailMessage.recipients] : [];
+    if (!recipients.length) throw new Error('Email message has no recipients.');
+
+    const total = recipients.length;
+    let subject = emailMessage.subject;
+    let content = emailMessage.content;
+
+    for (let index = 0; index < total; index += 1) {
+      const recipient = recipients[index];
+      const suffix = total > 1 ? ` ${index + 1} of ${total}` : '';
+      await updateEmailMessage(
+        id,
+        {
+          state: {
+            type: 'rendering',
+            progress: roundProgress(index / total),
+            message: `Rendering email${suffix}...`,
+          },
+        },
+        user
+      );
+
+      const objects = await loadRecipientObjects(emailMessage, recipient);
       const rendered = await templateManager.renderEmailTemplate(
         emailMessage.emailTemplate,
         undefined,
         undefined,
-      templateDataFromObjects(relatedObjectRefs(emailMessage)),
+        templateDataFromObjects(objects),
         {},
         false,
-      String(id)
+        recipient.pixelId
       );
       if (rendered?.error) throw new Error(rendered.error);
+      subject = rendered.subject;
+      content = rendered.html;
 
-    const { fileIds, mailAttachments } = await resolveEmailAttachments(emailMessage, user);
+      const { attachments, mailAttachments } = await resolveEmailAttachments(
-
+        emailMessage,
+        recipient,
+        user
+      );
       await updateEmailMessage(
         id,
         {
-        state: { type: 'sending', progress: 0.7, message: 'Sending email...' },
+          state: {
-        subject: rendered.subject,
+            type: 'sending',
-        content: rendered.html,
+            progress: roundProgress((index + 0.5) / total),
-        attachments: fileIds,
+            message: `Sending email${suffix}...`,
+          },
         },
         user
       );
       const result = await deliver({
         kind: 'emailMessage',
-      recipientEmail: emailMessage.recipientEmail,
+        recipientEmail: recipient.recipientEmail,
         from: account.fromName ? `${account.fromName} <${account.fromEmail}>` : account.fromEmail,
         subject: rendered.subject,
         html: rendered.htmlWithPixel,
@@ -309,12 +420,42 @@ export const processEmailMessage = async (id, account, user) => {
           password: account.password,
         },
       });
+      const sentAt = new Date();
+      recipients[index] = {
+        ...recipient,
+        sent: true,
+        sentAt,
+        messageId: result.messageId || undefined,
+        attachments,
+      };
+      await updateEmailMessage(
+        id,
+        {
+          state: {
+            type: 'sending',
+            progress: roundProgress((index + 1) / total),
+            message:
+              total > 1 ? `Sent ${index + 1} of ${total}` : 'Sending email...',
+          },
+          subject,
+          content,
+          recipients: recipients.map((entry) => ({ ...entry })),
+          sentAt: lastDate(recipients.map((entry) => entry.sentAt)),
+        },
+        user
+      );
+      if (index < total - 1) {
+        await wait(SEND_GAP_MS);
+      }
+    }
     await updateEmailMessage(
       id,
       {
         state: { type: 'sent', progress: 1 },
-        messageId: result.messageId || undefined,
+        subject,
-        sentAt: new Date(),
+        content,
+        recipients: recipients.map((entry) => ({ ...entry })),
+        sentAt: lastDate(recipients.map((recipient) => recipient.sentAt)),
       },
       user
     );
@@ -445,6 +586,11 @@ export const newEmailMessageRouteHandler = async (req, res) => {
     });
   }
 
+  const recipients = normalizeRecipients(req.body);
+  if (!recipients.length) {
+    return res.status(400).send({ error: 'At least one recipient is required.', code: 400 });
+  }
+
   const newData = {
     createdAt: new Date(),
     updatedAt: new Date(),
@@ -453,12 +599,9 @@ export const newEmailMessageRouteHandler = async (req, res) => {
     objectType: req.body.objectType,
     objects: relatedObjectIdsFromBody(req.body),
     emailAccount: account._id,
-    recipientEmail: req.body.recipientEmail,
+    recipients,
-    recipientType: req.body.recipientType,
-    recipient: req.body.recipient?._id ?? req.body.recipient,
     fromEmail: account.fromEmail,
     state: { type: 'queued', progress: 0, message: 'Queued' },
-    read: false,
   };
 
   const result = await newObject({
@@ -483,7 +626,11 @@ export const newEmailMessageRouteHandler = async (req, res) => {
   logger.debug(`New email message with ID: ${result._id}`);
 
   res.send(withLegacyObjects(result));
-  setImmediate(() => processEmailMessage(result._id, account, req.user));
+  setImmediate(() => {
+    processEmailMessage(result._id, account, req.user).catch((error) => {
+      logger.error(`Email message ${result._id} failed:`, error);
+    });
+  });
 };
 
 export const getEmailMessageStatsRouteHandler = async (req, res) => {
@@ -541,19 +688,36 @@ export const getEmailMessageNeighborsRouteHandler = async (
 };
 
 export const markEmailMessageAsReadRouteHandler = async (req, res) => {
-  const id = new mongoose.Types.ObjectId(req.params.id);
+  const pixelId = req.params.id;
 
-  logger.trace(`Email message with ID: ${id}`);
+  logger.trace(`Email message pixel: ${pixelId}`);
 
-  const updateData = {
+  const emailMessage = await emailMessageModel.findOne({ 'recipients.pixelId': pixelId }).lean();
-    updatedAt: new Date(),
+  if (!emailMessage) {
+    return res.status(404).send({ error: 'Email message not found.', code: 404 });
+  }
+
+  const now = new Date();
+  const recipients = (Array.isArray(emailMessage.recipients) ? emailMessage.recipients : []).map(
+    (recipient) =>
+      recipient?.pixelId === pixelId
+        ? {
+            ...recipient,
             read: true,
-    readAt: new Date(),
+            readAt: recipient?.readAt || now,
+          }
+        : recipient
+  );
+  const updateData = {
+    updatedAt: now,
+    recipients,
+    firstReadAt: emailMessage.firstReadAt || firstDate(recipients.map((recipient) => recipient.readAt)) || now,
+    lastReadAt: now,
     state: { type: 'read' },
   };
   const result = await editObject({
     model: emailMessageModel,
-    id,
+    id: emailMessage._id,
     updateData,
     user: 'system',
     populate,
@@ -565,7 +729,7 @@ export const markEmailMessageAsReadRouteHandler = async (req, res) => {
     return;
   }
 
-  logger.debug(`Marked email message as read with ID: ${id}`);
+  logger.debug(`Marked email message as read with pixel: ${pixelId}`);
 
   res.send(withLegacyObjects(result));
 };

src/services/misc/templateresources.js

@@ -15,7 +15,7 @@ export async function getTemplateResourceRouteHandler(req, res) {
   if (!TOKEN_PATTERN.test(token || '')) return notFound(res);
 
   try {
-    const payload = await redisServer.getAndDeleteKey(`${TEMPLATE_RESOURCE_KEY_PREFIX}${token}`);
+    const payload = await redisServer.getKey(`${TEMPLATE_RESOURCE_KEY_PREFIX}${token}`);
     if (!payload?.fileId || !mongoose.Types.ObjectId.isValid(payload.fileId)) {
       return notFound(res);
     }

src/templates/__tests__/templatemanager.test.js

@@ -73,6 +73,7 @@ jest.unstable_mockModule('../../database/redis.js', () => ({
   redisServer: {
     setKey: jest.fn().mockResolvedValue(undefined),
     getKey: jest.fn().mockResolvedValue(null),
+    deleteKey: jest.fn().mockResolvedValue(undefined),
   },
 }));
 
@@ -123,6 +124,7 @@ const ejs = (await import('ejs')).default;
 const { generatePDF } = await import('../pdffactory.js');
 const { eventManager } = await import('../../events/eventmanager.js');
 const { redisServer } = await import('../../database/redis.js');
+const { TEMPLATE_RESOURCE_TTL_SECONDS } = await import('../templateresources.js');
 
 describe('stripDocumentTemplateReference', () => {
   it('strips a DTP: prefix', () => {
@@ -248,6 +250,8 @@ describe('TemplateManager', () => {
 
   describe('renderTemplate', () => {
     it('replaces each valid resource occurrence before rendering', async () => {
+      jest.useFakeTimers();
+      try {
         getObject.mockResolvedValue({
           documentSize: { width: 100, height: 100, infiniteHeight: false },
           global: true,
@@ -267,8 +271,15 @@ describe('TemplateManager', () => {
         expect(redisServer.setKey).toHaveBeenCalledWith(
           expect.stringMatching(/^templateresources:[a-f0-9]{64}$/),
           { fileId: 'file-1' },
-        3
+          TEMPLATE_RESOURCE_TTL_SECONDS
         );
+        expect(redisServer.deleteKey).not.toHaveBeenCalled();
+
+        await jest.advanceTimersByTimeAsync(10000);
+        expect(redisServer.deleteKey).toHaveBeenCalledTimes(2);
+      } finally {
+        jest.useRealTimers();
+      }
     });
 
     it('resolves parent resources from the parent owner only', async () => {
@@ -290,12 +301,12 @@ describe('TemplateManager', () => {
       expect(redisServer.setKey).toHaveBeenCalledWith(
         expect.any(String),
         { fileId: 'child-file' },
-        3
+        TEMPLATE_RESOURCE_TTL_SECONDS
       );
       expect(redisServer.setKey).toHaveBeenCalledWith(
         expect.any(String),
         { fileId: 'parent-file' },
-        3
+        TEMPLATE_RESOURCE_TTL_SECONDS
       );
     });
 
@@ -789,6 +800,22 @@ describe('TemplateManager', () => {
       expect(result.pdf).toBeDefined();
     });
 
+    it('clears resource keys immediately after rendering a PDF', async () => {
+      getObject.mockResolvedValue({
+        documentSize: { width: 100, height: 100, infiniteHeight: false },
+        global: true,
+        resources: [{ name: 'logo', file: { _id: 'file-1' } }],
+      });
+
+      await templateManager.renderPDF('temp-id', '<Image src="@[logo]" />');
+
+      expect(generatePDF).toHaveBeenCalled();
+      expect(redisServer.deleteKey).toHaveBeenCalledTimes(1);
+      expect(redisServer.deleteKey).toHaveBeenCalledWith(
+        expect.stringMatching(/^templateresources:[a-f0-9]{64}$/)
+      );
+    });
+
     it('adds padding to page size when padding is requested', async () => {
       getObject.mockResolvedValue({
         documentSize: {

src/templates/__tests__/templateresources.test.js

@@ -1,20 +1,24 @@
 import { jest } from '@jest/globals';
 
 const setKey = jest.fn();
+const deleteKey = jest.fn();
 
 jest.unstable_mockModule('../../database/redis.js', () => ({
-  redisServer: { setKey },
+  redisServer: { setKey, deleteKey },
 }));
 jest.unstable_mockModule('../../config.js', () => ({
   default: { app: { urlApi: 'https://api.example.test' } },
 }));
 
 const {
+  clearTemplateResourceKeys,
   createTemplateResourceUrl,
   normalizeTemplateResourceName,
   normalizeTemplateResources,
   preprocessTemplateResources,
+  runWithTemplateResourceSession,
   validateTemplateResources,
+  TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS,
   TEMPLATE_RESOURCE_KEY_PREFIX,
   TEMPLATE_RESOURCE_TTL_SECONDS,
 } = await import('../templateresources.js');
@@ -64,7 +68,7 @@ describe('template resources', () => {
     expect(result).toContain('<Image src="@[unknown]" />');
   });
 
-  it('creates a crypto-random one-time token with an exact three-second TTL', async () => {
+  it('creates a crypto-random one-time token with a fallback TTL', async () => {
     const first = await createTemplateResourceUrl('507f1f77bcf86cd799439011');
     const second = await createTemplateResourceUrl('507f1f77bcf86cd799439011');
 
@@ -76,6 +80,34 @@ describe('template resources', () => {
       { fileId: '507f1f77bcf86cd799439011' },
       TEMPLATE_RESOURCE_TTL_SECONDS
     );
-    expect(TEMPLATE_RESOURCE_TTL_SECONDS).toBe(3);
+    expect(TEMPLATE_RESOURCE_TTL_SECONDS).toBeGreaterThan(
+      TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS / 1000
+    );
+  });
+
+  it('clears session keys immediately or after the HTML delay', async () => {
+    jest.useFakeTimers();
+    try {
+      await runWithTemplateResourceSession(async () => {
+        const url = await createTemplateResourceUrl('507f1f77bcf86cd799439011');
+        const key = `${TEMPLATE_RESOURCE_KEY_PREFIX}${url.split('/').at(-1)}`;
+
+        await clearTemplateResourceKeys(TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS);
+        expect(deleteKey).not.toHaveBeenCalled();
+
+        await jest.advanceTimersByTimeAsync(TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS);
+        expect(deleteKey).toHaveBeenCalledWith(key);
+      });
+
+      deleteKey.mockClear();
+      await runWithTemplateResourceSession(async () => {
+        const url = await createTemplateResourceUrl('507f1f77bcf86cd799439012');
+        const key = `${TEMPLATE_RESOURCE_KEY_PREFIX}${url.split('/').at(-1)}`;
+        await clearTemplateResourceKeys();
+        expect(deleteKey).toHaveBeenCalledWith(key);
+      });
+    } finally {
+      jest.useRealTimers();
+    }
   });
 });

src/templates/assets/previewpagination.js

@@ -4,6 +4,7 @@
   var UNSPLITTABLE = 'tr, thead, tfoot, img, svg, hr, canvas, video, picture, col, colgroup';
   var IGNORED = 'script, style, link, meta, .documentPageBreak';
   var PAGE_BREAK_SELECTOR = '.documentPageBreak, pagebreak';
+  var REPEATING = '.documentHeader, .documentFooter';
 
   function markPaginated() {
     document.documentElement.setAttribute('data-paginated', 'true');
@@ -72,7 +73,7 @@
 
   function findFirstOverflowingLeaf(page, clipBottom, locked) {
     function walk(el) {
-      if (locked.has(el) || isIgnored(el)) {
+      if (locked.has(el) || isIgnored(el) || isRepeating(el)) {
         return null;
       }
       if (!isOverflowing(el, clipBottom)) {
@@ -126,11 +127,64 @@
     return clone;
   }
 
+  function isRepeating(el) {
+    return el.matches(REPEATING);
+  }
+
+  function containsRepeating(el) {
+    return isRepeating(el) || !!el.querySelector(REPEATING);
+  }
+
+  function snapshotChildren(el) {
+    var children = [];
+    var child = el.firstElementChild;
+    while (child) {
+      children.push(child);
+      child = child.nextElementSibling;
+    }
+    return children;
+  }
+
+  function cloneRepeating(el) {
+    var clone = el.cloneNode(true);
+    clone.removeAttribute('id');
+    var nested = clone.querySelectorAll('[id]');
+    for (var i = 0; i < nested.length; i += 1) {
+      nested[i].removeAttribute('id');
+    }
+    return clone;
+  }
+
+  function copyRepeatingInPlace(dst, originalChildren, continuation) {
+    var before = [];
+    var after = [];
+    var bucket = before;
+    for (var i = 0; i < originalChildren.length; i += 1) {
+      var child = originalChildren[i];
+      if (child === continuation) {
+        bucket = after;
+        continue;
+      }
+      if (containsRepeating(child)) {
+        bucket.push(cloneRepeating(child));
+      }
+    }
+    var ref = dst.firstElementChild;
+    for (var i = 0; i < before.length; i += 1) {
+      dst.insertBefore(before[i], ref);
+    }
+    for (var i = 0; i < after.length; i += 1) {
+      dst.appendChild(after[i]);
+    }
+  }
+
   function moveFrom(startNode, destination) {
     var node = startNode;
     while (node) {
       var next = node.nextElementSibling;
+      if (!containsRepeating(node)) {
         destination.appendChild(node);
+      }
       node = next;
     }
   }
@@ -141,6 +195,10 @@
     for (var i = 0; i < breaks.length; i += 1) {
       breaks[i].remove();
     }
+    var repeating = clone.querySelectorAll(REPEATING);
+    for (var i = 0; i < repeating.length; i += 1) {
+      repeating[i].remove();
+    }
     if (clone.querySelector('img, svg, canvas, video, picture, hr, table, .documentBarcode')) {
       return false;
     }
@@ -158,6 +216,12 @@
       parent = parent.parentElement;
     }
 
+    var originalPageChildren = snapshotChildren(page);
+    var originalAncestorChildren = [];
+    for (var i = 0; i < ancestors.length; i += 1) {
+      originalAncestorChildren.push(snapshotChildren(ancestors[i]));
+    }
+
     var clonedAncestors = [];
     for (var i = 0; i < ancestors.length; i += 1) {
       clonedAncestors.push(cloneEmpty(ancestors[i]));
@@ -182,6 +246,16 @@
       moveFrom(ancestors[i].nextElementSibling, destination);
     }
 
+    for (var i = 0; i < ancestors.length; i += 1) {
+      var continuation = i === 0 ? splitNode : ancestors[i - 1];
+      copyRepeatingInPlace(clonedAncestors[i], originalAncestorChildren[i], continuation);
+    }
+    copyRepeatingInPlace(
+      newPage,
+      originalPageChildren,
+      ancestors.length > 0 ? ancestors[ancestors.length - 1] : splitNode
+    );
+
     if (isEffectivelyEmpty(newPage)) {
       return null;
     }
@@ -225,7 +299,7 @@
     }
 
     var nextPage = moveOverflowToNewPage(page, splitNode, {
-      discardSplitNode: discardSplitNode
+      discardSplitNode: discardSplitNode,
     });
     if (!nextPage && pageBreak && pageBreak.isConnected) {
       pageBreak.remove();
@@ -323,10 +397,28 @@
       page = nextPage;
     }
 
-    document.documentElement.setAttribute(
+    document.documentElement.setAttribute('data-page-count', String(paginationApi.getPageCount()));
-      'data-page-count',
+
-      String(paginationApi.getPageCount())
+    applyPagePlaceholders();
-    );
+  }
+
+  function applyPagePlaceholders() {
+    var pages = getPageElements();
+    var totalPages = pages.length || 1;
+    for (var i = 0; i < pages.length; i += 1) {
+      var pageNumber = String(i + 1);
+      var total = String(totalPages);
+      var nodes = pages[i].querySelectorAll('*');
+      for (var j = 0; j < nodes.length; j += 1) {
+        var html = String(nodes[j].innerHTML || '').trim();
+        if (html.includes('!pageNumber')) {
+          nodes[j].innerHTML = nodes[j].innerHTML.replaceAll('!pageNumber', pageNumber);
+        }
+        if (html.includes('!totalPages')) {
+          nodes[j].innerHTML = nodes[j].innerHTML.replaceAll('!totalPages', total);
+        }
+      }
+    }
   }
 
   function waitForImages() {

src/templates/templatemanager.js

@@ -24,7 +24,12 @@ import {
 } from './templateintellisense.js';
 import { eventManager } from '../events/eventmanager.js';
 import { redisServer } from '../database/redis.js';
-import { preprocessTemplateResources } from './templateresources.js';
+import {
+  clearTemplateResourceKeys,
+  preprocessTemplateResources,
+  runWithTemplateResourceSession,
+  TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS,
+} from './templateresources.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = dirname(__filename);
@@ -60,11 +65,25 @@ async function loadTemplates() {
 
 loadTemplates();
 
+function getNodeClass(existingClasses = '', attributes) {
+  var classes = existingClasses;
+  if (attributes?.header == 'true') {
+    classes += ' documentHeader';
+  }
+  if (attributes?.footer == 'true') {
+    classes += ' documentFooter';
+  }
+  return classes;
+}
+
 function getNodeStyles(attributes) {
   var styles = '';
   if (attributes?.padding) {
     styles += `padding: ${attributes.padding};`;
   }
+  if (attributes?.margin) {
+    styles += `margin: ${attributes.margin};`;
+  }
   if (attributes?.width) {
     styles += `width: ${attributes.width};`;
   }
@@ -137,38 +156,41 @@ async function transformCustomElements(content) {
       tree.match({ tag: 'Title1' }, (node) => ({
         ...node,
         tag: 'h1',
-        attrs: { class: 'documentTitle' },
+        attrs: { class: getNodeClass('documentTitle', node.attrs) },
       })),
     (tree) =>
       tree.match({ tag: 'Title2' }, (node) => ({
         ...node,
         tag: 'h2',
-        attrs: { class: 'documentTitle' },
+        attrs: { class: getNodeClass('documentTitle', node.attrs) },
       })),
     (tree) =>
       tree.match({ tag: 'Title3' }, (node) => ({
         ...node,
         tag: 'h3',
-        attrs: { class: 'documentText' },
+        attrs: { class: getNodeClass('documentText', node.attrs) },
       })),
     (tree) =>
       tree.match({ tag: 'Title4' }, (node) => ({
         ...node,
         tag: 'h4',
-        attrs: { class: 'documentText' },
+        attrs: { class: getNodeClass('documentText', node.attrs) },
       })),
     (tree) =>
       tree.match({ tag: 'Text' }, (node) => ({
         ...node,
         tag: 'p',
-        attrs: { class: 'documentText', style: getNodeStyles(node.attrs) },
+        attrs: {
+          class: getNodeClass('documentText', node.attrs),
+          style: getNodeStyles(node.attrs),
+        },
       })),
     (tree) =>
       tree.match({ tag: 'Bold' }, (node) => ({
         ...node,
         tag: 'strong',
         attrs: {
-          class: 'documentBoldText',
+          class: getNodeClass('documentBoldText', node.attrs),
           style: getNodeStyles(node.attrs),
         },
       })),
@@ -180,7 +202,7 @@ async function transformCustomElements(content) {
             {
               tag: 'svg',
               attrs: {
-                class: 'documentBarcode',
+                class: getNodeClass('documentBarcode', node.attrs),
                 'jsbarcode-displayValue': 'false',
                 'jsbarcode-value': node.content.at(0) || '',
                 'jsbarcode-format': node.attrs.format || 'code128',
@@ -190,7 +212,7 @@ async function transformCustomElements(content) {
             },
           ],
           attrs: {
-            class: 'documentBarcodeContainer',
+            class: getNodeClass('documentBarcodeContainer', node.attrs),
             style: getNodeStyles(node.attrs),
           },
         };
@@ -200,7 +222,7 @@ async function transformCustomElements(content) {
         ...node,
         tag: 'div',
         attrs: {
-          class: 'documentContainer',
+          class: getNodeClass('documentContainer', node.attrs),
           style: getNodeStyles(node.attrs),
         },
       })),
@@ -210,7 +232,7 @@ async function transformCustomElements(content) {
           ...node,
           tag: 'div',
           attrs: {
-            class: 'documentFlex',
+            class: getNodeClass('documentFlex', node.attrs),
             style: getNodeStyles(node.attrs),
           },
         };
@@ -221,7 +243,7 @@ async function transformCustomElements(content) {
           ...node,
           tag: 'hr',
           attrs: {
-            class: 'documentDivider',
+            class: getNodeClass('documentDivider', node.attrs),
             style: getNodeStyles(node.attrs),
           },
         };
@@ -239,7 +261,7 @@ async function transformCustomElements(content) {
           ...node,
           tag: 'div',
           attrs: {
-            class: 'documentProgressBar',
+            class: getNodeClass('documentProgressBar', node.attrs),
             style: getNodeStyles(node.attrs),
           },
           content: [
@@ -260,7 +282,7 @@ async function transformCustomElements(content) {
           content: [dateTime.format('YYYY-MM-DD hh:mm:ss')],
           tag: 'span',
           attrs: {
-            class: 'documentDateTime',
+            class: getNodeClass('documentDateTime', node.attrs),
             style: getNodeStyles(node.attrs),
           },
         };
@@ -271,7 +293,7 @@ async function transformCustomElements(content) {
           ...node,
           tag: 'table',
           attrs: {
-            class: 'documentTable',
+            class: getNodeClass('documentTable', node.attrs),
             style: getNodeStyles(node.attrs),
           },
         };
@@ -319,7 +341,7 @@ async function transformCustomElements(content) {
             tag: 'tr',
             content,
             attrs: {
-              class: 'documentTableRowHeader',
+              class: getNodeClass('documentTableRowHeader', node.attrs),
               style: getNodeStyles(node.attrs),
             },
           };
@@ -331,7 +353,7 @@ async function transformCustomElements(content) {
             tag: 'tr',
             content,
             attrs: {
-              class: 'documentTableRowFooter',
+              class: getNodeClass('documentTableRowFooter', node.attrs),
               style: getNodeStyles(node.attrs),
             },
           };
@@ -342,7 +364,7 @@ async function transformCustomElements(content) {
           tag: 'tr',
           content,
           attrs: {
-            class: 'documentTableRow',
+            class: getNodeClass('documentTableRow', node.attrs),
             style: getNodeStyles(node.attrs),
           },
         };
@@ -552,6 +574,17 @@ function getRenderObjects(resolvedData) {
   return [{}];
 }
 
+function flattenResolvedPreviewData(resolvedData) {
+  const objects = getRenderObjects(resolvedData).filter(
+    (item) => item != null && typeof item === 'object' && !Array.isArray(item)
+  );
+  const first = objects[0];
+  if (first == null || typeof first.toHexString === 'function') {
+    return resolvedData;
+  }
+  return { ...first, objects };
+}
+
 const OBJECT_PAGE_BREAK = '<hr class="documentPageBreak">';
 
 function buildTemplateData(documentTemplate, data = {}, fc) {
@@ -814,7 +847,7 @@ export class TemplateManager {
       const visited = new Set(template._reference ? [String(template._reference)] : []);
       let templateData;
       const fc = this.createEmailTemplateFc(() => templateData, renderOptions, visited);
-      templateData = buildTemplateData(template, resolvedData, fc);
+      templateData = buildTemplateData(template, flattenResolvedPreviewData(resolvedData), fc);
       const subjectSource = typeof subject === 'string' ? subject : template.subject || '';
       const contentSource = await preprocessTemplateResources(
         typeof content === 'string' ? content : template.content || '',
@@ -951,6 +984,8 @@ export class TemplateManager {
   }
 
   async renderTemplate(id, content, data = {}, _scale = 1, options = {}, preview = true) {
+    return runWithTemplateResourceSession(async () => {
+      const result = await (async () => {
         try {
           const {
             padding: paddingRequested = false,
@@ -1091,9 +1126,17 @@ export class TemplateManager {
           var innerHtml = null;
 
           if (preview == true) {
-        innerHtml = await ejs.render(previewTemplate, { content: templateHtml }, defaultOptions);
+            innerHtml = await ejs.render(
+              previewTemplate,
+              { content: templateHtml },
+              defaultOptions
+            );
           } else {
-        innerHtml = await ejs.render(renderTemplateEjs, { content: templateHtml }, defaultOptions);
+            innerHtml = await ejs.render(
+              renderTemplateEjs,
+              { content: templateHtml },
+              defaultOptions
+            );
           }
 
           if (innerHtml == null || typeof innerHtml !== 'string') {
@@ -1148,6 +1191,15 @@ export class TemplateManager {
           logger.warn('Error whilst previewing template:', error.message);
           return { error: error.message, code: 500 };
         }
+      })();
+
+      if (result?.error) {
+        await clearTemplateResourceKeys();
+      } else if (preview == true) {
+        await clearTemplateResourceKeys(TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS);
+      }
+      return result;
+    });
   }
 
   validateTemplate(templateString) {
@@ -1160,6 +1212,7 @@ export class TemplateManager {
   }
 
   async renderPDF(id, content, data = {}, options = {}) {
+    return runWithTemplateResourceSession(async () => {
       try {
         logger.debug('Rendering PDF for template:', id);
         const { onProgress, padding, ejsOptions, renderRequestId } = splitRenderOptions(options);
@@ -1176,6 +1229,7 @@ export class TemplateManager {
         );
 
         if (renderedTemplate.error != undefined) {
+          await clearTemplateResourceKeys();
           return { error: renderedTemplate.error, code: renderedTemplate.code };
         }
         const baseHtml = renderedTemplate.html;
@@ -1190,6 +1244,7 @@ export class TemplateManager {
           progressEnd: 0.9,
         });
 
+        await clearTemplateResourceKeys();
         return {
           pdf: pdfBuffer,
           width: renderedTemplate.width,
@@ -1198,8 +1253,10 @@ export class TemplateManager {
         };
       } catch (error) {
         logger.warn('Error whilst rendering PDF:', error.message);
+        await clearTemplateResourceKeys();
         return { error: error.message, code: 500 };
       }
+    });
   }
 
   async renderImages(id, content, data = {}, options = {}, format = 'jpeg') {
@@ -1401,7 +1458,7 @@ export class TemplateManager {
       const fc = isEmail
         ? this.createEmailTemplateFc(() => templateData, renderOptions, visited)
         : this.createTemplateFc(() => templateData, renderOptions, visited);
-      templateData = buildTemplateData(template, resolvedData, fc);
+      templateData = buildTemplateData(template, flattenResolvedPreviewData(resolvedData), fc);
 
       const session = createCursorIntellisenseRender(
         source,

src/templates/templateresources.js

@@ -1,11 +1,14 @@
+import { AsyncLocalStorage } from 'async_hooks';
 import { randomBytes } from 'crypto';
 import config from '../config.js';
 import { redisServer } from '../database/redis.js';
 
 const RESOURCE_NAME_PATTERN = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
 const TEMPLATE_RESOURCE_KEY_PREFIX = 'templateresources:';
-const TEMPLATE_RESOURCE_TTL_SECONDS = 3;
+const TEMPLATE_RESOURCE_TTL_SECONDS = 6000;
+const TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS = 10000;
 const RESOURCE_REFERENCE_PATTERN = /@\[([a-z0-9]+(?:-[a-z0-9]+)*)\]/g;
+const templateResourceSession = new AsyncLocalStorage();
 
 export function normalizeTemplateResourceName(value) {
   return String(value ?? '')
@@ -43,15 +46,44 @@ export function validateTemplateResources(resources) {
   return true;
 }
 
+export function runWithTemplateResourceSession(fn) {
+  const existing = templateResourceSession.getStore();
+  if (existing) return fn();
+  return templateResourceSession.run({ keys: [] }, fn);
+}
+
+function takeTemplateResourceKeys() {
+  const store = templateResourceSession.getStore();
+  if (!store?.keys?.length) return [];
+  return store.keys.splice(0, store.keys.length);
+}
+
+export async function clearTemplateResourceKeys(delayMs = 0) {
+  const keys = takeTemplateResourceKeys();
+  if (keys.length === 0) return;
+
+  const deleteKeys = async () => {
+    await Promise.all(keys.map((key) => redisServer.deleteKey(key)));
+  };
+
+  if (delayMs > 0) {
+    const timeout = setTimeout(() => {
+      deleteKeys().catch(() => {});
+    }, delayMs);
+    timeout.unref?.();
+    return;
+  }
+
+  await deleteKeys();
+}
+
 export async function createTemplateResourceUrl(file) {
   const fileId = file?._id ?? file;
   if (!fileId) return null;
   const token = randomBytes(32).toString('hex');
-  await redisServer.setKey(
+  const key = `${TEMPLATE_RESOURCE_KEY_PREFIX}${token}`;
-    `${TEMPLATE_RESOURCE_KEY_PREFIX}${token}`,
+  await redisServer.setKey(key, { fileId: String(fileId) }, TEMPLATE_RESOURCE_TTL_SECONDS);
-    { fileId: String(fileId) },
+  templateResourceSession.getStore()?.keys?.push(key);
-    TEMPLATE_RESOURCE_TTL_SECONDS
-  );
   return `${config.app.urlApi}/template-resources/${token}`;
 }
 
@@ -208,4 +240,9 @@ export async function preprocessTemplateResources(
   return output + source.slice(cursor);
 }
 
-export { RESOURCE_NAME_PATTERN, TEMPLATE_RESOURCE_KEY_PREFIX, TEMPLATE_RESOURCE_TTL_SECONDS };
+export {
+  RESOURCE_NAME_PATTERN,
+  TEMPLATE_RESOURCE_KEY_PREFIX,
+  TEMPLATE_RESOURCE_TTL_SECONDS,
+  TEMPLATE_RESOURCE_HTML_CLEAR_DELAY_MS,
+};

src/utils.js

@@ -2174,17 +2174,37 @@ async function getFileMeta(file) {
   }
 }
 
+function getSchemaTypeRef(schemaType) {
+  if (!schemaType) return undefined;
+  return (
+    schemaType.options?.ref ||
+    schemaType.caster?.options?.ref ||
+    schemaType.embeddedSchemaType?.options?.ref
+  );
+}
+
+function forEachSchemaPath(schema, callback) {
+  if (!schema || typeof callback !== 'function') return;
+
+  schema.eachPath((pathName, schemaType) => {
+    callback(pathName, schemaType);
+  });
+
+  const subpaths = schema.subpaths || {};
+  for (const [pathName, schemaType] of Object.entries(subpaths)) {
+    if (pathName.endsWith('.$')) continue;
+    callback(pathName, schemaType);
+  }
+}
+
 function modelHasRef(model, refName) {
   if (!model || !model.schema) {
     return false;
   }
 
   let hasRef = false;
-  model.schema.eachPath((pathName, schemaType) => {
+  forEachSchemaPath(model.schema, (_pathName, schemaType) => {
-    const directRef = schemaType?.options?.ref;
+    if (getSchemaTypeRef(schemaType) === refName) {
-    const arrayRef = schemaType?.caster?.options?.ref;
-    const ref = directRef || arrayRef;
-    if (ref === refName) {
       hasRef = true;
     }
   });
@@ -2198,18 +2218,39 @@ function getFieldsByRef(model, refName) {
   }
 
   const fields = [];
-  model.schema.eachPath((pathName, schemaType) => {
+  const seen = new Set();
-    const directRef = schemaType?.options?.ref;
+  forEachSchemaPath(model.schema, (pathName, schemaType) => {
-    const arrayRef = schemaType?.caster?.options?.ref;
+    if (getSchemaTypeRef(schemaType) !== refName || seen.has(pathName)) return;
-    const ref = directRef || arrayRef;
+    seen.add(pathName);
-    if (ref === refName) {
     fields.push(pathName);
-    }
   });
 
   return fields;
 }
 
+function collectIdsAtPath(object, pathName) {
+  if (object == null || !pathName) return [];
+
+  const parts = String(pathName).split('.').filter(Boolean);
+  let nodes = [object];
+  for (const part of parts) {
+    nodes = nodes.flatMap((node) => {
+      if (node == null || typeof node !== 'object') return [];
+      const next = node[part];
+      if (next == null || next === '') return [];
+      return Array.isArray(next) ? next : [next];
+    });
+  }
+
+  return nodes
+    .map((value) => {
+      if (value == null || value === '') return null;
+      if (typeof value === 'object') return value._id != null ? String(value._id) : null;
+      return String(value);
+    })
+    .filter(Boolean);
+}
+
 // Build a nested populate specification by walking the schema graph,
 // instead of recursing over already-populated documents.
 function buildDeepPopulateSpec(object, model, populated = new Set()) {
@@ -2221,9 +2262,7 @@ function buildDeepPopulateSpec(object, model, populated = new Set()) {
   const populateSpec = [];
 
   schema.eachPath((pathname, schemaType) => {
-    const directRef = schemaType.options?.ref;
+    const ref = getSchemaTypeRef(schemaType);
-    const arrayRef = schemaType.caster?.options?.ref;
-    const ref = directRef || arrayRef;
     if (!ref) return;
 
     const refName = typeof ref === 'function' ? ref.call(object) : ref;
@@ -2305,6 +2344,7 @@ export {
   getFileMeta,
   modelHasRef,
   getFieldsByRef,
+  collectIdsAtPath,
   jsonToCacheKey,
   subscribeAuditLog,
   unsubscribeAuditLog,