Enhance database schemas and introduce marketplace event handling

- Updated the `notificationUserFromOwner` function to support 'marketplace' as a valid owner type.
- Added a new `marketplaceEvent` schema to manage marketplace events, including fields for `externalReference`, `topic`, and `status`.
- Enhanced existing schemas (e.g., `orderitem`, `shipment`, `client`, `listing`, `listingVarient`, `salesOrder`, `courierService`) to include `externalReference` fields for better integration with external systems.
- Introduced unique indexes for `externalReference` in relevant schemas to ensure data integrity and prevent duplicates.
- Improved the `marketplace` schema to include additional fields for eBay shipping services, enhancing its configurability.

src/database/schemas/inventory/orderitem.schema.js

@@ -63,6 +63,9 @@ const orderItemSchema = new Schema(
     invoicedQuantityRemaining: { type: Number, required: false, default: 0 },
     timestamp: { type: Date, default: Date.now },
     shipment: { type: Schema.Types.ObjectId, ref: 'shipment', required: false },
+    listing: { type: Schema.Types.ObjectId, ref: 'listing', required: false },
+    listingVarient: { type: Schema.Types.ObjectId, ref: 'listingVarient', required: false },
+    externalReference: { type: String, required: false },
     orderedAt: { type: Date, required: false },
     receivedAt: { type: Date, required: false },
   },
@@ -70,6 +73,7 @@ const orderItemSchema = new Schema(
 );
 
 orderItemSchema.index({ name: 'text', itemType: 'text', orderType: 'text' });
+orderItemSchema.index({ order: 1, externalReference: 1 }, { unique: true, sparse: true });
 
 const rollupConfigs = [
   {
@@ -132,7 +136,13 @@ orderItemSchema.statics.recalculate = async function (orderItem, user) {
         cached: true,
       });
       if (sku) {
-        const parentId = sku.part?._id || sku.part || sku.product?._id || sku.product || sku.filament?._id || sku.filament;
+        const parentId =
+          sku.part?._id ||
+          sku.part ||
+          sku.product?._id ||
+          sku.product ||
+          sku.filament?._id ||
+          sku.filament;
         if (syncAmount === 'itemCost') {
           if (sku.overrideCost && sku.cost != null) {
             effectiveItemAmount = sku.cost;

src/database/schemas/inventory/productstock.schema.js

@@ -1,7 +1,7 @@
 import mongoose from 'mongoose';
 import { generateId } from '../../utils.js';
 const { Schema } = mongoose;
-import { aggregateRollups, aggregateRollupsHistory } from '../../database.js';
+import { aggregateRollups, aggregateRollupsHistory, editObject } from '../../database.js';
 
 const partStockUsageSchema = new Schema({
   partStock: { type: Schema.Types.ObjectId, ref: 'partStock', required: false },
@@ -9,6 +9,12 @@ const partStockUsageSchema = new Schema({
   quantity: { type: Number, required: true },
 });
 
+const toId = (value) => {
+  if (value == null) return null;
+  if (typeof value === 'object' && value._id) return String(value._id);
+  return String(value);
+};
+
 // Define the main productStock schema - tracks assembled products consisting of part stocks
 const productStockSchema = new Schema(
   {
@@ -76,6 +82,64 @@ productStockSchema.statics.history = async function (from, to) {
   return results;
 };
 
+productStockSchema.statics.recalculate = async function (productStock, user) {
+  const productSkuId = toId(productStock?.productSku);
+  const stockLocationId = toId(productStock?.stockLocation);
+  if (!productSkuId || !stockLocationId) {
+    return;
+  }
+
+  let productId = toId(productStock?.productSku?.product);
+  if (!productId) {
+    const productSku = await mongoose.model('productSku').findById(productSkuId).select('product').lean();
+    productId = toId(productSku?.product);
+  }
+  if (!productId) {
+    return;
+  }
+
+  const rollupResults = await aggregateRollups({
+    model: this,
+    baseFilter: {
+      productSku: new mongoose.Types.ObjectId(productSkuId),
+      stockLocation: new mongoose.Types.ObjectId(stockLocationId),
+    },
+    rollupConfigs: [
+      {
+        name: 'stockQuantity',
+        rollups: [{ name: 'stockQuantity', property: 'currentQuantity', operation: 'sum' }],
+      },
+    ],
+  });
+  const stockQuantity = rollupResults.stockQuantity?.sum || 0;
+
+  const listingVarientModel = mongoose.model('listingVarient');
+  const varients = await listingVarientModel
+    .find({ productSku: productSkuId })
+    .populate('listing')
+    .lean();
+
+  for (const varient of varients) {
+    const varientProductId =
+      toId(varient.product) || toId(varient.listing?.product);
+    const varientLocationId = toId(varient.listing?.stockLocation);
+    if (varientProductId !== productId || varientLocationId !== stockLocationId) {
+      continue;
+    }
+    if (varient.stockQuantity === stockQuantity) {
+      await listingVarientModel.recalculate(varient, user);
+      continue;
+    }
+
+    await editObject({
+      model: listingVarientModel,
+      id: varient._id,
+      updateData: { stockQuantity },
+      user,
+    });
+  }
+};
+
 // Add virtual id getter
 productStockSchema.virtual('id').get(function () {
   return this._id;

src/database/schemas/inventory/shipment.schema.js

@@ -13,6 +13,7 @@ const shipmentSchema = new Schema(
     order: { type: Schema.Types.ObjectId, refPath: 'orderType', required: true },
     courierService: { type: Schema.Types.ObjectId, ref: 'courierService', required: false },
     trackingNumber: { type: String, required: false },
+    externalReference: { type: String, required: false },
     amount: { type: Number, required: true },
     amountWithTax: { type: Number, required: true },
     taxRate: { type: Schema.Types.ObjectId, ref: 'taxRate', required: false },
@@ -35,6 +36,7 @@ const shipmentSchema = new Schema(
 );
 
 shipmentSchema.index({ trackingNumber: 'text', orderType: 'text' });
+shipmentSchema.index({ order: 1, externalReference: 1 }, { unique: true, sparse: true });
 
 shipmentSchema.statics.recalculate = async function (shipment, user) {
   if (shipment.orderType !== 'purchaseOrder' && shipment.orderType !== 'salesOrder') {

src/database/schemas/inventory/stockevent.schema.js

@@ -190,7 +190,7 @@ const recalculateParentStock = async (parentType, parentId, user) => {
       buildParentUpdateData(parentType, parentStock, events)
     ),
     user,
-    recalculate: false,
+    recalculate: parentType === 'productStock',
   });
 };
 

src/database/schemas/management/auditlog.schema.js

@@ -30,7 +30,7 @@ const auditLogSchema = new Schema(
     ownerType: {
       type: String,
       required: true,
-      enum: ['user', 'printer', 'host'],
+      enum: ['user', 'printer', 'host', 'marketplace'],
     },
   },
   { timestamps: true }

src/database/schemas/management/courier.schema.js

@@ -14,7 +14,14 @@ const courierSchema = new mongoose.Schema(
   { timestamps: true }
 );
 
-courierSchema.index({ name: 'text', website: 'text', email: 'text', phone: 'text', contact: 'text', country: 'text' });
+courierSchema.index({
+  name: 'text',
+  website: 'text',
+  email: 'text',
+  phone: 'text',
+  contact: 'text',
+  country: 'text',
+});
 
 courierSchema.virtual('id').get(function () {
   return this._id;

src/database/schemas/management/courierservice.schema.js

@@ -2,6 +2,14 @@ import mongoose from 'mongoose';
 import { generateId } from '../../utils.js';
 const { Schema } = mongoose;
 
+const marketplaceMappingSchema = new mongoose.Schema(
+  {
+    marketplace: { type: Schema.Types.ObjectId, ref: 'marketplace', required: true },
+    externalReference: { type: String, required: false },
+  },
+  { _id: true }
+);
+
 const courierServiceSchema = new mongoose.Schema(
   {
     _reference: { type: String, default: () => generateId()() },
@@ -18,6 +26,7 @@ const courierServiceSchema = new mongoose.Schema(
     additionalCostWithTax: { required: false, type: Number },
     shippingCurrency: { required: true, type: String, default: 'GBP' },
     international: { required: true, type: Boolean, default: false },
+    marketplaces: { type: [marketplaceMappingSchema], default: [] },
   },
   { timestamps: true }
 );

src/database/schemas/management/documenttemplate.schema.js

@@ -2,6 +2,53 @@ import mongoose from 'mongoose';
 import { generateId } from '../../utils.js';
 const { Schema } = mongoose;
 
+const RENDER_DOCUMENT_TEMPLATE_CALL =
+  /fc\.renderDocumentTemplate\s*\(\s*(['"])([^'"]+)\1/g;
+
+function extractRenderDocumentTemplateReferences(content) {
+  if (content == null || typeof content !== 'string' || content === '') {
+    return [];
+  }
+
+  const references = [];
+  const seen = new Set();
+  const regex = new RegExp(RENDER_DOCUMENT_TEMPLATE_CALL.source, 'g');
+  let match;
+  while ((match = regex.exec(content)) !== null) {
+    const reference = match[2]?.trim();
+    if (!reference || seen.has(reference)) {
+      continue;
+    }
+    seen.add(reference);
+    references.push(reference);
+  }
+  return references;
+}
+
+function objectIdsFromFilterValue(value) {
+  if (value == null) {
+    return [];
+  }
+  if (value instanceof mongoose.Types.ObjectId) {
+    return [value];
+  }
+  if (typeof value === 'string' && /^[a-f\d]{24}$/i.test(value)) {
+    return [new mongoose.Types.ObjectId(value)];
+  }
+  if (Array.isArray(value)) {
+    return value.flatMap(objectIdsFromFilterValue);
+  }
+  if (typeof value === 'object') {
+    if (Array.isArray(value.$in)) {
+      return objectIdsFromFilterValue(value.$in);
+    }
+    if (value._id != null) {
+      return objectIdsFromFilterValue(value._id);
+    }
+  }
+  return [];
+}
+
 const documentTemplateSchema = new Schema(
   {
     _reference: { type: String, default: () => generateId()() },
@@ -39,6 +86,13 @@ const documentTemplateSchema = new Schema(
         required: false,
       },
     ],
+    referencedTemplates: [
+      {
+        type: Schema.Types.ObjectId,
+        ref: 'documentTemplate',
+        required: false,
+      },
+    ],
     content: {
       type: String,
       required: false,
@@ -63,4 +117,50 @@ documentTemplateSchema.virtual('id').get(function () {
 // Configure JSON serialization to include virtuals
 documentTemplateSchema.set('toJSON', { virtuals: true });
 
+documentTemplateSchema.statics.recalculate = async function (documentTemplate, user) {
+  const documentTemplateId = documentTemplate?._id || documentTemplate;
+  if (!documentTemplateId) {
+    return;
+  }
+
+  const stillExists = await this.exists({ _id: documentTemplateId });
+  if (!stillExists) {
+    return;
+  }
+
+  const { getFilter } = await import('../../../utils.js');
+  const references = extractRenderDocumentTemplateReferences(documentTemplate?.content);
+  const referencedTemplateIds = [];
+  const seenIds = new Set();
+
+  for (const reference of references) {
+    const filter = await getFilter({ parent: reference }, ['parent'], true, this);
+    const ids = objectIdsFromFilterValue(filter.parent);
+    for (const id of ids) {
+      const idString = String(id);
+      if (!idString || seenIds.has(idString)) {
+        continue;
+      }
+      seenIds.add(idString);
+      referencedTemplateIds.push(id);
+    }
+  }
+
+  if (documentTemplate && typeof documentTemplate === 'object' && !documentTemplate._bsontype) {
+    documentTemplate.referencedTemplates = referencedTemplateIds.map((id) => ({
+      _id: String(id),
+    }));
+  }
+
+  const { editObject } = await import('../../database.js');
+  await editObject({
+    model: this,
+    id: documentTemplateId,
+    updateData: { referencedTemplates: referencedTemplateIds },
+    user,
+    populate: [{ path: 'referencedTemplates', strictPopulate: false }],
+    recalculate: false,
+  });
+};
+
 export const documentTemplateModel = mongoose.model('documentTemplate', documentTemplateSchema);

src/database/schemas/models.js

@@ -48,6 +48,7 @@ import { salesOrderModel } from './sales/salesorder.schema.js';
 import { marketplaceModel } from './sales/marketplace.schema.js';
 import { listingModel } from './sales/listing.schema.js';
 import { listingVarientModel } from './sales/listingvarient.schema.js';
+import { marketplaceEventModel } from './sales/marketplaceevent.schema.js';
 import { paymentModel } from './finance/payment.schema.js';
 
 // Map prefixes to models and id fields
@@ -396,6 +397,13 @@ export const models = {
     label: 'Listing Varient',
     referenceField: '_reference',
   },
+  MKE: {
+    model: marketplaceEventModel,
+    idField: '_id',
+    type: 'marketplaceEvent',
+    label: 'Marketplace Event',
+    referenceField: '_reference',
+  },
   PAY: {
     model: paymentModel,
     idField: '_id',

src/database/schemas/sales/client.schema.js

@@ -16,6 +16,7 @@ const clientSchema = new mongoose.Schema(
     _reference: { type: String, default: () => generateId()() },
     name: { required: true, type: String },
     marketplace: { type: mongoose.Schema.Types.ObjectId, ref: 'marketplace', required: false },
+    externalReference: { type: String, required: false },
     email: { required: false, type: String },
     phone: { required: false, type: String },
     country: { required: false, type: String },
@@ -27,6 +28,7 @@ const clientSchema = new mongoose.Schema(
 );
 
 clientSchema.index({ name: 'text', email: 'text', phone: 'text', country: 'text', tags: 'text' });
+clientSchema.index({ marketplace: 1, externalReference: 1 }, { unique: true, sparse: true });
 
 clientSchema.virtual('id').get(function () {
   return this._id;

src/database/schemas/sales/listing.schema.js

@@ -19,15 +19,43 @@ const listingSchema = new Schema(
       message: { type: String, required: false },
     },
     url: { type: String, required: false },
+    description: { type: String, required: false },
+    externalReference: { type: String, required: false },
     price: { type: Number, required: false },
     currency: { type: String, required: false },
     lastSyncedAt: { type: Date, required: false },
+    stockQuantity: { type: Number, required: false, default: 0 },
+    condition: {
+      type: String,
+      enum: [
+        'new',
+        'likeNew',
+        'newOther',
+        'newWithDefects',
+        'manufacturerRefurbished',
+        'certifiedRefurbished',
+        'excellentRefurbished',
+        'veryGoodRefurbished',
+        'goodRefurbished',
+        'sellerRefurbished',
+        'usedExcellent',
+        'usedVeryGood',
+        'usedGood',
+        'usedAcceptable',
+        'forPartsOrNotWorking',
+        'preOwnedExcellent',
+        'preOwnedFair',
+      ],
+      default: 'new',
+      required: false,
+    },
     courierServices: [{ type: Schema.Types.ObjectId, ref: 'courierService', required: true }],
   },
   { timestamps: true }
 );
 
 listingSchema.index({ title: 'text', url: 'text' });
+listingSchema.index({ marketplace: 1, externalReference: 1 }, { unique: true, sparse: true });
 
 listingSchema.virtual('id').get(function () {
   return this._id;

src/database/schemas/sales/listingvarient.schema.js

@@ -1,5 +1,6 @@
 import mongoose from 'mongoose';
 import { generateId } from '../../utils.js';
+import { aggregateRollups, editObject } from '../../database.js';
 const { Schema } = mongoose;
 
 const listingVarientSchema = new Schema(
@@ -16,16 +17,19 @@ const listingVarientSchema = new Schema(
       },
       message: { type: String, required: false },
     },
+    externalReference: { type: String, required: false },
     price: { type: Number, required: false },
     currency: { type: String, required: false },
     priceTaxRate: { type: Schema.Types.ObjectId, ref: 'taxRate', required: false },
     priceWithTax: { type: Number, required: false },
     lastSyncedAt: { type: Date, required: false },
+    stockQuantity: { type: Number, required: false, default: 0 },
   },
   { timestamps: true }
 );
 
 listingVarientSchema.index({ currency: 'text', 'state.type': 'text' });
+listingVarientSchema.index({ listing: 1, externalReference: 1 }, { unique: true, sparse: true });
 
 listingVarientSchema.virtual('id').get(function () {
   return this._id;
@@ -42,4 +46,32 @@ listingVarientSchema.set('toJSON', {
   },
 });
 
+listingVarientSchema.statics.recalculate = async function (listingVarient, user) {
+  const listingId = listingVarient?.listing?._id || listingVarient?.listing;
+  if (!listingId) {
+    return;
+  }
+
+  const rollupResults = await aggregateRollups({
+    model: this,
+    baseFilter: { listing: new mongoose.Types.ObjectId(listingId) },
+    rollupConfigs: [
+      {
+        name: 'stockQuantity',
+        rollups: [{ name: 'stockQuantity', property: 'stockQuantity', operation: 'sum' }],
+      },
+    ],
+  });
+
+  await editObject({
+    model: mongoose.model('listing'),
+    id: listingId,
+    updateData: {
+      stockQuantity: rollupResults.stockQuantity?.sum || 0,
+    },
+    user,
+    recalculate: false,
+  });
+};
+
 export const listingVarientModel = mongoose.model('listingVarient', listingVarientSchema);

src/database/schemas/sales/marketplace.schema.js

@@ -17,13 +17,16 @@ const marketplaceSchema = new mongoose.Schema(
     state: {
       type: {
         type: String,
-        enum: ['active', 'inactive', 'suspended', 'ready', 'offline', 'syncing'],
+        enum: ['active', 'inactive', 'suspended', 'ready', 'offline', 'syncing', 'disconnected'],
         default: 'offline',
       },
       message: { type: String, required: false },
     },
     // Provider-specific API configuration (flexible for eBay, Etsy, TikTok Shop)
     config: { type: mongoose.Schema.Types.Mixed, default: {} },
+    eBay: {
+      availableShippingServices: { type: [String], default: [] },
+    },
   },
   { timestamps: true }
 );

src/database/schemas/sales/marketplaceevent.schema.js

@@ -0,0 +1,31 @@
+import mongoose from 'mongoose';
+import { generateId } from '../../utils.js';
+
+const marketplaceEventSchema = new mongoose.Schema(
+  {
+    _reference: { type: String, default: () => generateId()() },
+    marketplace: { type: mongoose.Schema.Types.ObjectId, ref: 'marketplace', required: true },
+    externalReference: { type: String, required: true },
+    topic: { type: String, required: true },
+    status: {
+      type: String,
+      required: true,
+      enum: ['received', 'processed', 'failed'],
+      default: 'received',
+    },
+    processedAt: { type: Date, required: false },
+    error: { type: String, required: false },
+  },
+  { timestamps: true }
+);
+
+marketplaceEventSchema.index({ marketplace: 1, externalReference: 1 }, { unique: true });
+marketplaceEventSchema.index({ topic: 'text', status: 'text' });
+
+marketplaceEventSchema.virtual('id').get(function () {
+  return this._id;
+});
+
+marketplaceEventSchema.set('toJSON', { virtuals: true });
+
+export const marketplaceEventModel = mongoose.model('marketplaceEvent', marketplaceEventSchema);

src/database/schemas/sales/salesorder.schema.js

@@ -1,11 +1,7 @@
 import mongoose from 'mongoose';
 import { generateId } from '../../utils.js';
 const { Schema } = mongoose;
-import {
-  aggregateRollups,
-  aggregateRollupsHistory,
-  editObject,
-} from '../../database.js';
+import { aggregateRollups, aggregateRollupsHistory, editObject } from '../../database.js';
 
 const salesOrderSchema = new Schema(
   {
@@ -19,6 +15,7 @@ const salesOrderSchema = new Schema(
     timestamp: { type: Date, default: Date.now },
     client: { type: Schema.Types.ObjectId, ref: 'client', required: true },
     marketplace: { type: Schema.Types.ObjectId, ref: 'marketplace', required: false },
+    externalReference: { type: String, required: false },
     state: {
       type: { type: String, required: true, default: 'draft' },
     },
@@ -31,6 +28,7 @@ const salesOrderSchema = new Schema(
 );
 
 salesOrderSchema.index({ 'state.type': 'text' });
+salesOrderSchema.index({ marketplace: 1, externalReference: 1 }, { unique: true, sparse: true });
 
 const rollupConfigs = [
   {

src/database/utils.js

@@ -40,7 +40,7 @@ function notificationUserFromOwner(owner, ownerType) {
   if (ownerType === 'user') {
     return owner;
   }
-  if (ownerType === 'host') {
+  if (ownerType === 'host' || ownerType === 'marketplace') {
     return {
       _id: owner._id,
       firstName: owner.name ?? 'unknown',