import { mongoose } from 'mongoose';
import { auditLogModel } from './database/schemas/management/auditlog.schema.js';
import { notificationModel } from './database/schemas/misc/notification.schema.js';
import { userNotifierModel } from './database/schemas/misc/usernotifier.schema.js';
import { userModel } from './database/schemas/management/user.schema.js';
import exifr from 'exifr';
import { natsServer } from './database/nats.js';
import log4js from 'log4js';
import config from './config.js';
import crypto from 'crypto';
import canonicalize from 'canonical-json';
import { getModelByName, getModelByPrefix } from './services/misc/model.js';
import { models } from './database/schemas/models.js';
import { createEmailRenderAuthCode } from './services/misc/emailRenderAuth.js';
import { Worker } from 'worker_threads';
import path from 'path';
import { fileURLToPath } from 'url';
import { diffJson } from 'diff';
import { resolveAuditOwner, actorDisplayName } from './auditOwner.js';

const __dirname = path.dirname(fileURLToPath(import.meta.url));

const logger = log4js.getLogger('Utils');
logger.level = config.server.logLevel;

// ---------------------------------------------------------------------------
// Filter expression parsing (Microsoft Dynamics NAV / Business Central style)
//
// Supported syntax (all compiled down to MongoDB query operators):
//   =        equal to                  377            -> { field: 377 }
//   <>       not equal to              <>0            -> { field: { $ne: 0 } }
//   >        greater than              >1200          -> { field: { $gt: 1200 } }
//   >=       greater than or equal     >=1200         -> { field: { $gte: 1200 } }
//   <        less than                 <1200          -> { field: { $lt: 1200 } }
//   <=       less than or equal        <=1200         -> { field: { $lte: 1200 } }
//   ..       interval                  1100..2100     -> { field: { $gte: 1100, $lte: 2100 } }
//            open-ended intervals      ..2500 / 23..
//   |        either / or               1200|1300      -> { field: { $in: [1200, 1300] } }
//   &        and                       <2000&>1000    -> { field: { $gt: 1000, $lt: 2000 } }
//   ( )      grouping / precedence     30|(>=10&<=20)
//   *        any number of characters  Co* / *Co / *Co*
//   ?        a single character        Hans?n
//   @        ignore case               @location (text matching is case-insensitive)
//   ''       empty (null or missing)   '' / ='' / <>''
//
// Date-typed fields (schema Date) interpret bare values as calendar
// days/datetimes, e.g. "22" => the whole of day 22 of the current month/year,
// "22..23" => start of 22 through end of 23.
//
// Relative date operands:
//   TODAY      current calendar day
//   TODAY..    from start of today onwards
//   -7D..      from 7 days ago onwards (last 7 days)
//   -30D..     from 30 days ago onwards
//   -1W        previous calendar week (Monday–Sunday)
//   -1M        previous calendar month (as a whole day-range)
//   -1M..-CM   previous calendar month (from its start up to current month)
//   -1Y        previous calendar year
//   -nD / +nD  n days before / after today
//   -nW / +nW  n calendar weeks before / after the current week
//   -nM / +nM  n calendar months before / after the current month
//   -nY / +nY  n calendar years before / after the current year
//   -nH / +nH  n hours before / after the current hour
//   -nm / +nm  n minutes before / after the current minute (lowercase m)
//   CW / CM / CY / CH / Cm   current week / month / year / hour / minute
//   -CW / -CM / -CY / -CH / -Cm
//                current-unit boundary (start of this unit / end of last unit)
//
// Unit letters: D day, W week, M month, Y year, H hour, m minute.
// M (month) and m (minute) are case-sensitive; other units are not.
//
// An invalid date operand (anything other than a date, relative date, or
// filter operator) omits that field from the query instead of sending it to
// MongoDB.
//
// Note: BC's space-delimited date shorthand is locale dependent; here numeric
// date operands are read day-first ("D M Y H Min S") with missing components
// filled from the current date and the interval boundary.
// ---------------------------------------------------------------------------

function buildWildcardRegexPattern(input) {
  // Escape all regex special chars except * and ? (which we treat as wildcards)
  const escaped = String(input).replace(/[.+^${}()|[\]\\]/g, '\\$&');
  // * matches any run of characters, ? matches exactly one character
  const withWildcards = escaped.replace(/\*/g, '.*').replace(/\?/g, '.');
  // Anchor so that, without wildcards, this is an exact match
  return `^${withWildcards}$`;
}

const NO_MATCH_CONDITION = { op: { $in: [] } };
const EMPTY_OPERAND = "''";

function isEmptyOperand(value) {
  return String(value).trim() === EMPTY_OPERAND;
}

// MongoDB null equality matches both null and missing (undefined) fields.
function emptyCondition(operator = 'eq') {
  if (operator === 'ne') return { op: { $ne: null } };
  return { value: null };
}

function isBooleanField(property, model = null) {
  const schema = model?.schema;
  if (schema) {
    const path = schema.path(property);
    return path?.instance === 'Boolean';
  }

  let found = false;
  for (const entry of Object.values(models)) {
    const path = entry.model?.schema?.path(property);
    if (!path) continue;
    found = true;
    if (path.instance !== 'Boolean') return false;
  }
  return found;
}

function isNumberSchemaPath(path) {
  if (!path) return false;
  if (path.instance === 'Number') return true;
  if (path.instance === 'Array') {
    return getEmbeddedSchemaType(path)?.instance === 'Number';
  }
  return false;
}

function isNumberField(property, model = null) {
  if (model?.schema) {
    return isNumberSchemaPath(model.schema.path(property));
  }

  for (const entry of Object.values(models)) {
    if (isNumberSchemaPath(entry.model?.schema?.path(property))) return true;
  }
  return false;
}

function isDateTimeSchemaPath(path) {
  if (!path) return false;

  if (path.instance === 'Date') return true;

  if (path.instance === 'Array') {
    return getEmbeddedSchemaType(path)?.instance === 'Date';
  }

  return false;
}

function isDateTimeField(property, model = null) {
  if (model?.schema) {
    return isDateTimeSchemaPath(model.schema.path(property));
  }

  for (const entry of Object.values(models)) {
    if (isDateTimeSchemaPath(entry.model?.schema?.path(property))) return true;
  }

  return false;
}

function buildNumberRegexCondition(property, pattern, negated = false) {
  const expr = {
    $regexMatch: {
      input: { $toString: `$${property}` },
      regex: pattern,
      options: 'i',
    },
  };
  return { query: { $expr: negated ? { $not: expr } : expr } };
}

function isObjectIdPath(property) {
  return property === '_id' || property.endsWith('._id');
}

function getBaseProperty(property) {
  if (property === '_id') return '_id';
  if (property.endsWith('._id')) return property.slice(0, -4);
  return property;
}

function getEmbeddedSchemaType(path) {
  if (!path) return null;
  return path.embeddedSchemaType ?? path.$embeddedSchemaType ?? path.caster ?? null;
}

function getObjectIdSchemaTypeFromPath(path) {
  if (!path) return null;
  const schemaType = path.instance === 'Array' ? getEmbeddedSchemaType(path) : path;
  if (!schemaType) return null;
  if (schemaType.instance === 'ObjectId' || schemaType.instance === 'ObjectID') {
    return schemaType;
  }
  return null;
}

function getSchemaPathFromModels(property, model = null) {
  if (model?.schema?.path(property)) return model.schema.path(property);
  for (const entry of Object.values(models)) {
    const path = entry.model?.schema?.path(property);
    if (path) return path;
  }
  return null;
}

function getFilterFieldKind(property, model = null) {
  const baseProperty = getBaseProperty(property);

  const inspectPath = (path) => {
    if (!path) return null;

    const objectIdType = getObjectIdSchemaTypeFromPath(path);
    if (isObjectIdPath(property) || objectIdType) {
      const ref = objectIdType?.options?.ref ?? path.options?.ref;
      if (property.endsWith('._id') && property !== '_id' && ref) {
        return { kind: 'objectId', property: baseProperty };
      }
      if (ref) {
        return { kind: 'objectRef', property: baseProperty, ref };
      }
      return { kind: 'objectId', property: baseProperty };
    }

    return { kind: 'default', property: baseProperty };
  };

  if (model) {
    return (
      inspectPath(model.schema?.path(baseProperty)) || {
        kind: 'default',
        property: baseProperty,
      }
    );
  }

  const path = getSchemaPathFromModels(baseProperty);
  const kind = inspectPath(path);
  if (kind) return kind;

  return { kind: 'default', property: baseProperty };
}

function parsePrefixedValue(value) {
  const trimmed = String(value).trim();
  if (trimmed.length > 4 && trimmed.charAt(3) === ':') {
    return {
      prefix: trimmed.slice(0, 3).toUpperCase(),
      suffix: trimmed.slice(4).trim(),
      hadPrefix: true,
    };
  }
  return { prefix: null, suffix: trimmed, hadPrefix: false };
}

const FILTER_OPERATOR_PREFIXES = ['<>', '>=', '<=', '>', '<', '='];

function splitLeadingOperator(str) {
  const text = String(str).trim();
  for (const op of FILTER_OPERATOR_PREFIXES) {
    if (text.startsWith(op)) {
      return { operator: op, operand: text.slice(op.length).trim() };
    }
  }
  return { operator: '', operand: text };
}

// Strips a model prefix (e.g. GCF:) from an expression, including after a leading operator.
function stripModelPrefixFromExpression(expression) {
  const { operator, operand } = splitLeadingOperator(expression);
  const { suffix, hadPrefix } = parsePrefixedValue(operand);
  if (!hadPrefix) return expression;
  return operator + suffix;
}

function buildRegexOp(pattern, useOptions = true) {
  const op = { $regex: pattern };
  if (useOptions) op.$options = 'i';
  return op;
}

function getSchemaRefName(property, model = null) {
  const baseProperty = getBaseProperty(property);
  const path = model?.schema?.path(baseProperty) ?? getSchemaPathFromModels(baseProperty);
  const embeddedType = getEmbeddedSchemaType(path);
  return path?.options?.ref ?? embeddedType?.options?.ref ?? null;
}

function getRefModelEntryFromSchemaRef(refName) {
  if (!refName) return null;

  let refModel;
  try {
    refModel = mongoose.model(refName);
  } catch {
    refModel = getModelByName(refName)?.model ?? null;
  }

  if (!refModel) return null;

  const registryEntry = getModelByName(refName);
  return {
    model: refModel,
    idField: registryEntry?.idField ?? '_id',
    referenceField: registryEntry?.referenceField ?? '_reference',
  };
}

function getRefModelEntryFromPrefix(prefix) {
  const entry = getModelByPrefix(prefix);
  if (!entry?.model) return null;
  return {
    model: entry.model,
    idField: entry.idField ?? '_id',
    referenceField: entry.referenceField ?? '_reference',
  };
}

function stripPrefixFromOperand(operand) {
  return stripModelPrefixFromExpression(operand);
}

function getRefModelEntryForToken(token, fallbackRefName) {
  const { operand } = splitLeadingOperator(token);
  const { prefix, hadPrefix } = parsePrefixedValue(operand);
  if (hadPrefix) {
    const entry = getRefModelEntryFromPrefix(prefix);
    if (entry) return entry;
  }
  return getRefModelEntryFromSchemaRef(fallbackRefName);
}

function parseBooleanOperand(value) {
  const lower = String(value).trim().toLowerCase();
  if (lower === 'yes') return true;
  if (lower === 'no') return false;
  if (lower === 'true') return true;
  if (lower === 'false') return false;
  if (lower === '1') return true;
  if (lower === '0') return false;
  if (lower === 'on') return true;
  if (lower === 'off') return false;
  if (lower === 'y') return true;
  if (lower === 'n') return false;
  if (typeof value === 'boolean') return value;
  return undefined;
}

function buildBooleanEquality(value) {
  const bool = parseBooleanOperand(value);
  if (bool === undefined) return NO_MATCH_CONDITION;
  return { value: bool };
}

function buildBooleanComparison(name, value) {
  if (name === 'eq') return buildBooleanEquality(value);

  if (name === 'ne') {
    const bool = parseBooleanOperand(value);
    if (bool === undefined) return NO_MATCH_CONDITION;
    return { op: { $ne: bool } };
  }

  return NO_MATCH_CONDITION;
}

function isObjectIdString(value) {
  return /^[a-f\d]{24}$/i.test(value);
}

function isNumeric(value) {
  return value.trim() !== '' && !isNaN(value);
}

function startOfDay(date) {
  const d = new Date(date);
  d.setHours(0, 0, 0, 0);
  return d;
}

function endOfDay(date) {
  const d = new Date(date);
  d.setHours(23, 59, 59, 999);
  return d;
}

function startOfMonth(date) {
  return new Date(date.getFullYear(), date.getMonth(), 1, 0, 0, 0, 0);
}

function endOfMonth(date) {
  return new Date(date.getFullYear(), date.getMonth() + 1, 0, 23, 59, 59, 999);
}

function shiftCalendarMonths(date, n) {
  return new Date(date.getFullYear(), date.getMonth() + n, 1);
}

function startOfWeek(date) {
  const d = startOfDay(date);
  const day = d.getDay();
  const mondayOffset = day === 0 ? -6 : 1 - day;
  d.setDate(d.getDate() + mondayOffset);
  return d;
}

function endOfWeek(date) {
  const start = startOfWeek(date);
  const end = new Date(start);
  end.setDate(end.getDate() + 6);
  return endOfDay(end);
}

function shiftWeeks(date, n) {
  const d = new Date(date);
  d.setDate(d.getDate() + n * 7);
  return d;
}

function startOfYear(date) {
  return new Date(date.getFullYear(), 0, 1, 0, 0, 0, 0);
}

function endOfYear(date) {
  return new Date(date.getFullYear(), 11, 31, 23, 59, 59, 999);
}

function shiftYears(date, n) {
  return new Date(date.getFullYear() + n, 0, 1);
}

function startOfHour(date) {
  const d = new Date(date);
  d.setMinutes(0, 0, 0);
  return d;
}

function endOfHour(date) {
  const d = new Date(date);
  d.setMinutes(59, 59, 999);
  return d;
}

function startOfMinute(date) {
  const d = new Date(date);
  d.setSeconds(0, 0);
  return d;
}

function endOfMinute(date) {
  const d = new Date(date);
  d.setSeconds(59, 999);
  return d;
}

function normalizeYear(year) {
  if (year >= 100) return year;
  return year < 70 ? 2000 + year : 1900 + year;
}

const RELATIVE_OFFSET_RE = /^([+-])?(\d+)([DdWwYyHhMm])$/;
const RELATIVE_CURRENT_RE = /^([+-])?C([DdWwYyHhMm])$/;

function normalizeRelativeUnit(letter) {
  if (letter === 'M' || letter === 'm') return letter;
  return letter.toUpperCase();
}

function resolveRelativeUnit(now, unit, amount, boundary, isCurrent, minusCurrent) {
  const end = boundary === 'end';

  if (unit === 'D') {
    if (isCurrent) return end ? endOfDay(now) : startOfDay(now);
    const date = new Date(now);
    date.setDate(date.getDate() + amount);
    return end ? endOfDay(date) : startOfDay(date);
  }

  if (unit === 'W') {
    if (isCurrent && minusCurrent) {
      return end ? endOfWeek(shiftWeeks(now, -1)) : startOfWeek(now);
    }
    const target = isCurrent ? now : shiftWeeks(now, amount);
    return end ? endOfWeek(target) : startOfWeek(target);
  }

  if (unit === 'M') {
    if (isCurrent) {
      if (minusCurrent) {
        return end ? endOfMonth(shiftCalendarMonths(now, -1)) : startOfMonth(now);
      }
      return end ? endOfMonth(now) : startOfMonth(now);
    }
    const month = shiftCalendarMonths(now, amount);
    return end ? endOfMonth(month) : startOfMonth(month);
  }

  if (unit === 'Y') {
    if (isCurrent) {
      if (minusCurrent) {
        return end ? endOfYear(shiftYears(now, -1)) : startOfYear(now);
      }
      return end ? endOfYear(now) : startOfYear(now);
    }
    const year = shiftYears(now, amount);
    return end ? endOfYear(year) : startOfYear(year);
  }

  if (unit === 'H') {
    if (isCurrent) {
      if (minusCurrent) {
        const prev = new Date(now);
        prev.setHours(prev.getHours() - 1);
        return end ? endOfHour(prev) : startOfHour(now);
      }
      return end ? endOfHour(now) : startOfHour(now);
    }
    const date = new Date(now);
    date.setHours(date.getHours() + amount);
    return end ? endOfHour(date) : startOfHour(date);
  }

  if (unit === 'm') {
    if (isCurrent) {
      if (minusCurrent) {
        const prev = new Date(now);
        prev.setMinutes(prev.getMinutes() - 1);
        return end ? endOfMinute(prev) : startOfMinute(now);
      }
      return end ? endOfMinute(now) : startOfMinute(now);
    }
    const date = new Date(now);
    date.setMinutes(date.getMinutes() + amount);
    return end ? endOfMinute(date) : startOfMinute(date);
  }

  return null;
}

function parseRelativeDateOperand(text, boundary = 'start') {
  const raw = String(text).trim();
  if (!raw) return null;

  if (/^TODAY$/i.test(raw)) {
    const now = new Date();
    return boundary === 'end' ? endOfDay(now) : startOfDay(now);
  }

  const offset = raw.match(RELATIVE_OFFSET_RE);
  if (offset) {
    const amount = (offset[1] === '-' ? -1 : 1) * Number(offset[2]);
    return resolveRelativeUnit(
      new Date(),
      normalizeRelativeUnit(offset[3]),
      amount,
      boundary,
      false,
      false
    );
  }

  const current = raw.match(RELATIVE_CURRENT_RE);
  if (current) {
    return resolveRelativeUnit(
      new Date(),
      normalizeRelativeUnit(current[2]),
      0,
      boundary,
      true,
      current[1] === '-'
    );
  }

  return null;
}

function dateComponentsMatch(date, year, month, day, hour, minute, second) {
  return (
    date.getFullYear() === year &&
    date.getMonth() === month - 1 &&
    date.getDate() === day &&
    date.getHours() === hour &&
    date.getMinutes() === minute &&
    date.getSeconds() === second
  );
}

// Parses a single date operand to a Date, filling missing parts from the
// current date and the supplied interval boundary ('start' | 'end').
function parseDateOperand(value, boundary = 'start') {
  const text = String(value).trim();
  if (!text) return null;
  const end = boundary === 'end';

  const relative = parseRelativeDateOperand(text, boundary);
  if (relative) return relative;

  // Values with explicit separators (ISO and similar) are parsed directly.
  if (/[-/T]/.test(text) || /\d:\d/.test(text)) {
    const parsed = new Date(text);
    if (isNaN(parsed.getTime())) return null;
    if (!/[T:]/.test(text)) {
      return end ? endOfDay(parsed) : startOfDay(parsed);
    }
    return parsed;
  }

  // Numeric, day-first form: "D [M] [Y] [H] [Min] [S]".
  const parts = text.split(/\s+/);
  if (!parts.every((part) => /^\d+$/.test(part))) return null;
  const nums = parts.map(Number);
  const now = new Date();
  const day = nums[0];
  const month = nums.length >= 2 ? nums[1] : now.getMonth() + 1;
  const year = nums.length >= 3 ? normalizeYear(nums[2]) : now.getFullYear();
  const hour = nums.length >= 4 ? nums[3] : end ? 23 : 0;
  const minute = nums.length >= 5 ? nums[4] : end ? 59 : 0;
  const second = nums.length >= 6 ? nums[5] : end ? 59 : 0;
  const ms = end ? 999 : 0;
  const date = new Date(year, month - 1, day, hour, minute, second, ms);
  if (isNaN(date.getTime())) return null;
  if (!dateComponentsMatch(date, year, month, day, hour, minute, second)) return null;
  return date;
}

function isValidDateOperand(value) {
  return parseDateOperand(value, 'start') != null;
}

function isValidDateFilterLeaf(rawToken) {
  const token = rawToken.trim();
  if (token === '') return true;
  if (isEmptyOperand(token)) return true;

  const rangeIdx = token.indexOf('..');
  if (rangeIdx !== -1) {
    const lo = stripIgnoreCase(token.slice(0, rangeIdx).trim());
    const hi = stripIgnoreCase(token.slice(rangeIdx + 2).trim());
    if (lo !== '' && !isValidDateOperand(lo)) return false;
    if (hi !== '' && !isValidDateOperand(hi)) return false;
    return true;
  }

  const operators = ['<>', '>=', '<=', '>', '<', '='];
  for (const symbol of operators) {
    if (token.startsWith(symbol)) {
      const operand = stripIgnoreCase(token.slice(symbol.length).trim());
      if ((symbol === '=' || symbol === '<>') && isEmptyOperand(operand)) return true;
      return operand !== '' && isValidDateOperand(operand);
    }
  }

  return isValidDateOperand(stripIgnoreCase(token));
}

function isValidDateFilterNode(node) {
  if (node.type === 'leaf') return isValidDateFilterLeaf(node.token);
  return node.items.every(isValidDateFilterNode);
}

function isValidDateFilterExpression(expression) {
  return isValidDateFilterNode(parseExpression(String(expression)));
}

// Coerces a literal to its strongest matching scalar type.
function coerceScalar(value) {
  const lower = value.toLowerCase();
  if (lower === 'true') return true;
  if (lower === 'false') return false;
  if (isObjectIdString(value)) return new mongoose.Types.ObjectId(value);
  if (isNumeric(value)) return Number(value);
  return value;
}

// Coerces an interval/comparison boundary, preferring a date for date fields.
function coerceBoundary(value, isDateField, boundary) {
  if (isDateField) {
    return parseDateOperand(value, boundary);
  }
  return coerceScalar(value);
}

// Splits on a separator while respecting parentheses depth.
function splitTopLevel(str, separator) {
  const parts = [];
  let depth = 0;
  let current = '';
  for (const ch of str) {
    if (ch === '(') depth++;
    else if (ch === ')') depth = Math.max(0, depth - 1);

    if (ch === separator && depth === 0) {
      parts.push(current);
      current = '';
    } else {
      current += ch;
    }
  }
  parts.push(current);
  return parts;
}

// True when a single outer pair of parentheses wraps the whole token.
function isWrappedInParens(str) {
  if (!str.startsWith('(') || !str.endsWith(')')) return false;
  let depth = 0;
  for (let i = 0; i < str.length; i++) {
    if (str[i] === '(') depth++;
    else if (str[i] === ')') {
      depth--;
      if (depth === 0 && i < str.length - 1) return false;
    }
  }
  return depth === 0;
}

// Parses a filter expression into an AST of or / and / leaf nodes.
function parseExpression(str) {
  const orParts = splitTopLevel(str, '|');
  if (orParts.length > 1) {
    return { type: 'or', items: orParts.map(parseExpression) };
  }
  const andParts = splitTopLevel(str, '&');
  if (andParts.length > 1) {
    return { type: 'and', items: andParts.map(parseExpression) };
  }
  const trimmed = str.trim();
  if (isWrappedInParens(trimmed)) {
    return parseExpression(trimmed.slice(1, -1));
  }
  return { type: 'leaf', token: trimmed };
}

// Strips a leading @ (ignore-case marker); text matching is already case-insensitive.
function stripIgnoreCase(str) {
  return str.startsWith('@') ? str.slice(1) : str;
}

// Builds an equality condition (value, date range, or wildcard regex).
function buildEquality(
  value,
  isDateField,
  isBooleanField = false,
  isObjectIdField = false,
  isNumberField = false,
  property = ''
) {
  if (isEmptyOperand(value)) return emptyCondition('eq');
  if (isBooleanField) return buildBooleanEquality(value);
  if (isObjectIdField) {
    const { suffix } = parsePrefixedValue(value);
    if (/[*?]/.test(suffix)) return NO_MATCH_CONDITION;
    if (isObjectIdString(suffix)) return { value: new mongoose.Types.ObjectId(suffix) };
    return NO_MATCH_CONDITION;
  }
  if (isDateField) {
    const start = parseDateOperand(value, 'start');
    const end = parseDateOperand(value, 'end');
    if (start && end) return { op: { $gte: start, $lte: end } };
    return NO_MATCH_CONDITION;
  }
  const lower = value.toLowerCase();
  if (lower === 'true') return { value: true };
  if (lower === 'false') return { value: false };
  if (isObjectIdString(value)) return { value: new mongoose.Types.ObjectId(value) };
  if (isNumeric(value)) return { value: Number(value) };
  if (isNumberField) {
    if (/[*?]/.test(value)) {
      return buildNumberRegexCondition(property, buildWildcardRegexPattern(value));
    }
    return NO_MATCH_CONDITION;
  }
  return { op: buildRegexOp(buildWildcardRegexPattern(value)) };
}

// Builds a comparison condition for a single operator.
function buildComparison(
  name,
  value,
  isDateField,
  isBooleanField = false,
  isObjectIdField = false,
  isNumberField = false,
  property = ''
) {
  if (isEmptyOperand(value)) {
    if (name === 'eq' || name === 'ne') return emptyCondition(name);
    return NO_MATCH_CONDITION;
  }
  if (isBooleanField) return buildBooleanComparison(name, value);
  if (isObjectIdField) {
    const { suffix } = parsePrefixedValue(value);
    if (/[*?]/.test(suffix)) return NO_MATCH_CONDITION;
    if (name === 'eq') return buildEquality(suffix, false, false, true);
    if (name === 'ne' && isObjectIdString(suffix)) {
      return { op: { $ne: new mongoose.Types.ObjectId(suffix) } };
    }
    return NO_MATCH_CONDITION;
  }
  if (name === 'eq') {
    return buildEquality(value, isDateField, false, false, isNumberField, property);
  }

  if (name === 'ne') {
    if (isDateField) {
      const start = parseDateOperand(value, 'start');
      const end = parseDateOperand(value, 'end');
      if (start && end) return { op: { $not: { $gte: start, $lte: end } } };
      return NO_MATCH_CONDITION;
    }
    if (/[*?]/.test(value)) {
      if (isNumberField) {
        return buildNumberRegexCondition(property, buildWildcardRegexPattern(value), true);
      }
      return { op: { $not: buildRegexOp(buildWildcardRegexPattern(value)) } };
    }
    if (isNumberField && !isNumeric(value)) return NO_MATCH_CONDITION;
    return { op: { $ne: coerceScalar(value) } };
  }

  if (isNumberField && !isNumeric(value)) return NO_MATCH_CONDITION;

  // For dates, < and >= align to the start of the day, > and <= to the end.
  const boundary = name === 'gt' || name === 'lte' ? 'end' : 'start';
  const bound = coerceBoundary(value, isDateField, boundary);
  if (isDateField && !bound) return NO_MATCH_CONDITION;
  return { op: { [`$${name}`]: bound } };
}

function parseLeafComparisonOperators(
  token,
  isDateField,
  isBooleanField,
  isObjectIdField,
  isNumberField,
  property
) {
  const operators = [
    ['<>', 'ne'],
    ['>=', 'gte'],
    ['<=', 'lte'],
    ['>', 'gt'],
    ['<', 'lt'],
    ['=', 'eq'],
  ];
  for (const [symbol, name] of operators) {
    if (token.startsWith(symbol)) {
      return buildComparison(
        name,
        stripIgnoreCase(token.slice(symbol.length).trim()),
        isDateField,
        isBooleanField,
        isObjectIdField,
        isNumberField,
        property
      );
    }
  }
  return null;
}

// Parses a leaf token (range, comparison, or plain value) into a condition.
function parseLeafCondition(
  rawToken,
  isDateField,
  isBooleanField = false,
  isObjectIdField = false,
  isNumberField = false,
  property = ''
) {
  const token = rawToken.trim();
  if (token === '') {
    if (isBooleanField || isObjectIdField || isNumberField) return NO_MATCH_CONDITION;
    return { op: buildRegexOp('^$') };
  }
  if (isEmptyOperand(token)) return emptyCondition('eq');

  // Interval: a..b, ..b, a..
  const rangeIdx = token.indexOf('..');
  if (rangeIdx !== -1) {
    if (isBooleanField || isObjectIdField) return NO_MATCH_CONDITION;
    const lo = stripIgnoreCase(token.slice(0, rangeIdx).trim());
    const hi = stripIgnoreCase(token.slice(rangeIdx + 2).trim());
    if (isNumberField) {
      if ((lo !== '' && !isNumeric(lo)) || (hi !== '' && !isNumeric(hi))) {
        return NO_MATCH_CONDITION;
      }
    }
    if (isDateField) {
      if (lo !== '' && !parseDateOperand(lo, 'start')) return NO_MATCH_CONDITION;
      if (hi !== '' && !parseDateOperand(hi, 'end')) return NO_MATCH_CONDITION;
    }
    const op = {};
    if (lo !== '') op.$gte = coerceBoundary(lo, isDateField, 'start');
    if (hi !== '') op.$lte = coerceBoundary(hi, isDateField, 'end');
    // A bare ".." places no constraint on the field.
    if (Object.keys(op).length === 0) return { query: {} };
    return { op };
  }

  const comparison = parseLeafComparisonOperators(
    token,
    isDateField,
    isBooleanField,
    isObjectIdField,
    isNumberField,
    property
  );
  if (comparison) return comparison;

  return buildEquality(
    stripIgnoreCase(token),
    isDateField,
    isBooleanField,
    isObjectIdField,
    isNumberField,
    property
  );
}

// Converts a condition descriptor into a MongoDB query object for `property`.
function conditionToQuery(cond, property) {
  if (cond.query) return cond.query;
  if (cond.op) return { [property]: cond.op };
  return { [property]: cond.value };
}

function isOnlyRegex(op) {
  return Object.keys(op).every((key) => key === '$regex' || key === '$options');
}

// Combines AND-ed conditions, merging operator objects on the same field where possible.
function combineAnd(children, property) {
  if (children.some((child) => child.query)) {
    return { query: { $and: children.map((child) => conditionToQuery(child, property)) } };
  }
  const merged = {};
  for (const child of children) {
    if (child.value !== undefined) {
      merged.$eq = child.value;
    } else if (child.op) {
      Object.assign(merged, child.op);
    }
  }
  return { op: merged };
}

// Combines OR-ed conditions, collapsing to $in or a single regex when possible.
function combineOr(children, property, isObjectIdField = false) {
  if (children.every((child) => child.value !== undefined)) {
    return { op: { $in: children.map((child) => child.value) } };
  }
  if (!isObjectIdField && children.every((child) => child.op && isOnlyRegex(child.op))) {
    const pattern = children.map((child) => child.op.$regex).join('|');
    const op = { $regex: pattern };
    if (children[0].op.$options) op.$options = children[0].op.$options;
    return { op };
  }
  return { query: { $or: children.map((child) => conditionToQuery(child, property)) } };
}

function buildCondition(
  node,
  property,
  isDateField,
  isBooleanField = false,
  isObjectIdField = false,
  isNumberField = false
) {
  if (node.type === 'leaf') {
    return parseLeafCondition(
      node.token,
      isDateField,
      isBooleanField,
      isObjectIdField,
      isNumberField,
      property
    );
  }
  const children = node.items.map((item) =>
    buildCondition(item, property, isDateField, isBooleanField, isObjectIdField, isNumberField)
  );
  return node.type === 'and'
    ? combineAnd(children, property)
    : combineOr(children, property, isObjectIdField);
}

function idsToCondition(ids) {
  if (!ids || ids.length === 0) return NO_MATCH_CONDITION;
  if (ids.length === 1) return { value: ids[0] };
  return { op: { $in: ids } };
}

function extractObjectIdFromOperand(operand) {
  const { suffix } = parsePrefixedValue(stripIgnoreCase(String(operand).trim()));
  if (isObjectIdString(suffix)) {
    return new mongoose.Types.ObjectId(suffix);
  }
  return null;
}

const REF_OPERATOR_SYMBOLS = {
  eq: '',
  ne: '<>',
  gt: '>',
  gte: '>=',
  lt: '<',
  lte: '<=',
};

function buildRefListExpression(operator, operand) {
  const text = String(operand).trim();
  if (operator === 'eq') return text;
  return `${REF_OPERATOR_SYMBOLS[operator]}${text}`;
}

// Pre-list the referenced model, matching the expression against name and/or _reference via parseFilter.
async function listRefModelIds(refModelEntry, expression) {
  const refModel = refModelEntry.model;
  const orClauses = [];

  if (refModel.schema.path('name')) {
    orClauses.push(await parseFilter('name', expression, refModel));
  }
  if (refModel.schema.path('_reference')) {
    orClauses.push(await parseFilter('_reference', expression, refModel));
  }

  if (orClauses.length === 0) return [];

  const query = orClauses.length === 1 ? orClauses[0] : { $or: orClauses };
  const docs = await refModel.find(query).select('_id').lean();
  return docs.map((doc) => doc._id);
}

function conditionToIds(cond) {
  if (cond.query) return null;
  if (cond.value !== undefined) return [cond.value];
  if (cond.op?.$in) return cond.op.$in;
  return null;
}

function combineOrRefIds(children) {
  const ids = new Set();
  for (const child of children) {
    if (child.query) return child;
    const childIds = conditionToIds(child);
    if (!childIds) return child;
    childIds.forEach((id) => ids.add(String(id)));
  }
  return idsToCondition([...ids].map((id) => new mongoose.Types.ObjectId(id)));
}

function combineAndRefIds(children) {
  let ids = null;
  for (const child of children) {
    if (child.query) return child;
    const childIds = conditionToIds(child);
    if (!childIds || childIds.length === 0) return NO_MATCH_CONDITION;
    const set = new Set(childIds.map(String));
    if (ids === null) {
      ids = set;
    } else {
      ids = new Set([...ids].filter((id) => set.has(id)));
    }
    if (ids.size === 0) return NO_MATCH_CONDITION;
  }
  return idsToCondition([...ids].map((id) => new mongoose.Types.ObjectId(id)));
}

async function resolveRefOperand(operand, refModelEntry, operator = 'eq') {
  const lookupOperand = String(operand).trim();
  if (isEmptyOperand(lookupOperand)) {
    if (operator === 'eq' || operator === 'ne') return emptyCondition(operator);
    return NO_MATCH_CONDITION;
  }
  const { prefix, suffix, hadPrefix } = parsePrefixedValue(lookupOperand);
  if (hadPrefix) {
    const prefixEntry = getRefModelEntryFromPrefix(prefix);
    if (prefixEntry) refModelEntry = prefixEntry;
  }

  if (!refModelEntry?.model) return NO_MATCH_CONDITION;

  const objectId = extractObjectIdFromOperand(lookupOperand);

  if (objectId) {
    if (operator === 'ne') return { op: { $ne: objectId } };
    if (operator === 'eq') return { value: objectId };
    return NO_MATCH_CONDITION;
  }

  const expressionOperand = hadPrefix ? suffix : lookupOperand;

  if (operator === 'ne') {
    const ids = await listRefModelIds(refModelEntry, expressionOperand);
    if (ids.length === 0) return { query: {} };
    return { op: { $nin: ids } };
  }

  const expression = buildRefListExpression(operator, expressionOperand);
  const ids = await listRefModelIds(refModelEntry, expression);
  return idsToCondition(ids);
}

async function resolveRefLeaf(rawToken, fallbackRefName) {
  const token = rawToken.trim();
  if (token === '') return NO_MATCH_CONDITION;
  if (isEmptyOperand(token)) return emptyCondition('eq');

  const refModelEntry = getRefModelEntryForToken(token, fallbackRefName);
  const lookupToken = stripPrefixFromOperand(token);

  if (!refModelEntry?.model) {
    const objectId = extractObjectIdFromOperand(lookupToken);
    if (objectId) return { value: objectId };
    return NO_MATCH_CONDITION;
  }

  const rangeIdx = lookupToken.indexOf('..');
  if (rangeIdx !== -1) {
    const ids = await listRefModelIds(refModelEntry, lookupToken);
    return idsToCondition(ids);
  }

  const operators = [
    ['<>', 'ne'],
    ['>=', 'gte'],
    ['<=', 'lte'],
    ['>', 'gt'],
    ['<', 'lt'],
    ['=', 'eq'],
  ];
  for (const [symbol, name] of operators) {
    if (lookupToken.startsWith(symbol)) {
      return resolveRefOperand(lookupToken.slice(symbol.length), refModelEntry, name);
    }
  }
  const objectId = extractObjectIdFromOperand(lookupToken);
  if (objectId) return { value: objectId };

  const ids = await listRefModelIds(refModelEntry, lookupToken);
  return idsToCondition(ids);
}

async function resolveRefCondition(node, fallbackRefName) {
  if (node.type === 'leaf') {
    return resolveRefLeaf(node.token, fallbackRefName);
  }

  const children = await Promise.all(
    node.items.map((item) => resolveRefCondition(item, fallbackRefName))
  );

  return node.type === 'and' ? combineAndRefIds(children) : combineOrRefIds(children);
}

async function resolveObjectRefFilter(filterProperty, expression, fallbackRefName) {
  const tree = parseExpression(expression);
  const condition = await resolveRefCondition(tree, fallbackRefName);
  return conditionToQuery(condition, filterProperty);
}

async function parseFilter(property, value, model = null) {
  // Normalize state filter to state.type for schemas with state: { type }
  if (property === 'state') {
    property = 'state.type';
  }

  const fieldKind = getFilterFieldKind(property, model);
  const filterProperty = fieldKind.property;
  const isBoolField = isBooleanField(property, model);

  if (value?._id !== undefined && value?._id !== null) {
    return { [filterProperty]: new mongoose.Types.ObjectId(value._id) };
  }

  // Non-string values pass through; boolean fields only accept actual booleans.
  if (typeof value !== 'string') {
    if (isBoolField && typeof value !== 'boolean') {
      return { [filterProperty]: { $in: [] } };
    }
    return { [filterProperty]: value };
  }

  const trimmed = value.trim();

  if ((fieldKind.kind === 'objectId' || fieldKind.kind === 'objectRef') && property != '_id') {
    const refName = fieldKind.ref ?? getSchemaRefName(property, model);
    return resolveObjectRefFilter(filterProperty, trimmed, refName);
  }

  let expression = trimmed;
  if (fieldKind.kind === 'default') {
    expression = stripModelPrefixFromExpression(expression);
  }

  const isDateField = isDateTimeField(property, model);
  const isObjectIdField = fieldKind.kind === 'objectId';
  const isNumField = isNumberField(filterProperty, model);
  const tree = parseExpression(expression);
  if (isDateField && !isValidDateFilterNode(tree)) {
    return {};
  }
  const condition = buildCondition(
    tree,
    filterProperty,
    isDateField,
    isBoolField,
    isObjectIdField,
    isNumField
  );
  return conditionToQuery(condition, filterProperty);
}

function convertToCamelCase(obj) {
  const result = {};

  for (const key in obj) {
    if (Object.prototype.hasOwnProperty.call(obj, key)) {
      const value = obj[key];

      // Convert the key to camelCase
      let camelKey = key
        // First handle special cases with spaces, brackets and other characters
        .replace(/\s*\[.*?\]\s*/g, '') // Remove brackets and their contents
        .replace(/\s+/g, ' ') // Normalize spaces
        .trim()
        // Split by common separators (space, underscore, hyphen)
        .split(/[\s_-]/)
        // Convert to camelCase
        .map((word, index) => {
          // Remove any non-alphanumeric characters
          word = word.replace(/[^a-zA-Z0-9]/g, '');

          // Lowercase first word, uppercase others
          return index === 0
            ? word.toLowerCase()
            : word.charAt(0).toUpperCase() + word.slice(1).toLowerCase();
        })
        .join('');

      // Handle values that are objects recursively
      if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
        result[camelKey] = convertToCamelCase(value);
      } else {
        result[camelKey] = value;
      }
    }
  }

  return result;
}

function extractGCodeConfigBlock(fileContent, useCamelCase = true) {
  const configObject = {};

  // Extract header information
  const headerBlockRegex = /; HEADER_BLOCK_START([\s\S]*?)(?:; HEADER_BLOCK_END|$)/;
  const headerBlockMatch = fileContent.match(headerBlockRegex);
  if (headerBlockMatch && headerBlockMatch[1]) {
    const headerLines = headerBlockMatch[1].split('\n');
    headerLines.forEach((line) => {
      const keyValueRegex = /^\s*;\s*([^:]+?):\s*(.*?)\s*$/;
      const simpleValueRegex = /^\s*;\s*(.*?)\s*$/;

      // Try key-value format first
      let match = line.match(keyValueRegex);
      if (match) {
        const key = match[1].trim();
        let value = match[2].trim();

        // Try to convert value to appropriate type
        if (!isNaN(value) && value !== '') {
          value = Number(value);
        }
        configObject[key] = value;
      } else {
        // Try the simple format like "; generated by OrcaSlicer 2.1.1 on 2025-04-28 at 13:30:11"
        match = line.match(simpleValueRegex);
        if (match && match[1] && !match[1].includes('HEADER_BLOCK')) {
          const text = match[1].trim();

          // Extract slicer info
          const slicerMatch = text.match(/generated by (.*?) on (.*?) at (.*?)$/);
          if (slicerMatch) {
            configObject['slicer'] = slicerMatch[1].trim();
            configObject['date'] = slicerMatch[2].trim();
            configObject['time'] = slicerMatch[3].trim();
          } else {
            // Just add as a general header entry if it doesn't match any specific pattern
            const key = `header_${Object.keys(configObject).length}`;
            configObject[key] = text;
          }
        }
      }
    });
  }

  // Extract thumbnail data
  const thumbnailBlockRegex = /; THUMBNAIL_BLOCK_START([\s\S]*?)(?:; THUMBNAIL_BLOCK_END|$)/;
  const thumbnailBlockMatch = fileContent.match(thumbnailBlockRegex);
  if (thumbnailBlockMatch && thumbnailBlockMatch[1]) {
    const thumbnailLines = thumbnailBlockMatch[1].split('\n');
    let base64Data = '';
    let thumbnailInfo = {};

    thumbnailLines.forEach((line) => {
      // Extract thumbnail dimensions and size from the line "thumbnail begin 640x640 27540"
      const thumbnailHeaderRegex = /^\s*;\s*thumbnail begin (\d+)x(\d+) (\d+)/;
      const match = line.match(thumbnailHeaderRegex);

      if (match) {
        thumbnailInfo.width = parseInt(match[1], 10);
        thumbnailInfo.height = parseInt(match[2], 10);
        thumbnailInfo.size = parseInt(match[3], 10);
      } else if (line.trim().startsWith('; ') && !line.includes('THUMBNAIL_BLOCK')) {
        // Collect base64 data (remove the leading semicolon and space and thumbnail end)
        const dataLine = line.trim().substring(2);
        if (dataLine && dataLine != 'thumbnail end') {
          base64Data += dataLine;
        }
      }
    });

    // Add thumbnail data to config object
    if (base64Data) {
      configObject.thumbnail = {
        data: base64Data,
        ...thumbnailInfo,
      };
    }
  }

  // Extract CONFIG_BLOCK
  const configBlockRegex = /; CONFIG_BLOCK_START([\s\S]*?)(?:; CONFIG_BLOCK_END|$)/;
  const configBlockMatch = fileContent.match(configBlockRegex);
  if (configBlockMatch && configBlockMatch[1]) {
    // Extract each config line
    const configLines = configBlockMatch[1].split('\n');
    // Process each line
    configLines.forEach((line) => {
      // Check if the line starts with a semicolon and has an equals sign
      const configLineRegex = /^\s*;\s*([^=]+?)\s*=\s*(.*?)\s*$/;
      const match = line.match(configLineRegex);
      if (match) {
        const key = match[1].trim();
        let value = match[2].trim();
        // Try to convert value to appropriate type
        if (value === 'true' || value === 'false') {
          value = value === 'true';
        } else if (!isNaN(value) && value !== '') {
          // Check if it's a number (but not a percentage)
          if (!value.includes('%')) {
            value = Number(value);
          }
        }
        configObject[key] = value;
      }
    });
  }

  // Extract additional variables that appear after EXECUTABLE_BLOCK_END
  const additionalVarsRegex =
    /; EXECUTABLE_BLOCK_(?:START|END)([\s\S]*?)(?:; CONFIG_BLOCK_START|$)/i;
  const additionalVarsMatch = fileContent.match(additionalVarsRegex);
  if (additionalVarsMatch && additionalVarsMatch[1]) {
    const additionalLines = additionalVarsMatch[1].split('\n');
    additionalLines.forEach((line) => {
      // Match both standard format and the special case for "total filament cost"
      const varRegex =
        /^\s*;\s*((?:filament used|filament cost|total filament used|total filament cost|total layers count|estimated printing time)[^=]*?)\s*=\s*(.*?)\s*$/;
      const match = line.match(varRegex);
      if (match) {
        const key = match[1].replace(/\[([^\]]+)\]/g, '$1').trim();
        let value = match[2].trim();
        // Clean up values - remove units in brackets and handle special cases
        if (key.includes('filament used')) {
          // Extract just the numeric value, ignoring units in brackets
          const numMatch = value.match(/(\d+\.\d+)/);
          if (numMatch) {
            value = parseFloat(numMatch[1]);
          }
        } else if (key.includes('filament cost')) {
          // Extract just the numeric value
          const numMatch = value.match(/(\d+\.\d+)/);
          if (numMatch) {
            value = parseFloat(numMatch[1]);
          }
        } else if (key.includes('total layers count')) {
          value = parseInt(value, 10);
        } else if (key.includes('estimated printing time')) {
          // Keep as string but trim any additional whitespace
          value = value.trim();
        }
        configObject[key] = value;
      }
    });
  }

  // Also extract extrusion width settings
  const extrusionWidthRegex = /;\s*(.*?)\s*extrusion width\s*=\s*(.*?)mm/g;
  let extrusionMatch;
  while ((extrusionMatch = extrusionWidthRegex.exec(fileContent)) !== null) {
    const settingName = extrusionMatch[1].trim();
    const settingValue = parseFloat(extrusionMatch[2].trim());
    configObject[`${settingName} extrusion width`] = settingValue;
  }

  // Extract additional parameters after CONFIG_BLOCK_END if they exist
  const postConfigParams = /; CONFIG_BLOCK_END\s*\n([\s\S]*?)$/;
  const postConfigMatch = fileContent.match(postConfigParams);
  if (postConfigMatch && postConfigMatch[1]) {
    const postConfigLines = postConfigMatch[1].split('\n');
    postConfigLines.forEach((line) => {
      // Match lines with format "; parameter_name = value"
      const paramRegex = /^\s*;\s*([^=]+?)\s*=\s*(.*?)\s*$/;
      const match = line.match(paramRegex);
      if (match) {
        const key = match[1].trim();
        let value = match[2].trim();

        // Try to convert value to appropriate type
        if (value === 'true' || value === 'false') {
          value = value === 'true';
        } else if (!isNaN(value) && value !== '') {
          // Check if it's a number (but not a percentage)
          if (!value.includes('%')) {
            value = Number(value);
          }
        }

        // Add to config object if not already present
        if (!configObject[key]) {
          configObject[key] = value;
        }
      }
    });
  }

  // Apply camelCase conversion if requested
  return useCamelCase ? convertToCamelCase(configObject) : configObject;
}

const AUDIT_IGNORED_KEYS = ['createdAt', 'updatedAt', '_id'];

function isDiffableObject(value) {
  return value && typeof value === 'object' && !Array.isArray(value) && value !== null;
}

function isNumericValue(value) {
  return (
    typeof value === 'number' ||
    (value !== null && value !== undefined && !isNaN(Number(value)) && value !== '')
  );
}

function normalizeDiffValue(value) {
  return isNumericValue(value) ? Number(value) : value;
}

function valuesDiffer(oldVal, newVal) {
  return diffJson(
    { value: normalizeDiffValue(oldVal) },
    { value: normalizeDiffValue(newVal) }
  ).some((part) => part.added || part.removed);
}

function getChangedValues(oldObj, newObj, old = false) {
  const changes = {};

  const combinedObj = { ...oldObj, ...newObj };

  // Check all keys in the new object
  for (const key in combinedObj) {
    // Skip if the key is _id or timestamps
    if (AUDIT_IGNORED_KEYS.includes(key)) continue;

    const oldVal = oldObj ? oldObj[key] : undefined;
    const newVal = newObj ? newObj[key] : undefined;

    // If both values are objects (but not arrays or null), recurse
    if (isDiffableObject(oldVal) && isDiffableObject(newVal)) {
      if (oldVal?._id || newVal?._id) {
        if (valuesDiffer(oldVal?._id, newVal?._id)) {
          changes[key] = old ? oldVal : newVal;
        }
      } else {
        const nestedChanges = getChangedValues(oldVal, newVal, old);
        if (Object.keys(nestedChanges).length > 0) {
          changes[key] = nestedChanges;
        }
      }
    } else {
      if (valuesDiffer(oldVal, newVal)) {
        // If the old value is different from the new value, include it
        changes[key] = old ? oldVal : newVal;
      }
    }
  }

  return changes;
}

const AUDIT_EXCLUDED_MODELS = ['notification', 'userNotifier', 'objectView', 'marketplaceEvent'];
const SENSITIVE_KEYS = ['secret', 'password'];

const DISTRIBUTE_KEYS = {
  _id: true,
  _reference: true,
  name: true,
  type: true,
  tags: true,
  state: true,
  createdAt: true,
  updatedAt: true,
};

function omitSensitive(obj) {
  if (obj == null || typeof obj !== 'object') return obj;
  if (Array.isArray(obj)) return obj.map(omitSensitive);
  const result = {};
  for (const [key, value] of Object.entries(obj)) {
    if (SENSITIVE_KEYS.includes(key)) continue;
    result[key] = omitSensitive(value);
  }
  return result;
}

async function newAuditLog(newValue, parentId, parentType, user) {
  if (AUDIT_EXCLUDED_MODELS.includes(parentType)) return;

  // Filter out createdAt, updatedAt, and sensitive fields from newValue
  const filteredNewValue = omitSensitive({ ...newValue });
  delete filteredNewValue.createdAt;
  delete filteredNewValue.updatedAt;
  const auditLog = new auditLogModel({
    changes: {
      new: filteredNewValue,
    },
    parent: parentId,
    parentType,
    ...resolveAuditOwner(user),
    operation: 'new',
  });

  await auditLog.save();

  await distributeNew(auditLog, 'auditLog');
}

async function editAuditLog(oldValue, newValue, parentId, parentType, user) {
  if (AUDIT_EXCLUDED_MODELS.includes(parentType)) return;

  // Get only the changed values
  const changedOldValues = omitSensitive(getChangedValues(oldValue, newValue, true));
  const changedNewValues = omitSensitive(getChangedValues(oldValue, newValue, false));

  // If no values changed, don't create an audit log
  if (Object.keys(changedOldValues).length === 0 || Object.keys(changedNewValues).length === 0) {
    return;
  }

  const auditLog = new auditLogModel({
    changes: {
      old: changedOldValues,
      new: changedNewValues,
    },
    parent: parentId,
    parentType,
    ...resolveAuditOwner(user),
    operation: 'edit',
  });

  await auditLog.save();

  await distributeNew(auditLog, 'auditLog');
}

async function editNotification(oldValue, newValue, parentId, parentType, user) {
  const model = getModelByName(parentType);
  const objectName = oldValue?.name ?? newValue?.name ?? model?.label ?? parentType;
  const changedOldValues = omitSensitive(getChangedValues(oldValue, newValue, true));
  const changedNewValues = omitSensitive(getChangedValues(oldValue, newValue, false));

  if (Object.keys(changedOldValues).length === 0 || Object.keys(changedNewValues).length === 0) {
    return;
  }

  await notfiyObjectUserNotifiers(
    parentId,
    parentType,
    `${objectName} edited by ${actorDisplayName(user)}`,
    `The ${parentType} ${parentId} has been updated.`,
    'editObject',
    {
      old: changedOldValues,
      new: changedNewValues,
      objectType: parentType,
      object: { _id: String(parentId ?? '') },
      user: {
        _id: String(user?._id ?? ''),
        firstName: user?.firstName ?? user?.name ?? '',
        lastName: user?.lastName ?? '',
      },
    }
  );
}

async function deleteAuditLog(deleteValue, parentId, parentType, user) {
  if (AUDIT_EXCLUDED_MODELS.includes(parentType)) return;

  const auditLog = new auditLogModel({
    changes: {
      old: omitSensitive(deleteValue),
    },
    parent: parentId,
    parentType,
    ...resolveAuditOwner(user),
    operation: 'delete',
  });

  await auditLog.save();

  await distributeNew(auditLog, 'auditLog');
}

async function deleteNotification(object, parentId, parentType, user) {
  const model = getModelByName(parentType);
  const objectName = object?.name || model?.label || parentType;
  await notfiyObjectUserNotifiers(
    parentId,
    parentType,
    `${objectName} deleted by ${actorDisplayName(user)}`,
    `The ${parentType} ${parentId} has been deleted.`,
    'deleteObject',
    {
      object: omitSensitive(object),
      objectType: parentType,
      object: { _id: parentId },
      user: {
        _id: user?._id,
        firstName: user?.firstName ?? user?.name ?? '',
        lastName: user?.lastName ?? '',
      },
    }
  );
}

async function newNoteNotification(note, user) {
  const model = getModelByName(note.parentType);
  const objectName = model?.label ?? note.parentType;
  await notfiyObjectUserNotifiers(
    note.parent,
    note.parentType,
    `New note added to ${objectName.toLowerCase()} by ${actorDisplayName(user)}`,
    `A new note has been created.`,
    'newNote',
    {
      objectType: note.parentType,
      object: { _id: String(note.parent ?? '') },
      note: note,
      user: {
        _id: String(user?._id ?? ''),
        firstName: user?.firstName,
        lastName: user?.lastName,
      },
    }
  );
}

async function subscribeAuditLog(parentId, parentType, user) {
  const auditLog = new auditLogModel({
    parent: parentId,
    parentType,
    ...resolveAuditOwner(user),
    operation: 'subscribe',
  });
  await auditLog.save();
  await distributeNew(auditLog, 'auditLog');
  return auditLog;
}

async function unsubscribeAuditLog(parentId, parentType, user) {
  const auditLog = new auditLogModel({
    parent: parentId,
    parentType,
    ...resolveAuditOwner(user),
    operation: 'unsubscribe',
  });
  await auditLog.save();
  await distributeNew(auditLog, 'auditLog');
  return auditLog;
}

async function getAuditLogs(idOrIds) {
  if (Array.isArray(idOrIds)) {
    return auditLogModel.find({ parent: { $in: idOrIds } }).populate('owner');
  } else {
    return auditLogModel.find({ parent: idOrIds }).populate('owner');
  }
}

function filterDistributeKeys(value, keys = DISTRIBUTE_KEYS) {
  if (value == null) return value;

  const obj = value?.toObject ? value.toObject() : value;
  if (typeof obj !== 'object' || Array.isArray(obj)) return obj;

  const hasKeys = keys && Object.keys(keys).length > 0;
  if (!hasKeys) return { ...obj };

  const result = {};
  for (const [key, val] of Object.entries(obj)) {
    if (keys[key] === true) {
      result[key] = val;
    }
  }
  return result;
}

async function distributeUpdate(value, id, type) {
  await natsServer.publish(`${type}s.${id}.object`, { ...value, _id: id });
}

async function distributeStats(value, type) {
  await natsServer.publish(`${type}s.stats`, filterDistributeKeys(value));
}

async function distributeNew(value, type) {
  await natsServer.publish(`${type}s.new`, value);
}

async function distributeDelete(value, type) {
  await natsServer.publish(`${type}s.delete`, value);
}

function getReferenceId(value) {
  if (value instanceof mongoose.Types.ObjectId) {
    return value;
  }
  if (value && typeof value === 'object' && value._id) {
    return getReferenceId(value._id);
  }
  return value;
}

function getReferenceIdString(value) {
  const id = getReferenceId(value);
  return id == null ? null : id.toString();
}

async function distributeChildUpdate(oldValue, newValue, id, model) {
  const oldPopulatedObjects = populateObjects(oldValue, model) || [];
  const oldPopulatedObjectIds = oldPopulatedObjects.map((populate) =>
    getReferenceIdString(populate._id)
  );
  const newPopulatedObjects = populateObjects(newValue, model) || [];
  const newPopulatedObjectIds = newPopulatedObjects.map((populate) =>
    getReferenceIdString(populate._id)
  );

  for (const populated of oldPopulatedObjects) {
    const populatedId = getReferenceIdString(populated._id);
    if (!populatedId) continue;
    if (!newPopulatedObjectIds.includes(populatedId)) {
      logger.debug(
        `Distributing child update for ${populated.ref}s.${populatedId}.events.childUpdate`
      );
      await natsServer.publish(`${populated.ref}s.${populatedId}.events.childUpdate`, {
        type: 'childUpdate',
        data: { parentId: id, parentType: model.modelName },
      });
    }
  }
  for (const populated of newPopulatedObjects) {
    const populatedId = getReferenceIdString(populated._id);
    if (!populatedId) continue;
    if (!oldPopulatedObjectIds.includes(populatedId)) {
      logger.debug(
        `Distributing child update for ${populated.ref}s.${populatedId}.events.childUpdate`
      );
      await natsServer.publish(`${populated.ref}s.${populatedId}.events.childUpdate`, {
        type: 'childUpdate',
        data: { parentId: id, parentType: model.modelName },
      });
    }
  }
}

async function distributeChildDelete(value, id, model) {
  const populatedObjects = populateObjects(value, model) || [];
  for (const populated of populatedObjects) {
    const populatedId = getReferenceIdString(populated._id);
    if (!populatedId) continue;
    logger.debug(
      `Distributing child delete for ${populated.ref}s.${populatedId}.events.childDelete`
    );
    await natsServer.publish(
      `${populated.ref}s.${populatedId}.events.childDelete`,
      filterDistributeKeys({
        type: 'childDelete',
        data: { parentId: id, parentType: model.modelName },
      })
    );
  }
}

async function distributeChildNew(value, id, model) {
  const populatedObjects = populateObjects(value, model) || [];
  for (const populated of populatedObjects) {
    const populatedId = getReferenceIdString(populated._id);
    if (!populatedId) continue;
    logger.debug(`Distributing child new for ${populated.ref}s.${populatedId}.events.childNew`);
    await natsServer.publish(
      `${populated.ref}s.${populatedId}.events.childNew`,
      filterDistributeKeys({
        type: 'childNew',
        data: { parentId: id, parentType: model.modelName },
      })
    );
  }
}

async function notfiyObjectUserNotifiers(id, objectType, title, message, type = 'info', metadata) {
  const userNotifiers = await userNotifierModel.find({ object: id, objectType: objectType });
  for (const userNotifier of userNotifiers) {
    await createNotification(userNotifier.user._id, title, message, type, metadata);
    if (userNotifier.email == true) {
      await sendEmailNotification(userNotifier.user, title, message, type, metadata);
    }
  }
}

async function createNotification(user, title, message, type = 'info', metadata) {
  const notification = new notificationModel({
    user,
    title,
    message,
    type,
    metadata: omitSensitive(metadata ?? {}),
  });
  await notification.save();
  const value = notification.toJSON ? notification.toJSON() : notification;
  await natsServer.publish(`notifications.${user._id}`, value);
  return notification;
}

async function createDocumentJobUserNotifier(documentJobId, user) {
  const userId = user?._id ?? user;
  if (!userId) return null;

  const existing = await userNotifierModel.findOne({
    user: userId,
    object: documentJobId,
    objectType: 'documentJob',
  });
  if (existing) return existing;

  const userNotifier = await userNotifierModel.create({
    user: userId,
    object: documentJobId,
    objectType: 'documentJob',
    email: false,
  });

  await distributeNew(userNotifier, 'userNotifier');
  return userNotifier;
}

let mailWorker = null;

function getMailWorker() {
  if (!mailWorker) {
    const workerPath = path.join(__dirname, 'mailworker.js');
    mailWorker = new Worker(workerPath);
    mailWorker.on('error', (err) => logger.error('MailWorker error:', err));
    mailWorker.on('exit', (code) => {
      if (code !== 0) logger.warn('MailWorker exited with code', code);
      mailWorker = null;
    });
  }
  return mailWorker;
}

/**
 * Sends an email notification asynchronously via a worker thread.
 * Renders a React template on the urlClient, captures HTML with Puppeteer, and emails the user.
 * Accepts the same input as createNotification: user, title, message, type, metadata.
 * Returns immediately; does not wait for the email to be sent.
 * @param {ObjectId|Object} user - User ID or user object (must have email)
 * @param {string} title - Notification title
 * @param {string} message - Notification message
 * @param {string} type - Notification type (info, editObject, deleteObject, error, success)
 * @param {Object} metadata - Optional metadata object
 */
async function sendEmailNotification(user, title, message, type = 'info', metadata) {
  let userDoc = user;
  if (user && (mongoose.Types.ObjectId.isValid(user) || user._id)) {
    const userId = user._id || user;
    userDoc = await userModel.findById(userId).lean();
  }
  if (!userDoc?.email) {
    logger.warn('sendEmailNotification: no email for user', user);
    return null;
  }

  const smtpConfig = config.smtp;
  if (!smtpConfig?.host) {
    logger.warn('sendEmailNotification: SMTP not configured, skipping email');
    return null;
  }

  const urlClient = config.app?.urlClient || 'http://localhost:3000';
  const authCode = createEmailRenderAuthCode(userDoc);

  const payload = {
    email: userDoc.email,
    title,
    message,
    type,
    metadata: omitSensitive(metadata || {}),
    createdAt: new Date(),
    updatedAt: new Date(),
    authCode,
    smtpConfig: {
      host: smtpConfig.host,
      port: smtpConfig.port || 587,
      secure: smtpConfig.secure || false,
      auth: smtpConfig.auth?.user ? smtpConfig.auth : undefined,
      from: smtpConfig.from || 'FarmControl <noreply@farmcontrol.app>',
    },
    urlClient,
  };

  try {
    getMailWorker().postMessage(payload);
  } catch (err) {
    logger.error('sendEmailNotification: failed to post to worker', err.message);
  }
  return null;
}

function flatternObjectIds(object) {
  if (!object || typeof object !== 'object') {
    return object;
  }

  const result = {};

  for (const [key, value] of Object.entries(object)) {
    if (value && typeof value === 'object' && value._id) {
      // If the value is an object with _id, convert to just the _id
      result[key] = value._id;
    } else {
      // Keep primitive values as is
      result[key] = value;
    }
  }

  return result;
}

function isByte(value) {
  const num = Number(value);
  return Number.isInteger(num) && num >= 0 && num <= 255;
}

function isTwelveByteBuffer(val) {
  if (!val) return false;
  if (Buffer.isBuffer(val) || val instanceof Uint8Array) {
    return val.length === 12;
  }
  if (typeof val !== 'object' || Array.isArray(val)) return false;
  for (let i = 0; i < 12; i++) {
    if (!isByte(val[i])) return false;
  }
  const indexKeys = Object.keys(val).filter((key) => /^\d+$/.test(key));
  return indexKeys.length === 12;
}

function objectIdToString(val) {
  if (val == null) return val;
  if (typeof val === 'string') return val;
  if (typeof val.toHexString === 'function') return val.toHexString();
  if (val instanceof mongoose.Types.ObjectId) return val.toString();
  if (isTwelveByteBuffer(val)) {
    const bytes =
      Buffer.isBuffer(val) || val instanceof Uint8Array
        ? val
        : Array.from({ length: 12 }, (_, i) => Number(val[i]));
    return Buffer.from(bytes).toString('hex');
  }
  if (val && typeof val === 'object' && isTwelveByteBuffer(val.buffer || val.id)) {
    return objectIdToString(val.buffer || val.id);
  }
  return String(val);
}

function expandObjectIds(input) {
  const excludedFields = ['createdAt', 'updatedAt', 'name', '_id'];

  function isObjectId(val) {
    if (val == null || typeof val === 'boolean' || typeof val === 'number') return false;
    if (val instanceof Date) return false;
    if (val instanceof mongoose.Types.ObjectId) return true;
    if (typeof val === 'string' && /^[a-fA-F\d]{24}$/.test(val)) return true;
    if (typeof val === 'object' && (val._bsontype === 'ObjectId' || val._bsontype === 'ObjectID')) {
      return true;
    }
    if (typeof val.toHexString === 'function' && isTwelveByteBuffer(val.id || val.buffer)) {
      return true;
    }
    if (isTwelveByteBuffer(val)) return true;
    if (val && typeof val === 'object' && !Array.isArray(val)) {
      const keys = Object.keys(val);
      if (
        keys.length > 0 &&
        keys.every((key) => key === 'buffer' || key === 'id') &&
        isTwelveByteBuffer(val.buffer || val.id)
      ) {
        return true;
      }
    }
    return false;
  }

  function expand(value) {
    if (Array.isArray(value)) {
      return value.map(expand);
    } else if (value instanceof Date) {
      return value;
    } else if (isObjectId(value)) {
      return { _id: objectIdToString(value) };
    } else if (value && typeof value === 'object') {
      var result = {};
      for (const [key, val] of Object.entries(value)) {
        if (excludedFields.includes(key)) {
          result[key] = val == null ? val : key === '_id' ? objectIdToString(val) : val.toString();
        } else if (isObjectId(val)) {
          result[key] = { _id: objectIdToString(val) };
        } else if (Array.isArray(val)) {
          result[key] = val.map(expand);
        } else if (val instanceof Date) {
          result[key] = val;
        } else if (val && typeof val === 'object') {
          result[key] = expand(val);
        } else {
          result[key] = val;
        }
      }
      return result;
    } else {
      return value;
    }
  }

  return expand(input);
}

// Merges per-field filter clauses, falling back to $and when clauses use
// logical operators or target the same field (so nothing is silently lost).
function mergeFilterClauses(clauses) {
  const valid = clauses.filter((clause) => clause && Object.keys(clause).length > 0);
  if (valid.length === 0) return {};
  if (valid.length === 1) return valid[0];

  const seen = new Set();
  let needsAnd = false;
  for (const clause of valid) {
    for (const key of Object.keys(clause)) {
      if (key === '$or' || key === '$and' || seen.has(key)) {
        needsAnd = true;
      }
      seen.add(key);
    }
  }

  return needsAnd ? { $and: valid } : Object.assign({}, ...valid);
}

// Returns a validated sort field based on allowed sorters
function getSort(sort, allowedSorters, defaultSort = 'createdAt') {
  if (sort && allowedSorters.includes(sort)) {
    return sort;
  }
  return defaultSort;
}

function normalizeFilterQuery(query = {}) {
  const queryClean = { ...query };

  // qs turns `order._id=` into `{ order: { _id } }`. Lift it so it can be parsed
  // as the order object filter instead of a nested query object.
  if (
    queryClean.order &&
    typeof queryClean.order === 'object' &&
    !Array.isArray(queryClean.order)
  ) {
    if (queryClean.order._id !== undefined) {
      queryClean['order._id'] = queryClean.order._id;
    }
    delete queryClean.order;
  }

  return queryClean;
}

// Returns a filter object based on allowed filters and req.query
async function getFilter(query, allowedFilters, parse = true, model = null) {
  const clauses = [];
  const queryClean = normalizeFilterQuery(query);
  for (const key of ['sortProperty', 'sortOrder', 'page', 'limit']) {
    if (key in queryClean) delete queryClean[key];
  }
  if (!parse && queryClean['order._id'] !== undefined && allowedFilters.includes('order')) {
    queryClean.order = queryClean['order._id'];
    delete queryClean['order._id'];
  }
  for (const [key, value] of Object.entries(queryClean)) {
    if (allowedFilters.includes(key)) {
      if (parse) {
        const clause = await parseFilter(key, value, model);
        // Drop date filters that contain anything other than operators and dates.
        if (clause && Object.keys(clause).length > 0) {
          clauses.push(clause);
        }
      } else {
        clauses.push({ [key]: value });
      }
    }
  }
  return mergeFilterClauses(clauses);
}

// Converts a properties argument (string or array) to an array of strings
function convertPropertiesString(properties) {
  if (typeof properties === 'string') {
    return properties.split(',');
  } else if (!Array.isArray(properties)) {
    return [];
  }
  return properties;
}

async function getFileMeta(file) {
  try {
    if (!file) return {};

    const originalName = file.originalname || '';
    const lowerName = originalName.toLowerCase();

    if (lowerName.endsWith('.g') || lowerName.endsWith('.gcode')) {
      const content = file.buffer ? file.buffer.toString('utf8') : '';
      if (!content) return {};
      return extractGCodeConfigBlock(content);
    }

    // Image EXIF metadata
    if (file.mimetype && file.mimetype.startsWith('image/') && file.buffer) {
      try {
        const exif = await exifr.parse(file.buffer);
        return exif || {};
      } catch (_) {
        // Ignore EXIF parse errors and fall through
      }
    }

    return {};
  } catch (_) {
    return {};
  }
}

function getSchemaTypeRef(schemaType) {
  if (!schemaType) return undefined;
  return (
    schemaType.options?.ref ||
    schemaType.caster?.options?.ref ||
    schemaType.embeddedSchemaType?.options?.ref
  );
}

function forEachSchemaPath(schema, callback) {
  if (!schema || typeof callback !== 'function') return;

  schema.eachPath((pathName, schemaType) => {
    callback(pathName, schemaType);
  });

  const subpaths = schema.subpaths || {};
  for (const [pathName, schemaType] of Object.entries(subpaths)) {
    if (pathName.endsWith('.$')) continue;
    callback(pathName, schemaType);
  }
}

function modelHasRef(model, refName) {
  if (!model || !model.schema) {
    return false;
  }

  let hasRef = false;
  forEachSchemaPath(model.schema, (_pathName, schemaType) => {
    if (getSchemaTypeRef(schemaType) === refName) {
      hasRef = true;
    }
  });

  return hasRef;
}

function getFieldsByRef(model, refName) {
  if (!model || !model.schema) {
    return [];
  }

  const fields = [];
  const seen = new Set();
  forEachSchemaPath(model.schema, (pathName, schemaType) => {
    if (getSchemaTypeRef(schemaType) !== refName || seen.has(pathName)) return;
    seen.add(pathName);
    fields.push(pathName);
  });

  return fields;
}

function collectIdsAtPath(object, pathName) {
  if (object == null || !pathName) return [];

  const parts = String(pathName).split('.').filter(Boolean);
  let nodes = [object];
  for (const part of parts) {
    nodes = nodes.flatMap((node) => {
      if (node == null || typeof node !== 'object') return [];
      const next = node[part];
      if (next == null || next === '') return [];
      return Array.isArray(next) ? next : [next];
    });
  }

  return nodes
    .map((value) => {
      if (value == null || value === '') return null;
      if (typeof value === 'object') return value._id != null ? String(value._id) : null;
      return String(value);
    })
    .filter(Boolean);
}

// Build a nested populate specification by walking the schema graph,
// instead of recursing over already-populated documents.
function buildDeepPopulateSpec(object, model, populated = new Set()) {
  // prevent infinite recursion across cyclic model relationships
  if (populated.has(model.modelName)) return [];
  populated.add(model.modelName);

  const schema = model.schema;
  const populateSpec = [];

  schema.eachPath((pathname, schemaType) => {
    const ref = getSchemaTypeRef(schemaType);
    if (!ref) return;

    const refName = typeof ref === 'function' ? ref.call(object) : ref;
    if (!refName) return;

    const refModel = model.db.model(refName);
    const childPopulate = buildDeepPopulateSpec(object, refModel, populated);

    const values = Array.isArray(object[pathname]) ? object[pathname] : [object[pathname]];
    const ids = values.map(getReferenceId).filter(Boolean);

    for (const id of ids) {
      if (childPopulate.length > 0) {
        populateSpec.push({ path: pathname, populate: childPopulate, ref: refName, _id: id });
      } else {
        populateSpec.push({ path: pathname, ref: refName, _id: id });
      }
    }
  });

  return populateSpec;
}

function populateObjects(object, model, populated = new Set()) {
  const populateSpec = buildDeepPopulateSpec(object, model, populated);

  return populateSpec;
}

function jsonToCacheKey(obj) {
  const normalized = canonicalize(obj);
  const hash = crypto.createHash('sha256').update(normalized).digest('hex');
  return hash;
}

export function getQueryToCacheKey({ model, id, populate }) {
  const populateKey = [];
  if (populate) {
    const populateArray = Array.isArray(populate) ? populate : [populate];
    for (const pop of populateArray) {
      if (typeof pop === 'string') {
        populateKey.push(pop);
      } else if (typeof pop === 'object' && pop.path) {
        populateKey.push(pop.path);
      }
    }
  }
  return `${model}:${id?.toString()}-${populateKey.join(',')}`;
}

export {
  parseFilter,
  convertToCamelCase,
  newAuditLog,
  editNotification,
  deleteNotification,
  editAuditLog,
  deleteAuditLog,
  getAuditLogs,
  flatternObjectIds,
  expandObjectIds,
  newNoteNotification,
  filterDistributeKeys,
  DISTRIBUTE_KEYS,
  distributeUpdate,
  distributeStats,
  distributeNew,
  distributeDelete,
  distributeChildUpdate,
  distributeChildDelete,
  distributeChildNew,
  notfiyObjectUserNotifiers,
  createNotification,
  createDocumentJobUserNotifier,
  sendEmailNotification,
  getFilter,
  getSort,
  convertPropertiesString,
  getFileMeta,
  modelHasRef,
  getFieldsByRef,
  collectIdsAtPath,
  jsonToCacheKey,
  subscribeAuditLog,
  unsubscribeAuditLog,
  resolveAuditOwner,
  actorDisplayName,
};