import { mongoose } from 'mongoose';
import { auditLogModel } from './database/schemas/management/auditlog.schema.js';
import { notificationModel } from './database/schemas/misc/notification.schema.js';
import { userNotifierModel } from './database/schemas/misc/usernotifier.schema.js';
import { userModel } from './database/schemas/management/user.schema.js';
import exifr from 'exifr';
import { natsServer } from './database/nats.js';
import log4js from 'log4js';
import config from './config.js';
import crypto from 'crypto';
import canonicalize from 'canonical-json';
import { getModelByName } from './services/misc/model.js';
import { createEmailRenderAuthCode } from './services/misc/emailRenderAuth.js';
import { Worker } from 'worker_threads';
import path from 'path';
import { fileURLToPath } from 'url';
import { diffJson } from 'diff';

const __dirname = path.dirname(fileURLToPath(import.meta.url));

const logger = log4js.getLogger('Utils');
logger.level = config.server.logLevel;

// ---------------------------------------------------------------------------
// Filter expression parsing (Microsoft Dynamics NAV / Business Central style)
//
// Supported syntax (all compiled down to MongoDB query operators):
//   =        equal to                  377            -> { field: 377 }
//   <>       not equal to              <>0            -> { field: { $ne: 0 } }
//   >        greater than              >1200          -> { field: { $gt: 1200 } }
//   >=       greater than or equal     >=1200         -> { field: { $gte: 1200 } }
//   <        less than                 <1200          -> { field: { $lt: 1200 } }
//   <=       less than or equal        <=1200         -> { field: { $lte: 1200 } }
//   ..       interval                  1100..2100     -> { field: { $gte: 1100, $lte: 2100 } }
//            open-ended intervals      ..2500 / 23..
//   |        either / or               1200|1300      -> { field: { $in: [1200, 1300] } }
//   &        and                       <2000&>1000    -> { field: { $gt: 1000, $lt: 2000 } }
//   ( )      grouping / precedence     30|(>=10&<=20)
//   *        any number of characters  Co* / *Co / *Co*
//   ?        a single character        Hans?n
//   @        ignore case               @location (text matching is case-insensitive)
//
// Date-typed fields (detected by name, e.g. *At / *Date / *Time) interpret bare
// values as calendar days/datetimes, e.g. "22" => the whole of day 22 of the
// current month/year, "22..23" => start of 22 through end of 23.
//
// Note: BC's space-delimited date shorthand is locale dependent; here numeric
// date operands are read day-first ("D M Y H Min S") with missing components
// filled from the current date and the interval boundary.
// ---------------------------------------------------------------------------

function buildWildcardRegexPattern(input) {
  // Escape all regex special chars except * and ? (which we treat as wildcards)
  const escaped = String(input).replace(/[.+^${}()|[\]\\]/g, '\\$&');
  // * matches any run of characters, ? matches exactly one character
  const withWildcards = escaped.replace(/\*/g, '.*').replace(/\?/g, '.');
  // Anchor so that, without wildcards, this is an exact match
  return `^${withWildcards}$`;
}

function looksLikeDateField(property) {
  const last = String(property).split('.').pop();
  return /[a-z](?:At|Date|Time)$/.test(last) || /^(?:date|time|datetime)$/i.test(last);
}

function isObjectIdString(value) {
  return /^[a-f\d]{24}$/i.test(value);
}

function isNumeric(value) {
  return value.trim() !== '' && !isNaN(value);
}

function startOfDay(date) {
  const d = new Date(date);
  d.setHours(0, 0, 0, 0);
  return d;
}

function endOfDay(date) {
  const d = new Date(date);
  d.setHours(23, 59, 59, 999);
  return d;
}

function normalizeYear(year) {
  if (year >= 100) return year;
  return year < 70 ? 2000 + year : 1900 + year;
}

// Parses a single date operand to a Date, filling missing parts from the
// current date and the supplied interval boundary ('start' | 'end').
function parseDateOperand(value, boundary = 'start') {
  const text = String(value).trim();
  if (!text) return null;
  const end = boundary === 'end';

  // Values with explicit separators (ISO and similar) are parsed directly.
  if (/[-/T]/.test(text) || /\d:\d/.test(text)) {
    const parsed = new Date(text);
    if (isNaN(parsed.getTime())) return null;
    if (!/[T:]/.test(text)) {
      return end ? endOfDay(parsed) : startOfDay(parsed);
    }
    return parsed;
  }

  // Numeric, day-first form: "D [M] [Y] [H] [Min] [S]".
  const parts = text.split(/\s+/);
  if (!parts.every((part) => /^\d+$/.test(part))) return null;
  const nums = parts.map(Number);
  const now = new Date();
  const day = nums[0];
  const month = nums.length >= 2 ? nums[1] : now.getMonth() + 1;
  const year = nums.length >= 3 ? normalizeYear(nums[2]) : now.getFullYear();
  const hour = nums.length >= 4 ? nums[3] : end ? 23 : 0;
  const minute = nums.length >= 5 ? nums[4] : end ? 59 : 0;
  const second = nums.length >= 6 ? nums[5] : end ? 59 : 0;
  const ms = end ? 999 : 0;
  const date = new Date(year, month - 1, day, hour, minute, second, ms);
  return isNaN(date.getTime()) ? null : date;
}

// Coerces a literal to its strongest matching scalar type.
function coerceScalar(value) {
  const lower = value.toLowerCase();
  if (lower === 'true') return true;
  if (lower === 'false') return false;
  if (isObjectIdString(value)) return new mongoose.Types.ObjectId(value);
  if (isNumeric(value)) return Number(value);
  return value;
}

// Coerces an interval/comparison boundary, preferring a date for date fields.
function coerceBoundary(value, isDateField, boundary) {
  if (isDateField) {
    const date = parseDateOperand(value, boundary);
    if (date) return date;
  }
  return coerceScalar(value);
}

// Splits on a separator while respecting parentheses depth.
function splitTopLevel(str, separator) {
  const parts = [];
  let depth = 0;
  let current = '';
  for (const ch of str) {
    if (ch === '(') depth++;
    else if (ch === ')') depth = Math.max(0, depth - 1);

    if (ch === separator && depth === 0) {
      parts.push(current);
      current = '';
    } else {
      current += ch;
    }
  }
  parts.push(current);
  return parts;
}

// True when a single outer pair of parentheses wraps the whole token.
function isWrappedInParens(str) {
  if (!str.startsWith('(') || !str.endsWith(')')) return false;
  let depth = 0;
  for (let i = 0; i < str.length; i++) {
    if (str[i] === '(') depth++;
    else if (str[i] === ')') {
      depth--;
      if (depth === 0 && i < str.length - 1) return false;
    }
  }
  return depth === 0;
}

// Parses a filter expression into an AST of or / and / leaf nodes.
function parseExpression(str) {
  const orParts = splitTopLevel(str, '|');
  if (orParts.length > 1) {
    return { type: 'or', items: orParts.map(parseExpression) };
  }
  const andParts = splitTopLevel(str, '&');
  if (andParts.length > 1) {
    return { type: 'and', items: andParts.map(parseExpression) };
  }
  const trimmed = str.trim();
  if (isWrappedInParens(trimmed)) {
    return parseExpression(trimmed.slice(1, -1));
  }
  return { type: 'leaf', token: trimmed };
}

// Strips a leading @ (ignore-case marker); text matching is already case-insensitive.
function stripIgnoreCase(str) {
  return str.startsWith('@') ? str.slice(1) : str;
}

// Builds an equality condition (value, date range, or wildcard regex).
function buildEquality(value, isDateField) {
  if (isDateField) {
    const start = parseDateOperand(value, 'start');
    const end = parseDateOperand(value, 'end');
    if (start && end) return { op: { $gte: start, $lte: end } };
  }
  const lower = value.toLowerCase();
  if (lower === 'true') return { value: true };
  if (lower === 'false') return { value: false };
  if (isObjectIdString(value)) return { value: new mongoose.Types.ObjectId(value) };
  if (isNumeric(value)) return { value: Number(value) };
  return { op: { $regex: buildWildcardRegexPattern(value), $options: 'i' } };
}

// Builds a comparison condition for a single operator.
function buildComparison(name, value, isDateField) {
  if (name === 'eq') return buildEquality(value, isDateField);

  if (name === 'ne') {
    if (isDateField) {
      const start = parseDateOperand(value, 'start');
      const end = parseDateOperand(value, 'end');
      if (start && end) return { op: { $not: { $gte: start, $lte: end } } };
    }
    if (/[*?]/.test(value)) {
      return { op: { $not: { $regex: buildWildcardRegexPattern(value), $options: 'i' } } };
    }
    return { op: { $ne: coerceScalar(value) } };
  }

  // For dates, < and >= align to the start of the day, > and <= to the end.
  const boundary = name === 'gt' || name === 'lte' ? 'end' : 'start';
  return { op: { [`$${name}`]: coerceBoundary(value, isDateField, boundary) } };
}

// Parses a leaf token (range, comparison, or plain value) into a condition.
function parseLeafCondition(rawToken, isDateField) {
  const token = rawToken.trim();
  if (token === '') {
    return { op: { $regex: '^$', $options: 'i' } };
  }

  // Interval: a..b, ..b, a..
  const rangeIdx = token.indexOf('..');
  if (rangeIdx !== -1) {
    const lo = stripIgnoreCase(token.slice(0, rangeIdx).trim());
    const hi = stripIgnoreCase(token.slice(rangeIdx + 2).trim());
    const op = {};
    if (lo !== '') op.$gte = coerceBoundary(lo, isDateField, 'start');
    if (hi !== '') op.$lte = coerceBoundary(hi, isDateField, 'end');
    // A bare ".." places no constraint on the field.
    if (Object.keys(op).length === 0) return { query: {} };
    return { op };
  }

  // Comparison operators, longest symbols first.
  const operators = [
    ['<>', 'ne'],
    ['>=', 'gte'],
    ['<=', 'lte'],
    ['>', 'gt'],
    ['<', 'lt'],
    ['=', 'eq'],
  ];
  for (const [symbol, name] of operators) {
    if (token.startsWith(symbol)) {
      return buildComparison(name, stripIgnoreCase(token.slice(symbol.length).trim()), isDateField);
    }
  }

  return buildEquality(stripIgnoreCase(token), isDateField);
}

// Converts a condition descriptor into a MongoDB query object for `property`.
function conditionToQuery(cond, property) {
  if (cond.query) return cond.query;
  if (cond.op) return { [property]: cond.op };
  return { [property]: cond.value };
}

function isOnlyRegex(op) {
  return Object.keys(op).every((key) => key === '$regex' || key === '$options');
}

// Combines AND-ed conditions, merging operator objects on the same field where possible.
function combineAnd(children, property) {
  if (children.some((child) => child.query)) {
    return { query: { $and: children.map((child) => conditionToQuery(child, property)) } };
  }
  const merged = {};
  for (const child of children) {
    if (child.value !== undefined) {
      merged.$eq = child.value;
    } else if (child.op) {
      Object.assign(merged, child.op);
    }
  }
  return { op: merged };
}

// Combines OR-ed conditions, collapsing to $in or a single regex when possible.
function combineOr(children, property) {
  if (children.every((child) => child.value !== undefined)) {
    return { op: { $in: children.map((child) => child.value) } };
  }
  if (children.every((child) => child.op && isOnlyRegex(child.op))) {
    const pattern = children.map((child) => child.op.$regex).join('|');
    return { op: { $regex: pattern, $options: 'i' } };
  }
  return { query: { $or: children.map((child) => conditionToQuery(child, property)) } };
}

function buildCondition(node, property, isDateField) {
  if (node.type === 'leaf') {
    return parseLeafCondition(node.token, isDateField);
  }
  const children = node.items.map((item) => buildCondition(item, property, isDateField));
  return node.type === 'and' ? combineAnd(children, property) : combineOr(children, property);
}

function parseFilter(property, value) {
  // Normalize state filter to state.type for schemas with state: { type }
  if (property === 'state') {
    property = 'state.type';
  }

  if (value?._id !== undefined && value?._id !== null) {
    return { [property]: { _id: new mongoose.Types.ObjectId(value._id) } };
  }

  // Non-string values (actual booleans, numbers, objects, etc.) pass through.
  if (typeof value !== 'string') {
    return { [property]: value };
  }

  let trimmed = value.trim();
  if (trimmed.charAt(3) === ':') {
    const afterColon = value.split(':')[1];
    trimmed = afterColon != null ? afterColon.trim() : '';
  }

  const isDateField = looksLikeDateField(property);
  const tree = parseExpression(trimmed);
  const condition = buildCondition(tree, property, isDateField);
  return conditionToQuery(condition, property);
}

function convertToCamelCase(obj) {
  const result = {};

  for (const key in obj) {
    if (Object.prototype.hasOwnProperty.call(obj, key)) {
      const value = obj[key];

      // Convert the key to camelCase
      let camelKey = key
        // First handle special cases with spaces, brackets and other characters
        .replace(/\s*\[.*?\]\s*/g, '') // Remove brackets and their contents
        .replace(/\s+/g, ' ') // Normalize spaces
        .trim()
        // Split by common separators (space, underscore, hyphen)
        .split(/[\s_-]/)
        // Convert to camelCase
        .map((word, index) => {
          // Remove any non-alphanumeric characters
          word = word.replace(/[^a-zA-Z0-9]/g, '');

          // Lowercase first word, uppercase others
          return index === 0
            ? word.toLowerCase()
            : word.charAt(0).toUpperCase() + word.slice(1).toLowerCase();
        })
        .join('');

      // Handle values that are objects recursively
      if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
        result[camelKey] = convertToCamelCase(value);
      } else {
        result[camelKey] = value;
      }
    }
  }

  return result;
}

function extractGCodeConfigBlock(fileContent, useCamelCase = true) {
  const configObject = {};

  // Extract header information
  const headerBlockRegex = /; HEADER_BLOCK_START([\s\S]*?)(?:; HEADER_BLOCK_END|$)/;
  const headerBlockMatch = fileContent.match(headerBlockRegex);
  if (headerBlockMatch && headerBlockMatch[1]) {
    const headerLines = headerBlockMatch[1].split('\n');
    headerLines.forEach((line) => {
      const keyValueRegex = /^\s*;\s*([^:]+?):\s*(.*?)\s*$/;
      const simpleValueRegex = /^\s*;\s*(.*?)\s*$/;

      // Try key-value format first
      let match = line.match(keyValueRegex);
      if (match) {
        const key = match[1].trim();
        let value = match[2].trim();

        // Try to convert value to appropriate type
        if (!isNaN(value) && value !== '') {
          value = Number(value);
        }
        configObject[key] = value;
      } else {
        // Try the simple format like "; generated by OrcaSlicer 2.1.1 on 2025-04-28 at 13:30:11"
        match = line.match(simpleValueRegex);
        if (match && match[1] && !match[1].includes('HEADER_BLOCK')) {
          const text = match[1].trim();

          // Extract slicer info
          const slicerMatch = text.match(/generated by (.*?) on (.*?) at (.*?)$/);
          if (slicerMatch) {
            configObject['slicer'] = slicerMatch[1].trim();
            configObject['date'] = slicerMatch[2].trim();
            configObject['time'] = slicerMatch[3].trim();
          } else {
            // Just add as a general header entry if it doesn't match any specific pattern
            const key = `header_${Object.keys(configObject).length}`;
            configObject[key] = text;
          }
        }
      }
    });
  }

  // Extract thumbnail data
  const thumbnailBlockRegex = /; THUMBNAIL_BLOCK_START([\s\S]*?)(?:; THUMBNAIL_BLOCK_END|$)/;
  const thumbnailBlockMatch = fileContent.match(thumbnailBlockRegex);
  if (thumbnailBlockMatch && thumbnailBlockMatch[1]) {
    const thumbnailLines = thumbnailBlockMatch[1].split('\n');
    let base64Data = '';
    let thumbnailInfo = {};

    thumbnailLines.forEach((line) => {
      // Extract thumbnail dimensions and size from the line "thumbnail begin 640x640 27540"
      const thumbnailHeaderRegex = /^\s*;\s*thumbnail begin (\d+)x(\d+) (\d+)/;
      const match = line.match(thumbnailHeaderRegex);

      if (match) {
        thumbnailInfo.width = parseInt(match[1], 10);
        thumbnailInfo.height = parseInt(match[2], 10);
        thumbnailInfo.size = parseInt(match[3], 10);
      } else if (line.trim().startsWith('; ') && !line.includes('THUMBNAIL_BLOCK')) {
        // Collect base64 data (remove the leading semicolon and space and thumbnail end)
        const dataLine = line.trim().substring(2);
        if (dataLine && dataLine != 'thumbnail end') {
          base64Data += dataLine;
        }
      }
    });

    // Add thumbnail data to config object
    if (base64Data) {
      configObject.thumbnail = {
        data: base64Data,
        ...thumbnailInfo,
      };
    }
  }

  // Extract CONFIG_BLOCK
  const configBlockRegex = /; CONFIG_BLOCK_START([\s\S]*?)(?:; CONFIG_BLOCK_END|$)/;
  const configBlockMatch = fileContent.match(configBlockRegex);
  if (configBlockMatch && configBlockMatch[1]) {
    // Extract each config line
    const configLines = configBlockMatch[1].split('\n');
    // Process each line
    configLines.forEach((line) => {
      // Check if the line starts with a semicolon and has an equals sign
      const configLineRegex = /^\s*;\s*([^=]+?)\s*=\s*(.*?)\s*$/;
      const match = line.match(configLineRegex);
      if (match) {
        const key = match[1].trim();
        let value = match[2].trim();
        // Try to convert value to appropriate type
        if (value === 'true' || value === 'false') {
          value = value === 'true';
        } else if (!isNaN(value) && value !== '') {
          // Check if it's a number (but not a percentage)
          if (!value.includes('%')) {
            value = Number(value);
          }
        }
        configObject[key] = value;
      }
    });
  }

  // Extract additional variables that appear after EXECUTABLE_BLOCK_END
  const additionalVarsRegex =
    /; EXECUTABLE_BLOCK_(?:START|END)([\s\S]*?)(?:; CONFIG_BLOCK_START|$)/i;
  const additionalVarsMatch = fileContent.match(additionalVarsRegex);
  if (additionalVarsMatch && additionalVarsMatch[1]) {
    const additionalLines = additionalVarsMatch[1].split('\n');
    additionalLines.forEach((line) => {
      // Match both standard format and the special case for "total filament cost"
      const varRegex =
        /^\s*;\s*((?:filament used|filament cost|total filament used|total filament cost|total layers count|estimated printing time)[^=]*?)\s*=\s*(.*?)\s*$/;
      const match = line.match(varRegex);
      if (match) {
        const key = match[1].replace(/\[([^\]]+)\]/g, '$1').trim();
        let value = match[2].trim();
        // Clean up values - remove units in brackets and handle special cases
        if (key.includes('filament used')) {
          // Extract just the numeric value, ignoring units in brackets
          const numMatch = value.match(/(\d+\.\d+)/);
          if (numMatch) {
            value = parseFloat(numMatch[1]);
          }
        } else if (key.includes('filament cost')) {
          // Extract just the numeric value
          const numMatch = value.match(/(\d+\.\d+)/);
          if (numMatch) {
            value = parseFloat(numMatch[1]);
          }
        } else if (key.includes('total layers count')) {
          value = parseInt(value, 10);
        } else if (key.includes('estimated printing time')) {
          // Keep as string but trim any additional whitespace
          value = value.trim();
        }
        configObject[key] = value;
      }
    });
  }

  // Also extract extrusion width settings
  const extrusionWidthRegex = /;\s*(.*?)\s*extrusion width\s*=\s*(.*?)mm/g;
  let extrusionMatch;
  while ((extrusionMatch = extrusionWidthRegex.exec(fileContent)) !== null) {
    const settingName = extrusionMatch[1].trim();
    const settingValue = parseFloat(extrusionMatch[2].trim());
    configObject[`${settingName} extrusion width`] = settingValue;
  }

  // Extract additional parameters after CONFIG_BLOCK_END if they exist
  const postConfigParams = /; CONFIG_BLOCK_END\s*\n([\s\S]*?)$/;
  const postConfigMatch = fileContent.match(postConfigParams);
  if (postConfigMatch && postConfigMatch[1]) {
    const postConfigLines = postConfigMatch[1].split('\n');
    postConfigLines.forEach((line) => {
      // Match lines with format "; parameter_name = value"
      const paramRegex = /^\s*;\s*([^=]+?)\s*=\s*(.*?)\s*$/;
      const match = line.match(paramRegex);
      if (match) {
        const key = match[1].trim();
        let value = match[2].trim();

        // Try to convert value to appropriate type
        if (value === 'true' || value === 'false') {
          value = value === 'true';
        } else if (!isNaN(value) && value !== '') {
          // Check if it's a number (but not a percentage)
          if (!value.includes('%')) {
            value = Number(value);
          }
        }

        // Add to config object if not already present
        if (!configObject[key]) {
          configObject[key] = value;
        }
      }
    });
  }

  // Apply camelCase conversion if requested
  return useCamelCase ? convertToCamelCase(configObject) : configObject;
}

const AUDIT_IGNORED_KEYS = ['createdAt', 'updatedAt', '_id'];

function isDiffableObject(value) {
  return value && typeof value === 'object' && !Array.isArray(value) && value !== null;
}

function isNumericValue(value) {
  return (
    typeof value === 'number' ||
    (value !== null && value !== undefined && !isNaN(Number(value)) && value !== '')
  );
}

function normalizeDiffValue(value) {
  return isNumericValue(value) ? Number(value) : value;
}

function valuesDiffer(oldVal, newVal) {
  return diffJson(
    { value: normalizeDiffValue(oldVal) },
    { value: normalizeDiffValue(newVal) }
  ).some((part) => part.added || part.removed);
}

function getChangedValues(oldObj, newObj, old = false) {
  const changes = {};

  const combinedObj = { ...oldObj, ...newObj };

  // Check all keys in the new object
  for (const key in combinedObj) {
    // Skip if the key is _id or timestamps
    if (AUDIT_IGNORED_KEYS.includes(key)) continue;

    const oldVal = oldObj ? oldObj[key] : undefined;
    const newVal = newObj ? newObj[key] : undefined;

    // If both values are objects (but not arrays or null), recurse
    if (isDiffableObject(oldVal) && isDiffableObject(newVal)) {
      if (oldVal?._id || newVal?._id) {
        if (valuesDiffer(oldVal?._id, newVal?._id)) {
          changes[key] = old ? oldVal : newVal;
        }
      } else {
        const nestedChanges = getChangedValues(oldVal, newVal, old);
        if (Object.keys(nestedChanges).length > 0) {
          changes[key] = nestedChanges;
        }
      }
    } else {
      if (valuesDiffer(oldVal, newVal)) {
        // If the old value is different from the new value, include it
        changes[key] = old ? oldVal : newVal;
      }
    }
  }

  return changes;
}

const AUDIT_EXCLUDED_MODELS = ['notification', 'userNotifier'];
const SENSITIVE_KEYS = ['secret'];

function omitSensitive(obj) {
  if (obj == null || typeof obj !== 'object') return obj;
  if (Array.isArray(obj)) return obj.map(omitSensitive);
  const result = {};
  for (const [key, value] of Object.entries(obj)) {
    if (SENSITIVE_KEYS.includes(key)) continue;
    result[key] = omitSensitive(value);
  }
  return result;
}

async function newAuditLog(newValue, parentId, parentType, user) {
  if (AUDIT_EXCLUDED_MODELS.includes(parentType)) return;

  // Filter out createdAt, updatedAt, and sensitive fields from newValue
  const filteredNewValue = omitSensitive({ ...newValue });
  delete filteredNewValue.createdAt;
  delete filteredNewValue.updatedAt;
  const auditLog = new auditLogModel({
    changes: {
      new: filteredNewValue,
    },
    parent: parentId,
    parentType,
    owner: user._id,
    ownerType: 'user',
    operation: 'new',
  });

  await auditLog.save();

  await distributeNew(auditLog, 'auditLog');
}

async function editAuditLog(oldValue, newValue, parentId, parentType, user) {
  if (AUDIT_EXCLUDED_MODELS.includes(parentType)) return;

  // Get only the changed values
  const changedOldValues = omitSensitive(getChangedValues(oldValue, newValue, true));
  const changedNewValues = omitSensitive(getChangedValues(oldValue, newValue, false));

  // If no values changed, don't create an audit log
  if (Object.keys(changedOldValues).length === 0 || Object.keys(changedNewValues).length === 0) {
    return;
  }

  const auditLog = new auditLogModel({
    changes: {
      old: changedOldValues,
      new: changedNewValues,
    },
    parent: parentId,
    parentType,
    owner: user._id,
    ownerType: 'user',
    operation: 'edit',
  });

  await auditLog.save();

  await distributeNew(auditLog, 'auditLog');
}

async function editNotification(oldValue, newValue, parentId, parentType, user) {
  const model = getModelByName(parentType);
  const objectName = oldValue?.name ?? newValue?.name ?? model?.label ?? parentType;
  const changedOldValues = omitSensitive(getChangedValues(oldValue, newValue, true));
  const changedNewValues = omitSensitive(getChangedValues(oldValue, newValue, false));

  if (Object.keys(changedOldValues).length === 0 || Object.keys(changedNewValues).length === 0) {
    return;
  }

  await notfiyObjectUserNotifiers(
    parentId,
    parentType,
    `${objectName} edited by ${user?.firstName ?? 'unknown'} ${user?.lastName ?? ''}`,
    `The ${parentType} ${parentId} has been updated.`,
    'editObject',
    {
      old: changedOldValues,
      new: changedNewValues,
      objectType: parentType,
      object: { _id: String(parentId ?? '') },
      user: {
        _id: String(user?._id ?? ''),
        firstName: user.firstName,
        lastName: user.lastName,
      },
    }
  );
}

async function deleteAuditLog(deleteValue, parentId, parentType, user) {
  if (AUDIT_EXCLUDED_MODELS.includes(parentType)) return;

  const auditLog = new auditLogModel({
    changes: {
      old: omitSensitive(deleteValue),
    },
    parent: parentId,
    parentType,
    owner: user._id,
    ownerType: 'user',
    operation: 'delete',
  });

  await auditLog.save();

  await distributeNew(auditLog, 'auditLog');
}

async function deleteNotification(object, parentId, parentType, user) {
  const model = getModelByName(parentType);
  const objectName = object?.name || model?.label || parentType;
  await notfiyObjectUserNotifiers(
    parentId,
    parentType,
    `${objectName} deleted by ${user?.firstName ?? 'unknown'} ${user?.lastName ?? ''}`,
    `The ${parentType} ${parentId} has been deleted.`,
    'deleteObject',
    {
      object: omitSensitive(object),
      objectType: parentType,
      object: { _id: parentId },
      user: { _id: user._id, firstName: user.firstName, lastName: user.lastName },
    }
  );
}

async function newNoteNotification(note, user) {
  const model = getModelByName(note.parentType);
  const objectName = model?.label ?? note.parentType;
  await notfiyObjectUserNotifiers(
    note.parent,
    note.parentType,
    `New note added to ${objectName.toLowerCase()} by ${user?.firstName ?? 'unknown'} ${user?.lastName ?? ''}`,
    `A new note has been created.`,
    'newNote',
    {
      objectType: note.parentType,
      object: { _id: String(note.parent ?? '') },
      note: note,
      user: {
        _id: String(user?._id ?? ''),
        firstName: user?.firstName,
        lastName: user?.lastName,
      },
    }
  );
}

async function subscribeAuditLog(parentId, parentType, user) {
  const auditLog = new auditLogModel({
    parent: parentId,
    parentType,
    owner: user._id,
    ownerType: 'user',
    operation: 'subscribe',
  });
  await auditLog.save();
  await distributeNew(auditLog, 'auditLog');
  return auditLog;
}

async function unsubscribeAuditLog(parentId, parentType, user) {
  const auditLog = new auditLogModel({
    parent: parentId,
    parentType,
    owner: user._id,
    ownerType: 'user',
    operation: 'unsubscribe',
  });
  await auditLog.save();
  await distributeNew(auditLog, 'auditLog');
  return auditLog;
}

async function getAuditLogs(idOrIds) {
  if (Array.isArray(idOrIds)) {
    return auditLogModel.find({ parent: { $in: idOrIds } }).populate('owner');
  } else {
    return auditLogModel.find({ parent: idOrIds }).populate('owner');
  }
}

async function distributeUpdate(value, id, type) {
  await natsServer.publish(`${type}s.${id}.object`, value);
}

async function distributeStats(value, type) {
  await natsServer.publish(`${type}s.stats`, value);
}

async function distributeNew(value, type) {
  await natsServer.publish(`${type}s.new`, value);
}

async function distributeDelete(value, type) {
  await natsServer.publish(`${type}s.delete`, value);
}

function getReferenceId(value) {
  if (value instanceof mongoose.Types.ObjectId) {
    return value;
  }
  if (value && typeof value === 'object' && value._id) {
    return getReferenceId(value._id);
  }
  return value;
}

function getReferenceIdString(value) {
  const id = getReferenceId(value);
  return id == null ? null : id.toString();
}

async function distributeChildUpdate(oldValue, newValue, id, model) {
  const oldPopulatedObjects = populateObjects(oldValue, model) || [];
  const oldPopulatedObjectIds = oldPopulatedObjects.map((populate) =>
    getReferenceIdString(populate._id)
  );
  const newPopulatedObjects = populateObjects(newValue, model) || [];
  const newPopulatedObjectIds = newPopulatedObjects.map((populate) =>
    getReferenceIdString(populate._id)
  );

  for (const populated of oldPopulatedObjects) {
    const populatedId = getReferenceIdString(populated._id);
    if (!populatedId) continue;
    if (!newPopulatedObjectIds.includes(populatedId)) {
      logger.debug(
        `Distributing child update for ${populated.ref}s.${populatedId}.events.childUpdate`
      );
      await natsServer.publish(`${populated.ref}s.${populatedId}.events.childUpdate`, {
        type: 'childUpdate',
        data: { parentId: id, parentType: model.modelName },
      });
    }
  }
  for (const populated of newPopulatedObjects) {
    const populatedId = getReferenceIdString(populated._id);
    if (!populatedId) continue;
    if (!oldPopulatedObjectIds.includes(populatedId)) {
      logger.debug(
        `Distributing child update for ${populated.ref}s.${populatedId}.events.childUpdate`
      );
      await natsServer.publish(`${populated.ref}s.${populatedId}.events.childUpdate`, {
        type: 'childUpdate',
        data: { parentId: id, parentType: model.modelName },
      });
    }
  }
}

async function distributeChildDelete(value, id, model) {
  const populatedObjects = populateObjects(value, model) || [];
  for (const populated of populatedObjects) {
    const populatedId = getReferenceIdString(populated._id);
    if (!populatedId) continue;
    logger.debug(
      `Distributing child delete for ${populated.ref}s.${populatedId}.events.childDelete`
    );
    await natsServer.publish(`${populated.ref}s.${populatedId}.events.childDelete`, {
      type: 'childDelete',
      data: { parentId: id, parentType: model.modelName },
    });
  }
}

async function distributeChildNew(value, id, model) {
  const populatedObjects = populateObjects(value, model) || [];
  for (const populated of populatedObjects) {
    const populatedId = getReferenceIdString(populated._id);
    if (!populatedId) continue;
    logger.debug(`Distributing child new for ${populated.ref}s.${populatedId}.events.childNew`);
    await natsServer.publish(`${populated.ref}s.${populatedId}.events.childNew`, {
      type: 'childNew',
      data: { parentId: id, parentType: model.modelName },
    });
  }
}

async function notfiyObjectUserNotifiers(id, objectType, title, message, type = 'info', metadata) {
  const userNotifiers = await userNotifierModel.find({ object: id, objectType: objectType });
  for (const userNotifier of userNotifiers) {
    await createNotification(userNotifier.user._id, title, message, type, metadata);
    if (userNotifier.email == true) {
      await sendEmailNotification(userNotifier.user, title, message, type, metadata);
    }
  }
}

async function createNotification(user, title, message, type = 'info', metadata) {
  const notification = new notificationModel({
    user,
    title,
    message,
    type,
    metadata: omitSensitive(metadata ?? {}),
  });
  await notification.save();
  const value = notification.toJSON ? notification.toJSON() : notification;
  await natsServer.publish(`notifications.${user._id}`, value);
  return notification;
}

let mailWorker = null;

function getMailWorker() {
  if (!mailWorker) {
    const workerPath = path.join(__dirname, 'mailworker.js');
    mailWorker = new Worker(workerPath);
    mailWorker.on('error', (err) => logger.error('MailWorker error:', err));
    mailWorker.on('exit', (code) => {
      if (code !== 0) logger.warn('MailWorker exited with code', code);
      mailWorker = null;
    });
  }
  return mailWorker;
}

/**
 * Sends an email notification asynchronously via a worker thread.
 * Renders a React template on the urlClient, captures HTML with Puppeteer, and emails the user.
 * Accepts the same input as createNotification: user, title, message, type, metadata.
 * Returns immediately; does not wait for the email to be sent.
 * @param {ObjectId|Object} user - User ID or user object (must have email)
 * @param {string} title - Notification title
 * @param {string} message - Notification message
 * @param {string} type - Notification type (info, editObject, deleteObject, error, success)
 * @param {Object} metadata - Optional metadata object
 */
async function sendEmailNotification(user, title, message, type = 'info', metadata) {
  let userDoc = user;
  if (user && (mongoose.Types.ObjectId.isValid(user) || user._id)) {
    const userId = user._id || user;
    userDoc = await userModel.findById(userId).lean();
  }
  if (!userDoc?.email) {
    logger.warn('sendEmailNotification: no email for user', user);
    return null;
  }

  const smtpConfig = config.smtp;
  if (!smtpConfig?.host) {
    logger.warn('sendEmailNotification: SMTP not configured, skipping email');
    return null;
  }

  const urlClient = config.app?.urlClient || 'http://localhost:3000';
  const authCode = createEmailRenderAuthCode(userDoc);

  const payload = {
    email: userDoc.email,
    title,
    message,
    type,
    metadata: omitSensitive(metadata || {}),
    createdAt: new Date(),
    updatedAt: new Date(),
    authCode,
    smtpConfig: {
      host: smtpConfig.host,
      port: smtpConfig.port || 587,
      secure: smtpConfig.secure || false,
      auth: smtpConfig.auth?.user ? smtpConfig.auth : undefined,
      from: smtpConfig.from || 'FarmControl <noreply@farmcontrol.app>',
    },
    urlClient,
  };

  try {
    getMailWorker().postMessage(payload);
  } catch (err) {
    logger.error('sendEmailNotification: failed to post to worker', err.message);
  }
  return null;
}

function flatternObjectIds(object) {
  if (!object || typeof object !== 'object') {
    return object;
  }

  const result = {};

  for (const [key, value] of Object.entries(object)) {
    if (value && typeof value === 'object' && value._id) {
      // If the value is an object with _id, convert to just the _id
      result[key] = value._id;
    } else {
      // Keep primitive values as is
      result[key] = value;
    }
  }

  return result;
}

function expandObjectIds(input) {
  const excludedFields = ['createdAt', 'updatedAt', 'name', '_id'];
  // Helper to check if a value is an ObjectId or a 24-char hex string
  function isObjectId(val) {
    // Check for MongoDB ObjectId instance
    if (val instanceof mongoose.Types.ObjectId) return true;
    // Check for exactly 24 hex characters (no special characters)
    if (typeof val === 'string' && /^[a-fA-F\d]{24}$/.test(val)) return true;
    return false;
  }

  // Recursive function
  function expand(value) {
    if (Array.isArray(value)) {
      return value.map(expand);
    } else if (value && typeof value === 'object' && !(value instanceof mongoose.Types.ObjectId)) {
      var result = {};
      for (const [key, val] of Object.entries(value)) {
        if (excludedFields.includes(key)) {
          // Do not expand keys that are excluded
          result[key] = val.toString();
        } else if (isObjectId(val)) {
          result[key] = { _id: val };
        } else if (Array.isArray(val)) {
          result[key] = val.map(expand);
        } else if (val instanceof Date) {
          result[key] = val;
        } else if (val && typeof val === 'object') {
          result[key] = expand(val);
        } else {
          result[key] = val;
        }
      }
      return result;
    } else if (isObjectId(value)) {
      return { _id: value };
    } else {
      return value;
    }
  }

  return expand(input);
}

// Merges per-field filter clauses, falling back to $and when clauses use
// logical operators or target the same field (so nothing is silently lost).
function mergeFilterClauses(clauses) {
  const valid = clauses.filter((clause) => clause && Object.keys(clause).length > 0);
  if (valid.length === 0) return {};
  if (valid.length === 1) return valid[0];

  const seen = new Set();
  let needsAnd = false;
  for (const clause of valid) {
    for (const key of Object.keys(clause)) {
      if (key === '$or' || key === '$and' || seen.has(key)) {
        needsAnd = true;
      }
      seen.add(key);
    }
  }

  return needsAnd ? { $and: valid } : Object.assign({}, ...valid);
}

// Returns a filter object based on allowed filters and req.query
function getFilter(query, allowedFilters, parse = true) {
  const clauses = [];
  for (const [key, value] of Object.entries(query)) {
    if (allowedFilters.includes(key)) {
      clauses.push(parse ? parseFilter(key, value) : { [key]: value });
    }
  }
  console.log('clauses', clauses);
  return mergeFilterClauses(clauses);
}

// Converts a properties argument (string or array) to an array of strings
function convertPropertiesString(properties) {
  if (typeof properties === 'string') {
    return properties.split(',');
  } else if (!Array.isArray(properties)) {
    return [];
  }
  return properties;
}

async function getFileMeta(file) {
  try {
    if (!file) return {};

    const originalName = file.originalname || '';
    const lowerName = originalName.toLowerCase();

    if (lowerName.endsWith('.g') || lowerName.endsWith('.gcode')) {
      const content = file.buffer ? file.buffer.toString('utf8') : '';
      if (!content) return {};
      return extractGCodeConfigBlock(content);
    }

    // Image EXIF metadata
    if (file.mimetype && file.mimetype.startsWith('image/') && file.buffer) {
      try {
        const exif = await exifr.parse(file.buffer);
        return exif || {};
      } catch (_) {
        // Ignore EXIF parse errors and fall through
      }
    }

    return {};
  } catch (_) {
    return {};
  }
}

function modelHasRef(model, refName) {
  if (!model || !model.schema) {
    return false;
  }

  let hasRef = false;
  model.schema.eachPath((pathName, schemaType) => {
    const directRef = schemaType?.options?.ref;
    const arrayRef = schemaType?.caster?.options?.ref;
    const ref = directRef || arrayRef;
    if (ref === refName) {
      hasRef = true;
    }
  });

  return hasRef;
}

function getFieldsByRef(model, refName) {
  if (!model || !model.schema) {
    return [];
  }

  const fields = [];
  model.schema.eachPath((pathName, schemaType) => {
    const directRef = schemaType?.options?.ref;
    const arrayRef = schemaType?.caster?.options?.ref;
    const ref = directRef || arrayRef;
    if (ref === refName) {
      fields.push(pathName);
    }
  });

  return fields;
}

// Build a nested populate specification by walking the schema graph,
// instead of recursing over already-populated documents.
function buildDeepPopulateSpec(object, model, populated = new Set()) {
  // prevent infinite recursion across cyclic model relationships
  if (populated.has(model.modelName)) return [];
  populated.add(model.modelName);

  const schema = model.schema;
  const populateSpec = [];

  schema.eachPath((pathname, schemaType) => {
    const directRef = schemaType.options?.ref;
    const arrayRef = schemaType.caster?.options?.ref;
    const ref = directRef || arrayRef;
    if (!ref) return;

    const refName = typeof ref === 'function' ? ref.call(object) : ref;
    if (!refName) return;

    const refModel = model.db.model(refName);
    const childPopulate = buildDeepPopulateSpec(object, refModel, populated);

    const values = Array.isArray(object[pathname]) ? object[pathname] : [object[pathname]];
    const ids = values.map(getReferenceId).filter(Boolean);

    for (const id of ids) {
      if (childPopulate.length > 0) {
        populateSpec.push({ path: pathname, populate: childPopulate, ref: refName, _id: id });
      } else {
        populateSpec.push({ path: pathname, ref: refName, _id: id });
      }
    }
  });

  return populateSpec;
}

function populateObjects(object, model, populated = new Set()) {
  const populateSpec = buildDeepPopulateSpec(object, model, populated);

  return populateSpec;
}

function jsonToCacheKey(obj) {
  const normalized = canonicalize(obj);
  const hash = crypto.createHash('sha256').update(normalized).digest('hex');
  return hash;
}

export function getQueryToCacheKey({ model, id, populate }) {
  const populateKey = [];
  if (populate) {
    const populateArray = Array.isArray(populate) ? populate : [populate];
    for (const pop of populateArray) {
      if (typeof pop === 'string') {
        populateKey.push(pop);
      } else if (typeof pop === 'object' && pop.path) {
        populateKey.push(pop.path);
      }
    }
  }
  return `${model}:${id?.toString()}-${populateKey.join(',')}`;
}

export {
  parseFilter,
  convertToCamelCase,
  newAuditLog,
  editNotification,
  deleteNotification,
  editAuditLog,
  deleteAuditLog,
  getAuditLogs,
  flatternObjectIds,
  expandObjectIds,
  newNoteNotification,
  distributeUpdate,
  distributeStats,
  distributeNew,
  distributeDelete,
  distributeChildUpdate,
  distributeChildDelete,
  distributeChildNew,
  notfiyObjectUserNotifiers,
  createNotification,
  sendEmailNotification,
  getFilter, // <-- add here
  convertPropertiesString,
  getFileMeta,
  modelHasRef,
  getFieldsByRef,
  jsonToCacheKey,
  subscribeAuditLog,
  unsubscribeAuditLog,
};