farmcontrol-api/src/services/management/files.js

import config from '../../config.js';
import { fileModel } from '../../database/schemas/management/file.schema.js';
import log4js from 'log4js';
import multer from 'multer';
import path from 'path';
import fs from 'fs';
import mongoose from 'mongoose';
import {
  deleteObject,
  listObjects,
  getObject,
  editObject,
  newObject,
  listObjectsByProperties,
  flushFile,
  getModelStats,
  getModelHistory,
  searchObjects,
  getPropertyValues,
} from '../../database/database.js';
import {
  uploadFile,
  downloadFile,
  deleteFile as deleteCephFile,
  BUCKETS,
} from '../../database/ceph.js';
import { getFileMeta } from '../../utils.js';

const logger = log4js.getLogger('Files');
logger.level = config.server.logLevel;

const getDownloadContentType = (file) => {
  // G-code is text, but it must be transferred byte-for-byte. Some slicers
  // report text/plain, causing proxies to compress an otherwise fixed-length
  // response and browsers to reject it with a content-length network error.
  if (file.extension?.toLowerCase() === '.gcode') {
    return 'application/octet-stream';
  }

  return file.type || 'application/octet-stream';
};

// Set storage engine to memory for Ceph upload
const fileStorage = multer.memoryStorage();

// Initialise upload
const fileUpload = multer({
  storage: fileStorage,
  limits: { fileSize: 500000000 }, // 500MB limit
  fileFilter: function (req, file, cb) {
    checkFileType(file, cb);
  },
}).single('file'); // The name attribute of the file input in the HTML form

export const listFilesRouteHandler = async (
  req,
  res,
  page = 1,
  limit = 25,
  property = '',
  filter = {},
  search = '',
  sort = '',
  order = 'ascend'
) => {
  const result = await listObjects({
    model: fileModel,
    page,
    limit,
    property,
    filter,
    search,
    sort,
    order,
  });

  if (result?.error) {
    logger.error('Error listing files.');
    res.status(result.code).send(result);
    return;
  }

  logger.debug(`List of files (Page ${page}, Limit ${limit}). Count: ${result.length}`);
  res.send(result);
};

export const listFilesByPropertiesRouteHandler = async (
  req,
  res,
  properties = '',
  filter = {},
  masterFilter = {}
) => {
  const result = await listObjectsByProperties({
    model: fileModel,
    properties,
    filter,
    masterFilter,
  });

  if (result?.error) {
    logger.error('Error listing files.');
    res.status(result.code).send(result);
    return;
  }

  logger.debug(`List of files. Count: ${result.length}`);
  res.send(result);
};

export const getFilePropertyValuesRouteHandler = async (req, res, property) => {
  const result = await getPropertyValues({
    model: fileModel,
    property,
  });
  res.send(result);
};

export const searchFilesRouteHandler = async (req, res, search) => {
  const result = await searchObjects({
    model: fileModel,
    search,
  });
  res.send(result);
};

export const getFileRouteHandler = async (req, res) => {
  const id = req.params.id;
  const result = await getObject({
    model: fileModel,
    id,
  });
  if (result?.error) {
    logger.warn(`File not found with supplied id.`);
    return res.status(result.code).send(result);
  }
  logger.debug(`Retreived file with ID: ${id}`);
  res.send(result);
};

export const flushFileRouteHandler = async (req, res) => {
  const id = req.params.id;
  const result = await flushFile({ user: req.user, id });
  res.send(result);
};

export const editFileRouteHandler = async (req, res) => {
  // Get ID from params
  const id = new mongoose.Types.ObjectId(req.params.id);

  logger.trace(`File with ID: ${id}`);

  const updateData = {
    updatedAt: new Date(),
    name: req.body.name,
  };

  const result = await editObject({
    model: fileModel,
    id,
    updateData,
    user: req.user,
  });

  if (result.error) {
    logger.error('Error editing file:', result.error);
    res.status(result).send(result);
    return;
  }

  logger.debug(`Edited file with ID: ${id}`);

  res.send(result);
};

export const newFileRouteHandler = async (req, res) => {
  try {
    fileUpload(req, res, async (err) => {
      if (err) {
        return res.status(500).send({
          error: err,
        });
      }

      if (req.file == undefined) {
        return res.send({
          message: 'No file selected!',
        });
      }

      try {
        // Create DB entry first without storage fields
        const extension = path.extname(req.file.originalname);
        const baseName = path.parse(req.file.originalname).name;
        const meta = await getFileMeta(req.file);
        const newData = {
          name: baseName,
          type: req.file.mimetype,
          extension,
          size: req.file.size,
          updatedAt: new Date(),
          metaData: {
            originalName: req.file.originalname,
            ...meta,
          },
        };

        const created = await newObject({
          model: fileModel,
          newData,
          user: req.user,
        });

        if (created.error) {
          logger.error('No file created:', created.error);
          return res.status(created.code).send(created);
        }

        // Use created document _id to generate Ceph key
        const cephKey = `files/${created._id}${extension}`;

        // Upload file to Ceph
        await uploadFile(BUCKETS.FILES, cephKey, req.file.buffer, req.file.mimetype, {
          originalName: req.file.originalname,
          uploadedBy: req.user?.username || 'unknown',
        });

        // Do not update DB with Ceph location. Return created DB record.
        logger.debug(`New file with ID: ${created._id} created and uploaded to Ceph`);

        res.send(created);
      } catch (createError) {
        logger.error('Error creating file record or uploading to storage:', createError);
        // If we created the DB entry but upload failed, remove the DB entry to avoid orphaned records
        try {
          if (created && created._id) {
            await deleteObject({ model: fileModel, id: created._id, user: req.user });
          }
        } catch (cleanupDbError) {
          logger.error('Error cleaning up DB record after upload failure:', cleanupDbError);
        }
        res.status(500).send({ error: createError.message });
      }
    });
  } catch (error) {
    logger.error('Error in newFileRouteHandler:', error);
    res.status(500).send({ error: error.message });
  }
};

export const deleteFileRouteHandler = async (req, res) => {
  // Get ID from params
  const id = new mongoose.Types.ObjectId(req.params.id);

  logger.trace(`File with ID: ${id}`);

  try {
    // First get the file to retrieve Ceph information
    const file = await getObject({
      model: fileModel,
      id,
    });
    if (!file) {
      return res.status(404).send({ error: 'File not found' });
    }

    // Delete from Ceph if it exists there
    if (file.cephBucket && file.cephKey) {
      try {
        await deleteCephFile(file.cephBucket, file.cephKey);
        logger.debug(`Deleted file from Ceph: ${file.cephKey}`);
      } catch (cephError) {
        logger.warn(`Failed to delete file from Ceph: ${cephError.message}`);
        // Continue with database deletion even if Ceph deletion fails
      }
    }

    const result = await deleteObject({
      model: fileModel,
      id,
      user: req.user,
    });
    if (result.error) {
      logger.error('No file deleted:', result.error);
      return res.status(result.code).send(result);
    }

    logger.debug(`Deleted file with ID: ${result._id}`);

    res.send(result);
  } catch (error) {
    logger.error('Error in deleteFileRouteHandler:', error);
    res.status(500).send({ error: error.message });
  }
};

// Check file type
function checkFileType(file, cb) {
  // Allow all file types for general file management
  // You can customize this to restrict specific file types if needed
  const allowedTypes = /.*/; // Allow all file types

  if (allowedTypes.test(file.mimetype)) {
    return cb(null, true);
  } else {
    cb('Error: File type not allowed!');
  }
}

export const getFileContentRouteHandler = async (req, res) => {
  const id = req.params.id;
  let bytesSent = 0;
  let sourceEnded = false;

  logger.info(`File download requested: ${id}`);

  req.once('aborted', () => {
    logger.warn(`File download request aborted by client: ${id} (${bytesSent} bytes sent)`);
  });

  res.once('finish', () => {
    logger.info(
      `File download response finished: ${id} (status ${res.statusCode}, ${bytesSent} bytes sent)`
    );
  });

  res.once('close', () => {
    if (!res.writableFinished) {
      logger.warn(
        `File download response closed early: ${id} (${bytesSent} bytes sent, source ended: ${sourceEnded})`
      );
    }
  });

  try {
    const file = await getObject({
      model: fileModel,
      id,
    });

    if (!file) {
      logger.warn(`File not found with supplied id.`);
      return res.status(404).send({ error: 'File not found.' });
    }

    logger.trace(`Returning file contents with ID: ${id}:`);
    logger.debug(`File record loaded for download: ${id}`, {
      name: file.name,
      extension: file.extension,
      recordedSize: file.size,
      type: file.type,
    });

    // Check if file is stored in Ceph
    if (file._id && file.extension) {
      const cephKey = `files/${id}${file.extension}`;
      logger.info(`Starting Ceph download: ${cephKey} (recorded size: ${file.size ?? 'unknown'})`);

      try {
        const body = await downloadFile(BUCKETS.FILES, cephKey);
        logger.debug(`Ceph response stream opened: ${cephKey}`, {
          readable: body?.readable,
          destroyed: body?.destroyed,
        });

        // Preserve the stored bytes across reverse proxies. This also repairs
        // downloads of existing slicer uploads recorded as text/plain.
        res.set('Content-Type', getDownloadContentType(file));
        res.set('Content-Disposition', `attachment; filename="${file.name}${file.extension}"`);
        res.set('Cache-Control', 'private, no-transform');

        // Stream or send buffer
        if (body && typeof body.pipe === 'function') {
          body.on('data', (chunk) => {
            bytesSent += chunk.length;
          });

          body.on('end', () => {
            sourceEnded = true;
            logger.info(`Ceph stream ended: ${cephKey} (${bytesSent} bytes read)`);

            if (
              typeof file.size === 'number' &&
              !Number.isNaN(file.size) &&
              bytesSent !== file.size
            ) {
              logger.warn(
                `File size mismatch for ${cephKey}: database=${file.size}, streamed=${bytesSent}`
              );
            }
          });

          body.on('close', () => {
            logger.debug(
              `Ceph stream closed: ${cephKey} (${bytesSent} bytes read, ended: ${sourceEnded})`
            );
          });

          // Handle stream errors
          body.on('error', (err) => {
            logger.error(
              `Error streaming file from Ceph: ${cephKey} (${bytesSent} bytes read)`,
              err
            );
            // If headers not sent, send a 500; otherwise destroy the response
            if (!res.headersSent) {
              try {
                res.status(500).send({ error: 'Error streaming file from storage.' });
              } catch (_) {
                // Ignore secondary errors
              }
            } else {
              res.destroy(err);
            }
          });

          // If client disconnects, stop reading from source
          res.on('close', () => {
            if (typeof body.destroy === 'function') {
              body.destroy();
            }
          });

          body.pipe(res);
          logger.debug(`Ceph stream piped to response: ${cephKey}`);
          return;
        }

        // Unknown body type
        logger.error('Unknown Ceph body type; cannot send response');
        return res.status(500).send({ error: 'Error reading file from storage.' });
      } catch (cephError) {
        logger.error('Error downloading file from Ceph:', cephError);
        // Fall through to local filesystem fallback below
      }

      // Fallback to local file system for backward compatibility
      const filePath = path.join(config.storage.fileStorage, file.fileName || file.name);
      logger.info(`Falling back to local file download: ${filePath}`);

      // Read the file
      fs.readFile(filePath, (err, data) => {
        if (err) {
          if (err.code === 'ENOENT') {
            return res.status(404).send({ error: 'File not found!' });
          }
          return res.status(500).send({ error: 'Error reading file.' });
        }

        bytesSent = data.length;
        sourceEnded = true;
        logger.info(`Local file read complete: ${filePath} (${data.length} bytes)`);

        res.set('Content-Type', getDownloadContentType(file));
        res.set('Content-Disposition', `inline; filename="${file.name}${file.extension || ''}"`);
        res.set('Cache-Control', 'private, no-transform');

        // Ensure Content-Length is set for progress events if possible.
        const length =
          typeof file.size === 'number' && !Number.isNaN(file.size) ? file.size : data.length;
        res.set('Content-Length', String(length));

        return res.send(data);
      });
    } else {
      logger.error('Error fetching file:', 'No ceph bucket or key supplied.');
      res.status(500).send({ error: 'No ceph bucket or key supplied.' });
    }
  } catch (error) {
    logger.error('Error fetching file:', error);
    res.status(500).send({ error: error.message });
  }
};

export const parseFileHandler = async (req, res) => {
  try {
    // Use the same upload middleware as the uploadFileContentRouteHandler
    fileUpload(req, res, async (err) => {
      if (err) {
        return res.status(500).send({
          error: err,
        });
      }

      if (req.file == undefined) {
        return res.send({
          message: 'No file selected!',
        });
      }

      try {
        // Read the file content from memory buffer
        const fileContent = req.file.buffer.toString('utf8');

        // Return basic file info as JSON
        const fileInfo = {
          filename: req.file.originalname,
          originalName: req.file.originalname,
          size: req.file.size,
          mimetype: req.file.mimetype,
          content: fileContent.substring(0, 1000), // First 1000 characters
        };

        res.json(fileInfo);
      } catch (parseError) {
        logger.error('Error parsing file:', parseError);
        res.status(500).send({ error: parseError.message });
      }
    });
  } catch (error) {
    logger.error('Error in parseFileHandler:', error);
    res.status(500).send({ error: error.message });
  }
};

export const getFileStatsRouteHandler = async (req, res) => {
  const result = await getModelStats({ model: fileModel });
  if (result?.error) {
    logger.error('Error fetching file stats:', result.error);
    return res.status(result.code).send(result);
  }
  logger.trace('File stats:', result);
  res.send(result);
};

export const getFileHistoryRouteHandler = async (req, res) => {
  const from = req.query.from;
  const to = req.query.to;
  const result = await getModelHistory({ model: fileModel, from, to });
  if (result?.error) {
    logger.error('Error fetching file history:', result.error);
    return res.status(result.code).send(result);
  }
  logger.trace('File history:', result);
  res.send(result);
};