farmcontrol-api/src/services/management/files.js

import dotenv from 'dotenv';
import { fileModel } from '../../schemas/management/file.schema.js';
import log4js from 'log4js';
import multer from 'multer';
import path from 'path';
import fs from 'fs';
import mongoose from 'mongoose';
import {
  deleteObject,
  listObjects,
  getObject,
  editObject,
  newObject,
  listObjectsByProperties,
  flushFile,
} from '../../database/database.js';
import {
  uploadFile,
  downloadFile,
  deleteFile as deleteCephFile,
  BUCKETS,
} from '../storage/ceph.js';
import { getFileMeta } from '../../utils.js';
dotenv.config();

const logger = log4js.getLogger('Files');
logger.level = process.env.LOG_LEVEL;

// Set storage engine to memory for Ceph upload
const fileStorage = multer.memoryStorage();

// Initialise upload
const fileUpload = multer({
  storage: fileStorage,
  limits: { fileSize: 500000000 }, // 500MB limit
  fileFilter: function (req, file, cb) {
    checkFileType(file, cb);
  },
}).single('file'); // The name attribute of the file input in the HTML form

export const listFilesRouteHandler = async (
  req,
  res,
  page = 1,
  limit = 25,
  property = '',
  filter = {},
  search = '',
  sort = '',
  order = 'ascend'
) => {
  const result = await listObjects({
    model: fileModel,
    page,
    limit,
    property,
    filter,
    search,
    sort,
    order,
  });

  if (result?.error) {
    logger.error('Error listing files.');
    res.status(result.code).send(result);
    return;
  }

  logger.debug(`List of files (Page ${page}, Limit ${limit}). Count: ${result.length}`);
  res.send(result);
};

export const listFilesByPropertiesRouteHandler = async (
  req,
  res,
  properties = '',
  filter = {},
  masterFilter = {}
) => {
  const result = await listObjectsByProperties({
    model: fileModel,
    properties,
    filter,
    masterFilter,
  });

  if (result?.error) {
    logger.error('Error listing files.');
    res.status(result.code).send(result);
    return;
  }

  logger.debug(`List of files. Count: ${result.length}`);
  res.send(result);
};

export const getFileRouteHandler = async (req, res) => {
  const id = req.params.id;
  const result = await getObject({
    model: fileModel,
    id,
  });
  if (result?.error) {
    logger.warn(`File not found with supplied id.`);
    return res.status(result.code).send(result);
  }
  logger.debug(`Retreived file with ID: ${id}`);
  res.send(result);
};

export const flushFileRouteHandler = async (req, res) => {
  const id = req.params.id;
  const result = await flushFile({ user: req.user, id });
  res.send(result);
};

export const editFileRouteHandler = async (req, res) => {
  // Get ID from params
  const id = new mongoose.Types.ObjectId(req.params.id);

  logger.trace(`File with ID: ${id}`);

  const updateData = {
    updatedAt: new Date(),
    name: req.body.name,
  };

  const result = await editObject({
    model: fileModel,
    id,
    updateData,
    user: req.user,
  });

  if (result.error) {
    logger.error('Error editing file:', result.error);
    res.status(result).send(result);
    return;
  }

  logger.debug(`Edited file with ID: ${id}`);

  res.send(result);
};

export const newFileRouteHandler = async (req, res) => {
  try {
    fileUpload(req, res, async (err) => {
      if (err) {
        return res.status(500).send({
          error: err,
        });
      }

      if (req.file == undefined) {
        return res.send({
          message: 'No file selected!',
        });
      }

      try {
        // Create DB entry first without storage fields
        const extension = path.extname(req.file.originalname);
        const baseName = path.parse(req.file.originalname).name;
        const meta = await getFileMeta(req.file);
        const newData = {
          name: baseName,
          type: req.file.mimetype,
          extension,
          size: req.file.size,
          updatedAt: new Date(),
          metaData: {
            originalName: req.file.originalname,
            ...meta,
          },
        };

        const created = await newObject({
          model: fileModel,
          newData,
          user: req.user,
        });

        if (created.error) {
          logger.error('No file created:', created.error);
          return res.status(created.code).send(created);
        }

        // Use created document _id to generate Ceph key
        const cephKey = `files/${created._id}${extension}`;

        // Upload file to Ceph
        await uploadFile(BUCKETS.FILES, cephKey, req.file.buffer, req.file.mimetype, {
          originalName: req.file.originalname,
          uploadedBy: req.user?.username || 'unknown',
        });

        // Do not update DB with Ceph location. Return created DB record.
        logger.debug(`New file with ID: ${created._id} created and uploaded to Ceph`);

        res.send(created);
      } catch (createError) {
        logger.error('Error creating file record or uploading to storage:', createError);
        // If we created the DB entry but upload failed, remove the DB entry to avoid orphaned records
        try {
          if (created && created._id) {
            await deleteObject({ model: fileModel, id: created._id, user: req.user });
          }
        } catch (cleanupDbError) {
          logger.error('Error cleaning up DB record after upload failure:', cleanupDbError);
        }
        res.status(500).send({ error: createError.message });
      }
    });
  } catch (error) {
    logger.error('Error in newFileRouteHandler:', error);
    res.status(500).send({ error: error.message });
  }
};

export const deleteFileRouteHandler = async (req, res) => {
  // Get ID from params
  const id = new mongoose.Types.ObjectId(req.params.id);

  logger.trace(`File with ID: ${id}`);

  try {
    // First get the file to retrieve Ceph information
    const file = await getObject({
      model: fileModel,
      id,
    });
    if (!file) {
      return res.status(404).send({ error: 'File not found' });
    }

    // Delete from Ceph if it exists there
    if (file.cephBucket && file.cephKey) {
      try {
        await deleteCephFile(file.cephBucket, file.cephKey);
        logger.debug(`Deleted file from Ceph: ${file.cephKey}`);
      } catch (cephError) {
        logger.warn(`Failed to delete file from Ceph: ${cephError.message}`);
        // Continue with database deletion even if Ceph deletion fails
      }
    }

    const result = await deleteObject({
      model: fileModel,
      id,
      user: req.user,
    });
    if (result.error) {
      logger.error('No file deleted:', result.error);
      return res.status(result.code).send(result);
    }

    logger.debug(`Deleted file with ID: ${result._id}`);

    res.send(result);
  } catch (error) {
    logger.error('Error in deleteFileRouteHandler:', error);
    res.status(500).send({ error: error.message });
  }
};

// Check file type
function checkFileType(file, cb) {
  // Allow all file types for general file management
  // You can customize this to restrict specific file types if needed
  const allowedTypes = /.*/; // Allow all file types

  if (allowedTypes.test(file.mimetype)) {
    console.log(file);
    return cb(null, true);
  } else {
    cb('Error: File type not allowed!');
  }
}

export const getFileContentRouteHandler = async (req, res) => {
  try {
    const id = req.params.id;

    const file = await getObject({
      model: fileModel,
      id,
    });

    if (!file) {
      logger.warn(`File not found with supplied id.`);
      return res.status(404).send({ error: 'File not found.' });
    }

    logger.trace(`Returning file contents with ID: ${id}:`);

    // Check if file is stored in Ceph
    if (file._id && file.extension) {
      const cephKey = `files/${id}${file.extension}`;
      try {
        const body = await downloadFile(BUCKETS.FILES, cephKey);

        // Set appropriate content type and disposition
        res.set('Content-Type', file.type || 'application/octet-stream');
        res.set('Content-Disposition', `attachment; filename="${file.name}${file.extension}"`);

        // Expose file size so clients can compute download progress
        if (typeof file.size === 'number' && !Number.isNaN(file.size)) {
          res.set('Content-Length', String(file.size));
        }

        // Stream or send buffer
        if (body && typeof body.pipe === 'function') {
          // Handle stream errors
          body.on('error', (err) => {
            logger.error('Error streaming file from Ceph:', err);
            // If headers not sent, send a 500; otherwise destroy the response
            if (!res.headersSent) {
              try {
                res.status(500).send({ error: 'Error streaming file from storage.' });
              } catch (_) {
                // Ignore secondary errors
              }
            } else {
              res.destroy(err);
            }
          });

          // If client disconnects, stop reading from source
          res.on('close', () => {
            if (typeof body.destroy === 'function') {
              body.destroy();
            }
          });

          body.pipe(res);
          logger.debug('Retrieved:', cephKey);
          return;
        }

        // Unknown body type
        logger.error('Unknown Ceph body type; cannot send response');
        return res.status(500).send({ error: 'Error reading file from storage.' });
      } catch (cephError) {
        logger.error('Error downloading file from Ceph:', cephError);
        // Fall through to local filesystem fallback below
      }

      // Fallback to local file system for backward compatibility
      const filePath = path.join(
        process.env.FILE_STORAGE || './uploads',
        file.fileName || file.name
      );

      // Read the file
      fs.readFile(filePath, (err, data) => {
        if (err) {
          if (err.code === 'ENOENT') {
            return res.status(404).send({ error: 'File not found!' });
          }
          return res.status(500).send({ error: 'Error reading file.' });
        }

        res.set('Content-Type', file.type || 'application/octet-stream');
        res.set('Content-Disposition', `inline; filename="${file.name}${file.extension || ''}"`);

        // Ensure Content-Length is set for progress events if possible.
        const length =
          typeof file.size === 'number' && !Number.isNaN(file.size) ? file.size : data.length;
        res.set('Content-Length', String(length));

        return res.send(data);
      });
    } else {
      logger.error('Error fetching file:', 'No ceph bucket or key supplied.');
      res.status(500).send({ error: 'No ceph bucket or key supplied.' });
    }
  } catch (error) {
    logger.error('Error fetching file:', error);
    res.status(500).send({ error: error.message });
  }
};

export const parseFileHandler = async (req, res) => {
  try {
    // Use the same upload middleware as the uploadFileContentRouteHandler
    fileUpload(req, res, async (err) => {
      if (err) {
        return res.status(500).send({
          error: err,
        });
      }

      if (req.file == undefined) {
        return res.send({
          message: 'No file selected!',
        });
      }

      try {
        // Read the file content from memory buffer
        const fileContent = req.file.buffer.toString('utf8');

        // Return basic file info as JSON
        const fileInfo = {
          filename: req.file.originalname,
          originalName: req.file.originalname,
          size: req.file.size,
          mimetype: req.file.mimetype,
          content: fileContent.substring(0, 1000), // First 1000 characters
        };

        res.json(fileInfo);
      } catch (parseError) {
        logger.error('Error parsing file:', parseError);
        res.status(500).send({ error: parseError.message });
      }
    });
  } catch (error) {
    logger.error('Error in parseFileHandler:', error);
    res.status(500).send({ error: error.message });
  }
};