import dotenv from 'dotenv'; import { fileModel } from '../../schemas/management/file.schema.js'; import log4js from 'log4js'; import multer from 'multer'; import path from 'path'; import fs from 'fs'; import mongoose from 'mongoose'; import { deleteObject, listObjects, getObject, editObject, newObject, listObjectsByProperties, flushFile, } from '../../database/database.js'; import { uploadFile, downloadFile, deleteFile as deleteCephFile, BUCKETS, } from '../storage/ceph.js'; import { getFileMeta } from '../../utils.js'; dotenv.config(); const logger = log4js.getLogger('Files'); logger.level = process.env.LOG_LEVEL; // Set storage engine to memory for Ceph upload const fileStorage = multer.memoryStorage(); // Initialise upload const fileUpload = multer({ storage: fileStorage, limits: { fileSize: 500000000 }, // 500MB limit fileFilter: function (req, file, cb) { checkFileType(file, cb); }, }).single('file'); // The name attribute of the file input in the HTML form export const listFilesRouteHandler = async ( req, res, page = 1, limit = 25, property = '', filter = {}, search = '', sort = '', order = 'ascend' ) => { const result = await listObjects({ model: fileModel, page, limit, property, filter, search, sort, order, }); if (result?.error) { logger.error('Error listing files.'); res.status(result.code).send(result); return; } logger.debug(`List of files (Page ${page}, Limit ${limit}). Count: ${result.length}`); res.send(result); }; export const listFilesByPropertiesRouteHandler = async ( req, res, properties = '', filter = {}, masterFilter = {} ) => { const result = await listObjectsByProperties({ model: fileModel, properties, filter, masterFilter, }); if (result?.error) { logger.error('Error listing files.'); res.status(result.code).send(result); return; } logger.debug(`List of files. Count: ${result.length}`); res.send(result); }; export const getFileRouteHandler = async (req, res) => { const id = req.params.id; const result = await getObject({ model: fileModel, id, }); if (result?.error) { logger.warn(`File not found with supplied id.`); return res.status(result.code).send(result); } logger.debug(`Retreived file with ID: ${id}`); res.send(result); }; export const flushFileRouteHandler = async (req, res) => { const id = req.params.id; const result = await flushFile({ user: req.user, id }); res.send(result); }; export const editFileRouteHandler = async (req, res) => { // Get ID from params const id = new mongoose.Types.ObjectId(req.params.id); logger.trace(`File with ID: ${id}`); const updateData = { updatedAt: new Date(), name: req.body.name, }; const result = await editObject({ model: fileModel, id, updateData, user: req.user, }); if (result.error) { logger.error('Error editing file:', result.error); res.status(result).send(result); return; } logger.debug(`Edited file with ID: ${id}`); res.send(result); }; export const newFileRouteHandler = async (req, res) => { try { fileUpload(req, res, async (err) => { if (err) { return res.status(500).send({ error: err, }); } if (req.file == undefined) { return res.send({ message: 'No file selected!', }); } try { // Create DB entry first without storage fields const extension = path.extname(req.file.originalname); const baseName = path.parse(req.file.originalname).name; const meta = await getFileMeta(req.file); const newData = { name: baseName, type: req.file.mimetype, extension, size: req.file.size, updatedAt: new Date(), metaData: { originalName: req.file.originalname, ...meta, }, }; const created = await newObject({ model: fileModel, newData, user: req.user, }); if (created.error) { logger.error('No file created:', created.error); return res.status(created.code).send(created); } // Use created document _id to generate Ceph key const cephKey = `files/${created._id}${extension}`; // Upload file to Ceph await uploadFile(BUCKETS.FILES, cephKey, req.file.buffer, req.file.mimetype, { originalName: req.file.originalname, uploadedBy: req.user?.username || 'unknown', }); // Do not update DB with Ceph location. Return created DB record. logger.debug(`New file with ID: ${created._id} created and uploaded to Ceph`); res.send(created); } catch (createError) { logger.error('Error creating file record or uploading to storage:', createError); // If we created the DB entry but upload failed, remove the DB entry to avoid orphaned records try { if (created && created._id) { await deleteObject({ model: fileModel, id: created._id, user: req.user }); } } catch (cleanupDbError) { logger.error('Error cleaning up DB record after upload failure:', cleanupDbError); } res.status(500).send({ error: createError.message }); } }); } catch (error) { logger.error('Error in newFileRouteHandler:', error); res.status(500).send({ error: error.message }); } }; export const deleteFileRouteHandler = async (req, res) => { // Get ID from params const id = new mongoose.Types.ObjectId(req.params.id); logger.trace(`File with ID: ${id}`); try { // First get the file to retrieve Ceph information const file = await getObject({ model: fileModel, id, }); if (!file) { return res.status(404).send({ error: 'File not found' }); } // Delete from Ceph if it exists there if (file.cephBucket && file.cephKey) { try { await deleteCephFile(file.cephBucket, file.cephKey); logger.debug(`Deleted file from Ceph: ${file.cephKey}`); } catch (cephError) { logger.warn(`Failed to delete file from Ceph: ${cephError.message}`); // Continue with database deletion even if Ceph deletion fails } } const result = await deleteObject({ model: fileModel, id, user: req.user, }); if (result.error) { logger.error('No file deleted:', result.error); return res.status(result.code).send(result); } logger.debug(`Deleted file with ID: ${result._id}`); res.send(result); } catch (error) { logger.error('Error in deleteFileRouteHandler:', error); res.status(500).send({ error: error.message }); } }; // Check file type function checkFileType(file, cb) { // Allow all file types for general file management // You can customize this to restrict specific file types if needed const allowedTypes = /.*/; // Allow all file types if (allowedTypes.test(file.mimetype)) { console.log(file); return cb(null, true); } else { cb('Error: File type not allowed!'); } } export const getFileContentRouteHandler = async (req, res) => { try { const id = req.params.id; const file = await getObject({ model: fileModel, id, }); if (!file) { logger.warn(`File not found with supplied id.`); return res.status(404).send({ error: 'File not found.' }); } logger.trace(`Returning file contents with ID: ${id}:`); // Check if file is stored in Ceph if (file._id && file.extension) { const cephKey = `files/${id}${file.extension}`; try { const body = await downloadFile(BUCKETS.FILES, cephKey); // Set appropriate content type and disposition res.set('Content-Type', file.type || 'application/octet-stream'); res.set('Content-Disposition', `attachment; filename="${file.name}${file.extension}"`); // Expose file size so clients can compute download progress if (typeof file.size === 'number' && !Number.isNaN(file.size)) { res.set('Content-Length', String(file.size)); } // Stream or send buffer if (body && typeof body.pipe === 'function') { // Handle stream errors body.on('error', (err) => { logger.error('Error streaming file from Ceph:', err); // If headers not sent, send a 500; otherwise destroy the response if (!res.headersSent) { try { res.status(500).send({ error: 'Error streaming file from storage.' }); } catch (_) { // Ignore secondary errors } } else { res.destroy(err); } }); // If client disconnects, stop reading from source res.on('close', () => { if (typeof body.destroy === 'function') { body.destroy(); } }); body.pipe(res); logger.debug('Retrieved:', cephKey); return; } // Unknown body type logger.error('Unknown Ceph body type; cannot send response'); return res.status(500).send({ error: 'Error reading file from storage.' }); } catch (cephError) { logger.error('Error downloading file from Ceph:', cephError); // Fall through to local filesystem fallback below } // Fallback to local file system for backward compatibility const filePath = path.join( process.env.FILE_STORAGE || './uploads', file.fileName || file.name ); // Read the file fs.readFile(filePath, (err, data) => { if (err) { if (err.code === 'ENOENT') { return res.status(404).send({ error: 'File not found!' }); } return res.status(500).send({ error: 'Error reading file.' }); } res.set('Content-Type', file.type || 'application/octet-stream'); res.set('Content-Disposition', `inline; filename="${file.name}${file.extension || ''}"`); // Ensure Content-Length is set for progress events if possible. const length = typeof file.size === 'number' && !Number.isNaN(file.size) ? file.size : data.length; res.set('Content-Length', String(length)); return res.send(data); }); } else { logger.error('Error fetching file:', 'No ceph bucket or key supplied.'); res.status(500).send({ error: 'No ceph bucket or key supplied.' }); } } catch (error) { logger.error('Error fetching file:', error); res.status(500).send({ error: error.message }); } }; export const parseFileHandler = async (req, res) => { try { // Use the same upload middleware as the uploadFileContentRouteHandler fileUpload(req, res, async (err) => { if (err) { return res.status(500).send({ error: err, }); } if (req.file == undefined) { return res.send({ message: 'No file selected!', }); } try { // Read the file content from memory buffer const fileContent = req.file.buffer.toString('utf8'); // Return basic file info as JSON const fileInfo = { filename: req.file.originalname, originalName: req.file.originalname, size: req.file.size, mimetype: req.file.mimetype, content: fileContent.substring(0, 1000), // First 1000 characters }; res.json(fileInfo); } catch (parseError) { logger.error('Error parsing file:', parseError); res.status(500).send({ error: parseError.message }); } }); } catch (error) { logger.error('Error in parseFileHandler:', error); res.status(500).send({ error: error.message }); } };