import config from '../../config.js'; import { fileModel } from '../../database/schemas/management/file.schema.js'; import log4js from 'log4js'; import multer from 'multer'; import path from 'path'; import fs from 'fs'; import mongoose from 'mongoose'; import { deleteObject, listObjects, getObject, editObject, newObject, listObjectsByProperties, flushFile, getModelStats, getModelHistory, searchObjects, getPropertyValues, } from '../../database/database.js'; import { uploadFile, downloadFile, deleteFile as deleteCephFile, BUCKETS, } from '../../database/ceph.js'; import { getFileMeta } from '../../utils.js'; const logger = log4js.getLogger('Files'); logger.level = config.server.logLevel; const getDownloadContentType = (file) => { // G-code is text, but it must be transferred byte-for-byte. Some slicers // report text/plain, causing proxies to compress an otherwise fixed-length // response and browsers to reject it with a content-length network error. if (file.extension?.toLowerCase() === '.gcode') { return 'application/octet-stream'; } return file.type || 'application/octet-stream'; }; // Set storage engine to memory for Ceph upload const fileStorage = multer.memoryStorage(); // Initialise upload const fileUpload = multer({ storage: fileStorage, limits: { fileSize: 500000000 }, // 500MB limit fileFilter: function (req, file, cb) { checkFileType(file, cb); }, }).single('file'); // The name attribute of the file input in the HTML form export const listFilesRouteHandler = async ( req, res, page = 1, limit = 25, property = '', filter = {}, search = '', sort = '', order = 'ascend' ) => { const result = await listObjects({ model: fileModel, page, limit, property, filter, search, sort, order, }); if (result?.error) { logger.error('Error listing files.'); res.status(result.code).send(result); return; } logger.debug(`List of files (Page ${page}, Limit ${limit}). Count: ${result.length}`); res.send(result); }; export const listFilesByPropertiesRouteHandler = async ( req, res, properties = '', filter = {}, masterFilter = {} ) => { const result = await listObjectsByProperties({ model: fileModel, properties, filter, masterFilter, }); if (result?.error) { logger.error('Error listing files.'); res.status(result.code).send(result); return; } logger.debug(`List of files. Count: ${result.length}`); res.send(result); }; export const getFilePropertyValuesRouteHandler = async (req, res, property) => { const result = await getPropertyValues({ model: fileModel, property, }); res.send(result); }; export const searchFilesRouteHandler = async (req, res, search) => { const result = await searchObjects({ model: fileModel, search, }); res.send(result); }; export const getFileRouteHandler = async (req, res) => { const id = req.params.id; const result = await getObject({ model: fileModel, id, }); if (result?.error) { logger.warn(`File not found with supplied id.`); return res.status(result.code).send(result); } logger.debug(`Retreived file with ID: ${id}`); res.send(result); }; export const flushFileRouteHandler = async (req, res) => { const id = req.params.id; const result = await flushFile({ user: req.user, id }); res.send(result); }; export const editFileRouteHandler = async (req, res) => { // Get ID from params const id = new mongoose.Types.ObjectId(req.params.id); logger.trace(`File with ID: ${id}`); const updateData = { updatedAt: new Date(), name: req.body.name, }; const result = await editObject({ model: fileModel, id, updateData, user: req.user, }); if (result.error) { logger.error('Error editing file:', result.error); res.status(result).send(result); return; } logger.debug(`Edited file with ID: ${id}`); res.send(result); }; export const newFileRouteHandler = async (req, res) => { try { fileUpload(req, res, async (err) => { if (err) { return res.status(500).send({ error: err, }); } if (req.file == undefined) { return res.send({ message: 'No file selected!', }); } try { // Create DB entry first without storage fields const extension = path.extname(req.file.originalname); const baseName = path.parse(req.file.originalname).name; const meta = await getFileMeta(req.file); const newData = { name: baseName, type: req.file.mimetype, extension, size: req.file.size, updatedAt: new Date(), metaData: { originalName: req.file.originalname, ...meta, }, }; const created = await newObject({ model: fileModel, newData, user: req.user, }); if (created.error) { logger.error('No file created:', created.error); return res.status(created.code).send(created); } // Use created document _id to generate Ceph key const cephKey = `files/${created._id}${extension}`; // Upload file to Ceph await uploadFile(BUCKETS.FILES, cephKey, req.file.buffer, req.file.mimetype, { originalName: req.file.originalname, uploadedBy: req.user?.username || 'unknown', }); // Do not update DB with Ceph location. Return created DB record. logger.debug(`New file with ID: ${created._id} created and uploaded to Ceph`); res.send(created); } catch (createError) { logger.error('Error creating file record or uploading to storage:', createError); // If we created the DB entry but upload failed, remove the DB entry to avoid orphaned records try { if (created && created._id) { await deleteObject({ model: fileModel, id: created._id, user: req.user }); } } catch (cleanupDbError) { logger.error('Error cleaning up DB record after upload failure:', cleanupDbError); } res.status(500).send({ error: createError.message }); } }); } catch (error) { logger.error('Error in newFileRouteHandler:', error); res.status(500).send({ error: error.message }); } }; export const deleteFileRouteHandler = async (req, res) => { // Get ID from params const id = new mongoose.Types.ObjectId(req.params.id); logger.trace(`File with ID: ${id}`); try { // First get the file to retrieve Ceph information const file = await getObject({ model: fileModel, id, }); if (!file) { return res.status(404).send({ error: 'File not found' }); } // Delete from Ceph if it exists there if (file.cephBucket && file.cephKey) { try { await deleteCephFile(file.cephBucket, file.cephKey); logger.debug(`Deleted file from Ceph: ${file.cephKey}`); } catch (cephError) { logger.warn(`Failed to delete file from Ceph: ${cephError.message}`); // Continue with database deletion even if Ceph deletion fails } } const result = await deleteObject({ model: fileModel, id, user: req.user, }); if (result.error) { logger.error('No file deleted:', result.error); return res.status(result.code).send(result); } logger.debug(`Deleted file with ID: ${result._id}`); res.send(result); } catch (error) { logger.error('Error in deleteFileRouteHandler:', error); res.status(500).send({ error: error.message }); } }; // Check file type function checkFileType(file, cb) { // Allow all file types for general file management // You can customize this to restrict specific file types if needed const allowedTypes = /.*/; // Allow all file types if (allowedTypes.test(file.mimetype)) { return cb(null, true); } else { cb('Error: File type not allowed!'); } } export const getFileContentRouteHandler = async (req, res) => { const id = req.params.id; let bytesSent = 0; let sourceEnded = false; logger.info(`File download requested: ${id}`); req.once('aborted', () => { logger.warn(`File download request aborted by client: ${id} (${bytesSent} bytes sent)`); }); res.once('finish', () => { logger.info( `File download response finished: ${id} (status ${res.statusCode}, ${bytesSent} bytes sent)` ); }); res.once('close', () => { if (!res.writableFinished) { logger.warn( `File download response closed early: ${id} (${bytesSent} bytes sent, source ended: ${sourceEnded})` ); } }); try { const file = await getObject({ model: fileModel, id, }); if (!file) { logger.warn(`File not found with supplied id.`); return res.status(404).send({ error: 'File not found.' }); } logger.trace(`Returning file contents with ID: ${id}:`); logger.debug(`File record loaded for download: ${id}`, { name: file.name, extension: file.extension, recordedSize: file.size, type: file.type, }); // Check if file is stored in Ceph if (file._id && file.extension) { const cephKey = `files/${id}${file.extension}`; logger.info(`Starting Ceph download: ${cephKey} (recorded size: ${file.size ?? 'unknown'})`); try { const body = await downloadFile(BUCKETS.FILES, cephKey); logger.debug(`Ceph response stream opened: ${cephKey}`, { readable: body?.readable, destroyed: body?.destroyed, }); // Preserve the stored bytes across reverse proxies. This also repairs // downloads of existing slicer uploads recorded as text/plain. res.set('Content-Type', getDownloadContentType(file)); res.set('Content-Disposition', `attachment; filename="${file.name}${file.extension}"`); res.set('Cache-Control', 'private, no-transform'); // Stream or send buffer if (body && typeof body.pipe === 'function') { body.on('data', (chunk) => { bytesSent += chunk.length; }); body.on('end', () => { sourceEnded = true; logger.info(`Ceph stream ended: ${cephKey} (${bytesSent} bytes read)`); if ( typeof file.size === 'number' && !Number.isNaN(file.size) && bytesSent !== file.size ) { logger.warn( `File size mismatch for ${cephKey}: database=${file.size}, streamed=${bytesSent}` ); } }); body.on('close', () => { logger.debug( `Ceph stream closed: ${cephKey} (${bytesSent} bytes read, ended: ${sourceEnded})` ); }); // Handle stream errors body.on('error', (err) => { logger.error( `Error streaming file from Ceph: ${cephKey} (${bytesSent} bytes read)`, err ); // If headers not sent, send a 500; otherwise destroy the response if (!res.headersSent) { try { res.status(500).send({ error: 'Error streaming file from storage.' }); } catch (_) { // Ignore secondary errors } } else { res.destroy(err); } }); // If client disconnects, stop reading from source res.on('close', () => { if (typeof body.destroy === 'function') { body.destroy(); } }); body.pipe(res); logger.debug(`Ceph stream piped to response: ${cephKey}`); return; } // Unknown body type logger.error('Unknown Ceph body type; cannot send response'); return res.status(500).send({ error: 'Error reading file from storage.' }); } catch (cephError) { logger.error('Error downloading file from Ceph:', cephError); // Fall through to local filesystem fallback below } // Fallback to local file system for backward compatibility const filePath = path.join(config.storage.fileStorage, file.fileName || file.name); logger.info(`Falling back to local file download: ${filePath}`); // Read the file fs.readFile(filePath, (err, data) => { if (err) { if (err.code === 'ENOENT') { return res.status(404).send({ error: 'File not found!' }); } return res.status(500).send({ error: 'Error reading file.' }); } bytesSent = data.length; sourceEnded = true; logger.info(`Local file read complete: ${filePath} (${data.length} bytes)`); res.set('Content-Type', getDownloadContentType(file)); res.set('Content-Disposition', `inline; filename="${file.name}${file.extension || ''}"`); res.set('Cache-Control', 'private, no-transform'); // Ensure Content-Length is set for progress events if possible. const length = typeof file.size === 'number' && !Number.isNaN(file.size) ? file.size : data.length; res.set('Content-Length', String(length)); return res.send(data); }); } else { logger.error('Error fetching file:', 'No ceph bucket or key supplied.'); res.status(500).send({ error: 'No ceph bucket or key supplied.' }); } } catch (error) { logger.error('Error fetching file:', error); res.status(500).send({ error: error.message }); } }; export const parseFileHandler = async (req, res) => { try { // Use the same upload middleware as the uploadFileContentRouteHandler fileUpload(req, res, async (err) => { if (err) { return res.status(500).send({ error: err, }); } if (req.file == undefined) { return res.send({ message: 'No file selected!', }); } try { // Read the file content from memory buffer const fileContent = req.file.buffer.toString('utf8'); // Return basic file info as JSON const fileInfo = { filename: req.file.originalname, originalName: req.file.originalname, size: req.file.size, mimetype: req.file.mimetype, content: fileContent.substring(0, 1000), // First 1000 characters }; res.json(fileInfo); } catch (parseError) { logger.error('Error parsing file:', parseError); res.status(500).send({ error: parseError.message }); } }); } catch (error) { logger.error('Error in parseFileHandler:', error); res.status(500).send({ error: error.message }); } }; export const getFileStatsRouteHandler = async (req, res) => { const result = await getModelStats({ model: fileModel }); if (result?.error) { logger.error('Error fetching file stats:', result.error); return res.status(result.code).send(result); } logger.trace('File stats:', result); res.send(result); }; export const getFileHistoryRouteHandler = async (req, res) => { const from = req.query.from; const to = req.query.to; const result = await getModelHistory({ model: fileModel, from, to }); if (result?.error) { logger.error('Error fetching file history:', result.error); return res.status(result.code).send(result); } logger.trace('File history:', result); res.send(result); };