import { S3Client, HeadBucketCommand, CreateBucketCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, HeadObjectCommand, ListObjectsV2Command, GetObjectCommand as GetObjectCmd, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; import log4js from 'log4js'; import config from '../config.js'; const logger = log4js.getLogger('CephStorage'); logger.level = config.server.logLevel; // Configure AWS SDK v3 for Ceph (S3-compatible) const s3Config = { credentials: { accessKeyId: config.storage.ceph.accessKeyId, secretAccessKey: config.storage.ceph.secretAccessKey, }, endpoint: config.storage.ceph.endpoint, // e.g., 'http://ceph-gateway:7480' forcePathStyle: true, // Required for Ceph (renamed from s3ForcePathStyle) region: config.storage.ceph.region, }; const s3Client = new S3Client(s3Config); // Default bucket names for different file types const BUCKETS = { FILES: config.storage.ceph.filesBucket, }; /** * Initialize buckets if they don't exist */ export const initializeBuckets = async () => { try { logger.info('Initializing Ceph buckets...'); for (const [type, bucketName] of Object.entries(BUCKETS)) { try { await s3Client.send(new HeadBucketCommand({ Bucket: bucketName })); logger.debug(`Bucket ${bucketName} already exists`); } catch (error) { if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) { await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); logger.info(`Created bucket: ${bucketName}`); } else { logger.error(`Error checking bucket ${bucketName}:`, error); } } } logger.info('Ceph buckets initialized successfully.'); } catch (error) { logger.error('Error initializing buckets:', error); throw error; } }; /** * Upload a file to Ceph * @param {string} bucket - Bucket name * @param {string} key - Object key (file path) * @param {Buffer} body - File content * @param {string} contentType - MIME type * @param {Object} metadata - Additional metadata * @returns {Promise} Upload result */ export const uploadFile = async (bucket, key, body, contentType, metadata = {}) => { try { const params = { Bucket: bucket, Key: key, Body: body, ContentType: contentType, Metadata: metadata, }; await s3Client.send(new PutObjectCommand(params)); const result = { Location: `${config.storage.ceph.endpoint}/${bucket}/${key}` }; logger.debug(`File uploaded successfully: ${key} to bucket ${bucket}`); return result; } catch (error) { logger.error(`Error uploading file ${key} to bucket ${bucket}:`, error); throw error; } }; /** * Download a file from Ceph * @param {string} bucket - Bucket name * @param {string} key - Object key (file path) * @returns {Promise} File content */ export const downloadFile = async (bucket, key) => { try { const params = { Bucket: bucket, Key: key, }; const result = await s3Client.send(new GetObjectCommand(params)); logger.debug(`File downloaded successfully: ${key} from bucket ${bucket}`); return result.Body; } catch (error) { if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) { logger.warn(`File not found: ${key} in bucket ${bucket}`); throw new Error('File not found'); } logger.error(`Error downloading file ${key} from bucket ${bucket}:`, error); throw error; } }; /** * Delete a file from Ceph * @param {string} bucket - Bucket name * @param {string} key - Object key (file path) * @returns {Promise} Delete result */ export const deleteFile = async (bucket, key) => { try { const params = { Bucket: bucket, Key: key, }; const result = await s3Client.send(new DeleteObjectCommand(params)); logger.debug(`File deleted successfully: ${key} from bucket ${bucket}`); return result; } catch (error) { logger.error(`Error deleting file ${key} from bucket ${bucket}:`, error); throw error; } }; /** * Check if a file exists in Ceph * @param {string} bucket - Bucket name * @param {string} key - Object key (file path) * @returns {Promise} True if file exists */ export const fileExists = async (bucket, key) => { try { await s3Client.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); return true; } catch (error) { if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) { return false; } logger.error(`Error checking file existence ${key} in bucket ${bucket}:`, error); throw error; } }; /** * List files in a bucket with optional prefix * @param {string} bucket - Bucket name * @param {string} prefix - Optional prefix to filter files * @returns {Promise} List of file objects */ export const listFiles = async (bucket, prefix = '') => { try { const params = { Bucket: bucket, Prefix: prefix, }; const result = await s3Client.send(new ListObjectsV2Command(params)); logger.debug(`Listed ${result.Contents.length} files in bucket ${bucket}`); return result.Contents; } catch (error) { logger.error(`Error listing files in bucket ${bucket}:`, error); throw error; } }; /** * Get file metadata from Ceph * @param {string} bucket - Bucket name * @param {string} key - Object key (file path) * @returns {Promise} File metadata */ export const getFileMetadata = async (bucket, key) => { try { const params = { Bucket: bucket, Key: key, }; const result = await s3Client.send(new HeadObjectCommand(params)); logger.debug(`Retrieved metadata for file: ${key} in bucket ${bucket}`); return result; } catch (error) { if (error.name === 'NotFound' || error.$metadata?.httpStatusCode === 404) { logger.warn(`File not found: ${key} in bucket ${bucket}`); throw new Error('File not found'); } logger.error(`Error getting metadata for file ${key} in bucket ${bucket}:`, error); throw error; } }; /** * Generate a presigned URL for file access * @param {string} bucket - Bucket name * @param {string} key - Object key (file path) * @param {number} expiresIn - URL expiration time in seconds (default: 3600) * @returns {Promise} Presigned URL */ export const getPresignedUrl = async (bucket, key, expiresIn = 3600) => { try { const params = { Bucket: bucket, Key: key, }; const url = await getSignedUrl(s3Client, new GetObjectCommand(params), { expiresIn }); logger.debug(`Generated presigned URL for file: ${key} in bucket ${bucket}`); return url; } catch (error) { logger.error(`Error generating presigned URL for file ${key} in bucket ${bucket}:`, error); throw error; } }; // Export bucket constants for use in other modules export { BUCKETS };