feat: update way to migrate to s3

.talismanrc

@@ -94,7 +94,7 @@ fileignoreconfig:
 - filename: e2e/register-login-organisateur.spec.ts
   checksum: 28122f4f8a3e21312046f7347bdffa570e24c8a38976417d011efc1e5004a0cf
 - filename: package.json
-  checksum: 8437030cbab7db5734c16f9924b7e25a26b9145414d2bd1c81031d2f9f5558bd
+  checksum: 180654b4eb22710ddeba4331c45b63ed92542e989744f390182a90a563eb8e1b
 - filename: packages/backend/src/config.js
   checksum: c0baf7f4f878d3f3406030471f15c092ed9fc671b1e66276fe9b071ea025b58c
 - filename: packages/backend/src/controllers/authentication/email/forgotten-password.js
@@ -257,6 +257,8 @@ fileignoreconfig:
   checksum: ce0b678503f6342414ed759fa31ac460060d1efd1daadffb3d7be0bfa049858c
 - filename: packages/migrations/src/scripts/migrate-files-to-s3.js
   checksum: 0c60a5a9f3321ab3c36025bdeb35663395586d5bca3d21984fdbe5997b46c6eb
+- filename: packages/migrations/src/scripts/reset-s3.js
+  checksum: 3698e6f2fb9b890b55ebc0efbe1ae738c655cbf3cbee900e6d88cc2901b39f57
 - filename: packages/shared/src/components/Chat.vue
   checksum: f2dbbf72bf098c7abd2c3aee230d220f5a5a106952883c0e116eb49f4f9d4db7
 - filename: packages/shared/src/components/DsfrMultiSelect.vue

package.json

@@ -33,5 +33,6 @@
   },
   "workspaces": [
     "packages/*"
-  ]
+  ],
+  "packageManager": "[email protected]+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
 }

packages/backend/src/controllers/documents/adminDownload.js

@@ -2,7 +2,6 @@ const stream = require("stream");
 const DocumentService = require("../../services/Document");
 const AppError = require("../../utils/error");
 const logger = require("../../utils/logger");
-const { decodeFilename } = require("../../utils/file");
 
 const log = logger(module.filename);
 
@@ -19,9 +18,7 @@ module.exports = async (req, res, next) => {
     }
     const readStream = new stream.PassThrough();
     readStream.end(file.file);
-    readStream.end(file.file);
-    const filename = decodeFilename(file.filename);
-    res.set("Content-disposition", `attachment; filename=${filename}`);
+    res.set("Content-disposition", `attachment; filename=${file.filename}`);
     res.set("Content-Type", "text/plain");
     readStream.pipe(res);
     log.i("DONE");

packages/backend/src/controllers/documents/download.js

@@ -2,7 +2,6 @@ const stream = require("stream");
 const DocumentService = require("../../services/Document");
 const AppError = require("../../utils/error");
 const logger = require("../../utils/logger");
-const { decodeFilename } = require("../../utils/file");
 
 const log = logger(module.filename);
 
@@ -28,8 +27,7 @@ module.exports = async (req, res, next) => {
     const file = await DocumentService.getFile(uuid);
     const readStream = new stream.PassThrough();
     readStream.end(file.file);
-    const filename = decodeFilename(metaData.name);
-    res.set("Content-disposition", `attachment; filename=${filename}`);
+    res.set("Content-disposition", `attachment; filename=${metaData.filename}`);
     res.set("Content-Type", "text/plain");
     readStream.pipe(res);
     // // -- next migration step: read files from S3

packages/backend/src/controllers/documents/upload.js

@@ -1,5 +1,4 @@
 const fs = require("node:fs/promises");
-const { fileTypeFromBuffer } = require("file-type");
 const DocumentService = require("../../services/Document");
 const AppError = require("../../utils/error");
 
@@ -24,7 +23,8 @@ module.exports = async (req, res, next) => {
   try {
     const { path, originalname: filename } = file;
     const fileBuffer = await fs.readFile(path);
-    const fileType = await fileTypeFromBuffer(fileBuffer);
+    const fileTypeLib = await import("file-type");
+    const fileType = await fileTypeLib.fileTypeFromBuffer(fileBuffer);
 
     if (!fileType) {
       log.w("DONE with error: Impossible de déterminer le type de fichier");
@@ -68,6 +68,7 @@ module.exports = async (req, res, next) => {
         }),
       );
     }
+
     const uuid = await DocumentService.createFile(
       filename,
       category,

packages/backend/src/services/Document.js

@@ -144,8 +144,9 @@ async function uploadToS3(filename, category, mimetype, userid, data, uuid) {
   log.i("uploadToS3 - In");
   try {
     log.d("uploadToS3", category, filename);
-    const { file, extension } = getFileNameAndExtension(filename);
-    const encodedFilename = encodeFilename(file);
+    const { name, extension } = getFileNameAndExtension(filename);
+
+    const encodedFilename = encodeFilename(name);
 
     await s3Client.send(
       new PutObjectCommand({

packages/backend/src/utils/file.js

@@ -11,7 +11,7 @@ module.exports.getFileNameAndExtension = (fileName) => {
 
   if (dotIndex === -1) {
     // Pas d'extension trouvée
-    return { extension: "", fileName };
+    return { extension: "", name };
   }
 
   const name = fileName.slice(0, dotIndex);

packages/migrations/Dockerfile

@@ -5,6 +5,7 @@ WORKDIR /app
 
 COPY ./package.json ./yarn.lock ./
 COPY packages/migrations/package.json packages/migrations/
+COPY packages/shared/package.json packages/shared/
 RUN yarn workspace @vao/migrations install --frozen-lockfile
 
 #--- Run stage
@@ -17,9 +18,9 @@ ENV NODE_ENV: production
 
 COPY --from=build /app/node_modules node_modules
 COPY package.json yarn.lock ./
-COPY packages/migrations/package.json packages/migrations/yarn.lock packages/migrations/
+COPY packages/migrations/package.json packages/migrations/
 COPY packages/migrations/src packages/migrations/src
 
-CMD ["./node_modules/.bin/knex", "--cwd", "packages/migrations/src", "migrate:latest"]
-
+COPY packages/shared packages/shared
 
+CMD ["./node_modules/.bin/knex", "--cwd", "packages/migrations/src", "migrate:latest"]

packages/migrations/src/scripts/migrate-files-to-s3.js

@@ -58,8 +58,8 @@ async function main() {
     const objectKey = `${S3_BUCKET_ROOT_DIR}/${row.uuid}`;
 
     try {
-      const { filename, extension } = getFileNameAndExtension(row.filename);
-      const encodedFilename = encodeFilename(filename);
+      const { name, extension } = getFileNameAndExtension(row.filename);
+      const encodedFilename = encodeFilename(name);
 
       const metadata = {
         category: String(row.category),

packages/migrations/src/scripts/reset-s3.js

@@ -0,0 +1,44 @@
+const {
+  S3Client,
+  DeleteObjectCommand,
+  ListObjectsV2Command,
+} = require("@aws-sdk/client-s3");
+
+const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME;
+
+async function deleteAllFiles(bucketName) {
+  try {
+    console.log(`Fetching objects from bucket: ${bucketName}`);
+    const listCommand = new ListObjectsV2Command({ Bucket: bucketName });
+    const { Contents } = await S3Client.send(listCommand);
+
+    if (!Contents || Contents.length === 0) {
+      console.log("Bucket is already empty.");
+      return;
+    }
+
+    for (const file of Contents) {
+      const deleteCommand = new DeleteObjectCommand({
+        Bucket: bucketName,
+        Key: file.Key,
+      });
+      await S3Client.send(deleteCommand);
+      console.log(`Deleted: ${file.Key}`);
+    }
+
+    console.log("All files have been deleted successfully.");
+  } catch (err) {
+    console.error("Error deleting files:", err);
+  }
+}
+
+async function main() {
+  if (!S3_BUCKET_NAME) {
+    console.error("S3_BUCKET_NAME is not defined in environment variables.");
+    return;
+  }
+
+  await deleteAllFiles(S3_BUCKET_NAME);
+}
+
+main();