🛠️ Cozy migration CLI tools and client (#876)

* feat: migration cli tools and client

* feat: creating folder tree / ref cli cmds / Dockerfile

* feat: refresh app auth token for requests

* feat: skip already migrated files

* ref: auth token refresh and download request

* ref: using cloudery to create user instace and generate token

* ref: using config for cloudery

* fix: docker for backend

* fix: docker for backend

* fix: docker for backend

* fix: docker for backend

* fix: lint/Dockerfile

* fix: docker for backend

* fix: docker for backend

* ref: delete client

* fix: user default company

* feat: stream file upload and progress

* feat: use default company

* ref: file upload

* ref: removed migration endpoints for external client

tdrive/backend/node/config/custom-environment-variables.json

@@ -162,5 +162,12 @@
     "timeout": "AV_TIMEOUT",
     "maxFileSize": "AV_MAX_FILE_SIZE",
     "deleteInfectedFiles": "AV_DELETE_INFECTED_FILES"
+  },
+  "migration": {
+    "cozyDomain": "COZY_DOMAIN",
+    "cozyManagerUrl": "COZY_MANAGER_URL",
+    "cozyManagerToken": "COZY_MANAGER_TOKEN",
+    "pollInterval": "MIGRATUION_POLL_INTERVAL",
+    "maxRetries": "MIGRATION_MAX_RETRIES"
   }
 }

tdrive/backend/node/config/default.json

@@ -149,6 +149,13 @@
     "maxFileSize": 4294967295,
     "deleteInfectedFiles": false
   },
+  "migration": {
+    "cozyDomain": "stg.lin-saas.com",
+    "cozyManagerUrl": "https://manager-int.cozycloud.cc/api/public",
+    "cozyManagerToken": "COZY_TOKEN_HERE",
+    "pollInterval": 5000,
+    "maxRetries": 3
+  },
   "applications": {
     "grid": [
       {

tdrive/backend/node/package.json

@@ -136,6 +136,11 @@
     "class-transformer": "^0.3.1",
     "cli-table": "^0.3.6",
     "config": "^3.3.2",
+    "cozy-client": "^57.7.0",
+    "cozy-device-helper": "^3.8.0",
+    "cozy-flags": "^4.7.0",
+    "cozy-intent": "^2.30.0",
+    "cozy-logger": "^1.17.0",
     "deep-object-diff": "^1.1.0",
     "emoji-name-map": "^1.2.9",
     "eta": "^2.2.0",

tdrive/backend/node/src/@types/react-override.d.ts

@@ -0,0 +1,7 @@
+// src/types/react-override.d.ts
+
+import "react";
+
+declare module "react" {
+  export type ReactChild = React.ReactNode;
+}

tdrive/backend/node/src/cli/cmds/migration.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Migrate data from Twake Drive",
+  command: "migrate <command>",
+  builder: yargs =>
+    yargs.commandDir("migration_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/migration_cmds/migrate-files.ts

@@ -0,0 +1,173 @@
+import runWithPlatform from "../../lib/run-with-platform";
+import runWithLoggerLevel from "../../utils/run-with-logger-level";
+import globalResolver from "../../../services/global-resolver";
+import User from "../../../services/user/entities/user";
+import yargs from "yargs";
+import { DriveFile, TYPE } from "../../../services/documents/entities/drive-file";
+import { getPath } from "../../../services/documents/utils";
+import CozyClient from "cozy-client";
+import {
+  uploadFile,
+  COZY_DOMAIN,
+  DEFAULT_COMPANY,
+  getDriveToken,
+  nodeReadableToWebReadable,
+} from "./utils";
+
+const purgeIndexesCommand: yargs.CommandModule<unknown, unknown> = {
+  command: "migrate-files",
+  describe: "Migrates files data from Twake Drive",
+  builder: {
+    dryRun: {
+      type: "boolean",
+      alias: "d",
+      description: "Simulate the migration and returns stats for the db.",
+      default: true,
+    },
+    emails: {
+      type: "string",
+      alias: "e",
+      description: "Comma-separated list of user emails to migrate files for specific users",
+    },
+  },
+  handler: async argv => {
+    const dryRun = argv.dryRun as boolean;
+    console.log("DRY RUN: ", dryRun);
+    const emailsArg = argv.emails as string | undefined;
+    const specifiedEmails = emailsArg
+      ? emailsArg.split(",").map(email => email.trim().toLowerCase())
+      : null;
+
+    if (specifiedEmails) {
+      console.log("Migrating only specified emails:", specifiedEmails.join(", "));
+    }
+
+    await runWithPlatform("Migrate files", async () => {
+      return await runWithLoggerLevel("fatal", async () => {
+        const usersRepo = await globalResolver.database.getRepository<User>("user", User);
+        const documentsRepo = await globalResolver.database.getRepository<DriveFile>(
+          TYPE,
+          DriveFile,
+        );
+
+        const allUsers = await (await usersRepo.find({})).getEntities();
+
+        let usersToMigrate = allUsers;
+        if (specifiedEmails) {
+          usersToMigrate = allUsers.filter(user =>
+            specifiedEmails.includes(user.email_canonical.toLowerCase()),
+          );
+        }
+
+        for (const user of usersToMigrate) {
+          const userCompany = DEFAULT_COMPANY;
+          const userFiles = await documentsRepo.find({ creator: user.id, is_directory: false });
+          const userId = user.email_canonical.split("@")[0];
+
+          console.log(`User ${user.id} has ${userFiles.getEntities().length} files`);
+
+          const userFilesObjects = [];
+
+          for (const userFile of userFiles.getEntities()) {
+            if (userFile.migrated) {
+              continue;
+            }
+            const filePathItems = await getPath(userFile.id, documentsRepo, true, {
+              company: { id: userCompany },
+            } as any);
+
+            const filePath = filePathItems
+              .slice(1, -1)
+              .map(p => p.name)
+              .join("/");
+
+            const fileObject = {
+              owner: userId,
+              _id: userFile.id,
+              is_in_trash: userFile.is_in_trash,
+              is_directory: userFile.is_directory,
+              name: userFile.name,
+              added: userFile.added,
+              last_modified: userFile.last_modified,
+              size: userFile.size,
+              path: filePath !== "My Drive" ? filePath : "",
+              company_id: userCompany,
+            };
+
+            userFilesObjects.push(fileObject);
+
+            if (!dryRun) {
+              try {
+                const cozyUrl = `${userId}.${COZY_DOMAIN}`;
+                const userToken = await getDriveToken(cozyUrl);
+                const client = new CozyClient({
+                  uri: `https://${cozyUrl}`,
+                  token: userToken.token,
+                });
+
+                let fileDirPath = "io.cozy.files.root-dir";
+                if (fileObject.path !== "") {
+                  const sanitizedPath = fileObject.path.replace(/^\//, "");
+                  fileDirPath = (
+                    await client.collection("io.cozy.files").createDirectoryByPath(sanitizedPath)
+                  ).data.id;
+                }
+                // 2. Download file from backend
+                const archiveOrFile = await globalResolver.services.documents.documents.download(
+                  userFile.id,
+                  userFile.last_version_cache.id,
+                  null, // No archive callback needed
+                  {
+                    company: { id: userCompany },
+                    user: { id: user.id },
+                  } as any,
+                );
+
+                if (!archiveOrFile.file) {
+                  console.error(`File ${userFile.id} was returned as archive. Skipping.`);
+                  continue;
+                }
+                let uploadedBytes = 0;
+                const totalSize = fileObject.size || 0;
+                const { file: fileStream } = archiveOrFile.file;
+                const fileReadable = nodeReadableToWebReadable(fileStream, chunkSize => {
+                  uploadedBytes += chunkSize;
+                  const percentage =
+                    totalSize > 0 ? ((uploadedBytes / totalSize) * 100).toFixed(2) : "0";
+                  process.stdout.write(`\rUploading ${fileObject.name}... ${percentage}%`);
+                });
+
+                const resp = await uploadFile(
+                  fileObject.name,
+                  userId,
+                  fileDirPath,
+                  userToken.token,
+                  fileReadable,
+                );
+
+                if (!resp.ok) {
+                  console.error(`❌ ERROR UPLOADING THE FILE: ${fileObject.name}`);
+                  console.error(`❌ ERROR: ${JSON.stringify(resp)}  ${resp}`);
+                  continue;
+                }
+                // 3. Migrate file
+                userFile.migrated = true;
+                userFile.migration_date = Date.now();
+                await documentsRepo.save(userFile);
+
+                console.log(`\n✅ File migrated successfully: ${fileObject.name}`);
+              } catch (error) {
+                console.error(`❌ ERROR CREATING THE FILE: ${fileObject.name}`);
+                console.error(`❌ ERROR: ${JSON.stringify(error)}  ${error}`);
+              }
+            } else {
+              console.log(`[DRY-RUN] Would create Cozy instance for user ${user.email_canonical}`);
+            }
+          }
+        }
+      });
+    });
+  },
+};
+
+export default purgeIndexesCommand;

tdrive/backend/node/src/cli/cmds/migration_cmds/migrate-users.ts

@@ -0,0 +1,85 @@
+import runWithPlatform from "../../lib/run-with-platform";
+import runWithLoggerLevel from "../../utils/run-with-logger-level";
+import globalResolver from "../../../services/global-resolver";
+import User from "../../../services/user/entities/user";
+import yargs from "yargs";
+import { createCozyInstance } from "./utils";
+
+const migrateUsersCommand: yargs.CommandModule<unknown, unknown> = {
+  command: "migrate-users",
+  describe: "Migrates users data from Twake Drive",
+  builder: {
+    dryRun: {
+      type: "boolean",
+      alias: "d",
+      description: "Simulate the migration and return stats for the DB.",
+      default: true,
+    },
+    emails: {
+      type: "string",
+      alias: "e",
+      description: "Comma-separated list of user emails to migrate specific users",
+    },
+  },
+  handler: async argv => {
+    const dryRun = argv.dryRun as boolean;
+    const emailsArg = argv.emails as string | undefined;
+    const specifiedEmails = emailsArg
+      ? emailsArg.split(",").map(email => email.trim().toLowerCase())
+      : null;
+
+    console.log("DRY RUN:", dryRun);
+    if (specifiedEmails) {
+      console.log("Migrating only specified emails:", specifiedEmails.join(", "));
+    }
+
+    await runWithPlatform("Migrate users", async () => {
+      await runWithLoggerLevel("info", async () => {
+        const usersRepo = await globalResolver.database.getRepository<User>("user", User);
+
+        const allUsers = await (await usersRepo.find({})).getEntities();
+        const migratedUsers = await (await usersRepo.find({ migrated: true })).getEntities();
+        const pendingUsers = allUsers.filter(
+          user => !migratedUsers.some(migratedUser => migratedUser.id === user.id),
+        );
+
+        let usersToMigrate = pendingUsers;
+        if (specifiedEmails) {
+          usersToMigrate = pendingUsers.filter(user =>
+            specifiedEmails.includes(user.email_canonical.toLowerCase()),
+          );
+        }
+
+        if (usersToMigrate.length === 0) {
+          console.log("✅ No users to migrate.");
+          return;
+        }
+
+        console.log(`🚀 Users to migrate: ${usersToMigrate.length}`);
+
+        for (const user of usersToMigrate) {
+          const userId = user.email_canonical.split("@")[0];
+          const userObject = {
+            _id: user.id,
+            id: userId,
+            email: user.email_canonical,
+            name: `${user.first_name} ${user.last_name}`,
+          };
+
+          if (!dryRun) {
+            try {
+              await createCozyInstance(userObject);
+            } catch (error) {
+              console.error(`❌ Failed to migrate user ${user.email_canonical}`, error);
+              // Even if error, continue to next user
+            }
+          } else {
+            console.log(`[DRY-RUN] Would create Cozy instance for user ${user.email_canonical}`);
+          }
+        }
+      });
+    });
+  },
+};
+
+export default migrateUsersCommand;

tdrive/backend/node/src/cli/cmds/migration_cmds/utils/index.ts

@@ -0,0 +1,167 @@
+/* eslint-disable prettier/prettier */
+import axios from "axios";
+import config from "config";
+
+export const DEFAULT_COMPANY = config.get<string>("drive.defaultCompany");
+export const COZY_DOMAIN = config.get<string>("migration.cozyDomain");
+const COZY_MANAGER_URL = config.get<string>("migration.cozyManagerUrl");
+const COZY_MANAGER_TOKEN = config.get<string>("migration.cozyManagerToken");
+const POLL_INTERVAL_MS = config.get<number>("migration.pollInterval");
+const MAX_RETRIES = config.get<number>("migration.maxRetries");
+
+function buildUploadUrl(baseUrl, params) {
+  const searchParams = new URLSearchParams(params);
+  return `${baseUrl}?${searchParams.toString()}`;
+}
+
+export function nodeReadableToWebReadable(readable, onProgress?: (chunkSize: number) => void) {
+  const reader = readable[Symbol.asyncIterator]();
+  return new ReadableStream({
+    async pull(controller) {
+      const { value, done } = await reader.next();
+      if (done) {
+        controller.close();
+      } else {
+        if (onProgress) onProgress(value.length); // report progress
+        controller.enqueue(value);
+      }
+    },
+    cancel() {
+      reader.return?.();
+    },
+  });
+}
+
+export async function createCozyInstance(user: {
+  id: string;
+  email: string;
+  name: string;
+  _id: string;
+}) {
+  console.log(`🚀 Creating Cozy instance for ${user.email}...`);
+
+  try {
+    // Step 1: Create the instance
+    const { data: createInstanceData } = await axios.post(
+      `${COZY_MANAGER_URL}/instances`,
+      {
+        offer: "twake",
+        slug: user.id,
+        domain: COZY_DOMAIN,
+        email: user.email,
+        public_name: user.name,
+        locale: "fr",
+        oidc: user.id,
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${COZY_MANAGER_TOKEN}`,
+          "Content-Type": "application/json",
+        },
+      },
+    );
+
+    const workflowId = createInstanceData.workflow;
+    console.log(`✅ Created instance for ${user.email}, workflow ID: ${workflowId}`);
+
+    // Step 2: Check workflow status
+    const checkWorkflowStatus = async () => {
+      const { data } = await axios.get(`${COZY_MANAGER_URL}/workflows/${workflowId}`, {
+        headers: {
+          Authorization: `Bearer ${COZY_MANAGER_TOKEN}`,
+          "Content-Type": "application/json",
+        },
+      });
+      return data.status;
+    };
+
+    let attempt = 0;
+    let finished = false;
+
+    while (attempt < MAX_RETRIES && !finished) {
+      attempt++;
+      console.log(
+        `🔄 Checking workflow status for ${user.email} (attempt ${attempt}/${MAX_RETRIES})...`,
+      );
+
+      const status = await checkWorkflowStatus();
+
+      if (status === "finished") {
+        console.log(`🎉 Workflow finished for ${user.email}`);
+        finished = true;
+      } else {
+        console.log(
+          `⏳ Workflow not finished yet for ${user.email}, waiting ${POLL_INTERVAL_MS / 1000}s...`,
+        );
+        await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL_MS));
+      }
+    }
+
+    if (!finished) {
+      console.warn(
+        `⚠️ Workflow for ${user.email} did not finish after ${MAX_RETRIES} tries. Moving on.`,
+      );
+    }
+  } catch (error: any) {
+    if (error.response) {
+      console.error(
+        `❌ Error migrating ${user.email}:`,
+        error.response.status,
+        error.response.data,
+      );
+    } else {
+      console.error(`❌ Error migrating ${user.email}:`, error.message);
+    }
+    throw error;
+  }
+}
+
+export async function getDriveToken(slugDomain: string): Promise<{ token: string }> {
+  const url = `${COZY_MANAGER_URL}/instances/${slugDomain}/drive_token`;
+
+  try {
+    const response = await axios.post(url, null, {
+      headers: {
+        Authorization: `Bearer ${COZY_MANAGER_TOKEN}`,
+        "Content-Type": "application/json",
+      },
+    });
+
+    return response.data; // Should be { token: "..." }
+  } catch (error: any) {
+    console.error(
+      `Failed to get drive token for ${slugDomain}`,
+      error.response?.data || error.message,
+      url,
+    );
+    throw new Error(`Could not retrieve drive token for ${slugDomain}`);
+  }
+}
+
+export async function uploadFile(
+  fileName: string,
+  userId: string,
+  fileDirPath: string,
+  userToken: string,
+  fileReadable: ReadableStream<any>,
+) {
+  const baseUrl = `https://${userId}.${COZY_DOMAIN}/files/${fileDirPath}`;
+  const params = {
+    Name: fileName,
+    Type: "file",
+    Executable: false,
+    Encrypted: false,
+    Size: "",
+  };
+  const uploadUrl = buildUploadUrl(baseUrl, params);
+  const resp = await fetch(uploadUrl, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${userToken}`,
+      "Content-Type": "application/octet-stream",
+    },
+    duplex: "half",
+    body: fileReadable,
+  } as RequestInit);
+  return resp;
+}

tdrive/backend/node/src/services/documents/entities/drive-file.ts

@@ -109,6 +109,12 @@ export class DriveFile {
   @Column("access_info", "encoded_json")
   access_info: AccessInformation;
 
+  @Column("migrated", "tdrive_boolean")
+  migrated: boolean;
+
+  @Column("migration_date", "number")
+  migration_date: number;
+
   /**
    * If this field is non-null, then an editing session is in progress (probably in OnlyOffice).
    * Use {@see EditingSessionKeyFormat} to generate and interpret it.

tdrive/backend/node/src/services/user/entities/user.ts

@@ -82,6 +82,12 @@ export default class User {
   @Column("deleted", "tdrive_boolean")
   deleted: boolean;
 
+  @Column("migrated", "tdrive_boolean")
+  migrated: boolean;
+
+  @Column("migration_date", "number")
+  migration_date: number;
+
   /**
    * If set, a restartable suppression process is currently incomplete.
    */

tdrive/docker/tdrive-node/Dockerfile

@@ -16,7 +16,7 @@ COPY backend/node/package*.json ./
 # Test Stage
 FROM node-base AS test
 
-RUN npm install
+RUN npm install --legacy-peer-deps
 COPY backend/node/ .
 
 # Add frontend Stage
@@ -25,7 +25,7 @@ FROM node-base AS installed-libs
 COPY backend/node/ .
 #Install dev dependancies for build
 ENV NODE_ENV=development
-RUN npm ci
+RUN npm ci --legacy-peer-deps
 
 #Build in production mode
 ENV NODE_ENV=production
@@ -40,7 +40,7 @@ FROM installed-libs AS development
 ENV NODE_ENV=development
 RUN npm install -g pino-pretty && \
     npm install -g tsc-watch && \
-    npm ci
+    npm ci --legacy-peer-deps
 CMD ["npm", "run", "dev:debug"]
 
 # Production Stage