🛠️ Cozy migration CLI tools and client (#876)
* feat: migration cli tools and client * feat: creating folder tree / ref cli cmds / Dockerfile * feat: refresh app auth token for requests * feat: skip already migrated files * ref: auth token refresh and download request * ref: using cloudery to create user instace and generate token * ref: using config for cloudery * fix: docker for backend * fix: docker for backend * fix: docker for backend * fix: docker for backend * fix: lint/Dockerfile * fix: docker for backend * fix: docker for backend * ref: delete client * fix: user default company * feat: stream file upload and progress * feat: use default company * ref: file upload * ref: removed migration endpoints for external client
This commit is contained in:
Montassar Ghanmy
@@ -162,5 +162,12 @@
|
||||
"timeout": "AV_TIMEOUT",
|
||||
"maxFileSize": "AV_MAX_FILE_SIZE",
|
||||
"deleteInfectedFiles": "AV_DELETE_INFECTED_FILES"
|
||||
},
|
||||
"migration": {
|
||||
"cozyDomain": "COZY_DOMAIN",
|
||||
"cozyManagerUrl": "COZY_MANAGER_URL",
|
||||
"cozyManagerToken": "COZY_MANAGER_TOKEN",
|
||||
"pollInterval": "MIGRATUION_POLL_INTERVAL",
|
||||
"maxRetries": "MIGRATION_MAX_RETRIES"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,6 +149,13 @@
|
||||
"maxFileSize": 4294967295,
|
||||
"deleteInfectedFiles": false
|
||||
},
|
||||
"migration": {
|
||||
"cozyDomain": "stg.lin-saas.com",
|
||||
"cozyManagerUrl": "https://manager-int.cozycloud.cc/api/public",
|
||||
"cozyManagerToken": "COZY_TOKEN_HERE",
|
||||
"pollInterval": 5000,
|
||||
"maxRetries": 3
|
||||
},
|
||||
"applications": {
|
||||
"grid": [
|
||||
{
|
||||
|
||||
Generated
+5353
-733
File diff suppressed because it is too large
Load Diff
@@ -136,6 +136,11 @@
|
||||
"class-transformer": "^0.3.1",
|
||||
"cli-table": "^0.3.6",
|
||||
"config": "^3.3.2",
|
||||
"cozy-client": "^57.7.0",
|
||||
"cozy-device-helper": "^3.8.0",
|
||||
"cozy-flags": "^4.7.0",
|
||||
"cozy-intent": "^2.30.0",
|
||||
"cozy-logger": "^1.17.0",
|
||||
"deep-object-diff": "^1.1.0",
|
||||
"emoji-name-map": "^1.2.9",
|
||||
"eta": "^2.2.0",
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
// src/types/react-override.d.ts
|
||||
|
||||
import "react";
|
||||
|
||||
declare module "react" {
|
||||
export type ReactChild = React.ReactNode;
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { CommandModule } from "yargs";
|
||||
|
||||
const command: CommandModule = {
|
||||
describe: "Migrate data from Twake Drive",
|
||||
command: "migrate <command>",
|
||||
builder: yargs =>
|
||||
yargs.commandDir("migration_cmds", {
|
||||
visit: commandModule => commandModule.default,
|
||||
}),
|
||||
// eslint-disable-next-line @typescript-eslint/no-empty-function
|
||||
handler: () => {},
|
||||
};
|
||||
|
||||
export default command;
|
||||
@@ -0,0 +1,173 @@
|
||||
import runWithPlatform from "../../lib/run-with-platform";
|
||||
import runWithLoggerLevel from "../../utils/run-with-logger-level";
|
||||
import globalResolver from "../../../services/global-resolver";
|
||||
import User from "../../../services/user/entities/user";
|
||||
import yargs from "yargs";
|
||||
import { DriveFile, TYPE } from "../../../services/documents/entities/drive-file";
|
||||
import { getPath } from "../../../services/documents/utils";
|
||||
import CozyClient from "cozy-client";
|
||||
import {
|
||||
uploadFile,
|
||||
COZY_DOMAIN,
|
||||
DEFAULT_COMPANY,
|
||||
getDriveToken,
|
||||
nodeReadableToWebReadable,
|
||||
} from "./utils";
|
||||
|
||||
const purgeIndexesCommand: yargs.CommandModule<unknown, unknown> = {
|
||||
command: "migrate-files",
|
||||
describe: "Migrates files data from Twake Drive",
|
||||
builder: {
|
||||
dryRun: {
|
||||
type: "boolean",
|
||||
alias: "d",
|
||||
description: "Simulate the migration and returns stats for the db.",
|
||||
default: true,
|
||||
},
|
||||
emails: {
|
||||
type: "string",
|
||||
alias: "e",
|
||||
description: "Comma-separated list of user emails to migrate files for specific users",
|
||||
},
|
||||
},
|
||||
handler: async argv => {
|
||||
const dryRun = argv.dryRun as boolean;
|
||||
console.log("DRY RUN: ", dryRun);
|
||||
const emailsArg = argv.emails as string | undefined;
|
||||
const specifiedEmails = emailsArg
|
||||
? emailsArg.split(",").map(email => email.trim().toLowerCase())
|
||||
: null;
|
||||
|
||||
if (specifiedEmails) {
|
||||
console.log("Migrating only specified emails:", specifiedEmails.join(", "));
|
||||
}
|
||||
|
||||
await runWithPlatform("Migrate files", async () => {
|
||||
return await runWithLoggerLevel("fatal", async () => {
|
||||
const usersRepo = await globalResolver.database.getRepository<User>("user", User);
|
||||
const documentsRepo = await globalResolver.database.getRepository<DriveFile>(
|
||||
TYPE,
|
||||
DriveFile,
|
||||
);
|
||||
|
||||
const allUsers = await (await usersRepo.find({})).getEntities();
|
||||
|
||||
let usersToMigrate = allUsers;
|
||||
if (specifiedEmails) {
|
||||
usersToMigrate = allUsers.filter(user =>
|
||||
specifiedEmails.includes(user.email_canonical.toLowerCase()),
|
||||
);
|
||||
}
|
||||
|
||||
for (const user of usersToMigrate) {
|
||||
const userCompany = DEFAULT_COMPANY;
|
||||
const userFiles = await documentsRepo.find({ creator: user.id, is_directory: false });
|
||||
const userId = user.email_canonical.split("@")[0];
|
||||
|
||||
console.log(`User ${user.id} has ${userFiles.getEntities().length} files`);
|
||||
|
||||
const userFilesObjects = [];
|
||||
|
||||
for (const userFile of userFiles.getEntities()) {
|
||||
if (userFile.migrated) {
|
||||
continue;
|
||||
}
|
||||
const filePathItems = await getPath(userFile.id, documentsRepo, true, {
|
||||
company: { id: userCompany },
|
||||
} as any);
|
||||
|
||||
const filePath = filePathItems
|
||||
.slice(1, -1)
|
||||
.map(p => p.name)
|
||||
.join("/");
|
||||
|
||||
const fileObject = {
|
||||
owner: userId,
|
||||
_id: userFile.id,
|
||||
is_in_trash: userFile.is_in_trash,
|
||||
is_directory: userFile.is_directory,
|
||||
name: userFile.name,
|
||||
added: userFile.added,
|
||||
last_modified: userFile.last_modified,
|
||||
size: userFile.size,
|
||||
path: filePath !== "My Drive" ? filePath : "",
|
||||
company_id: userCompany,
|
||||
};
|
||||
|
||||
userFilesObjects.push(fileObject);
|
||||
|
||||
if (!dryRun) {
|
||||
try {
|
||||
const cozyUrl = `${userId}.${COZY_DOMAIN}`;
|
||||
const userToken = await getDriveToken(cozyUrl);
|
||||
const client = new CozyClient({
|
||||
uri: `https://${cozyUrl}`,
|
||||
token: userToken.token,
|
||||
});
|
||||
|
||||
let fileDirPath = "io.cozy.files.root-dir";
|
||||
if (fileObject.path !== "") {
|
||||
const sanitizedPath = fileObject.path.replace(/^\//, "");
|
||||
fileDirPath = (
|
||||
await client.collection("io.cozy.files").createDirectoryByPath(sanitizedPath)
|
||||
).data.id;
|
||||
}
|
||||
// 2. Download file from backend
|
||||
const archiveOrFile = await globalResolver.services.documents.documents.download(
|
||||
userFile.id,
|
||||
userFile.last_version_cache.id,
|
||||
null, // No archive callback needed
|
||||
{
|
||||
company: { id: userCompany },
|
||||
user: { id: user.id },
|
||||
} as any,
|
||||
);
|
||||
|
||||
if (!archiveOrFile.file) {
|
||||
console.error(`File ${userFile.id} was returned as archive. Skipping.`);
|
||||
continue;
|
||||
}
|
||||
let uploadedBytes = 0;
|
||||
const totalSize = fileObject.size || 0;
|
||||
const { file: fileStream } = archiveOrFile.file;
|
||||
const fileReadable = nodeReadableToWebReadable(fileStream, chunkSize => {
|
||||
uploadedBytes += chunkSize;
|
||||
const percentage =
|
||||
totalSize > 0 ? ((uploadedBytes / totalSize) * 100).toFixed(2) : "0";
|
||||
process.stdout.write(`\rUploading ${fileObject.name}... ${percentage}%`);
|
||||
});
|
||||
|
||||
const resp = await uploadFile(
|
||||
fileObject.name,
|
||||
userId,
|
||||
fileDirPath,
|
||||
userToken.token,
|
||||
fileReadable,
|
||||
);
|
||||
|
||||
if (!resp.ok) {
|
||||
console.error(`❌ ERROR UPLOADING THE FILE: ${fileObject.name}`);
|
||||
console.error(`❌ ERROR: ${JSON.stringify(resp)} ${resp}`);
|
||||
continue;
|
||||
}
|
||||
// 3. Migrate file
|
||||
userFile.migrated = true;
|
||||
userFile.migration_date = Date.now();
|
||||
await documentsRepo.save(userFile);
|
||||
|
||||
console.log(`\n✅ File migrated successfully: ${fileObject.name}`);
|
||||
} catch (error) {
|
||||
console.error(`❌ ERROR CREATING THE FILE: ${fileObject.name}`);
|
||||
console.error(`❌ ERROR: ${JSON.stringify(error)} ${error}`);
|
||||
}
|
||||
} else {
|
||||
console.log(`[DRY-RUN] Would create Cozy instance for user ${user.email_canonical}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
export default purgeIndexesCommand;
|
||||
@@ -0,0 +1,85 @@
|
||||
import runWithPlatform from "../../lib/run-with-platform";
|
||||
import runWithLoggerLevel from "../../utils/run-with-logger-level";
|
||||
import globalResolver from "../../../services/global-resolver";
|
||||
import User from "../../../services/user/entities/user";
|
||||
import yargs from "yargs";
|
||||
import { createCozyInstance } from "./utils";
|
||||
|
||||
const migrateUsersCommand: yargs.CommandModule<unknown, unknown> = {
|
||||
command: "migrate-users",
|
||||
describe: "Migrates users data from Twake Drive",
|
||||
builder: {
|
||||
dryRun: {
|
||||
type: "boolean",
|
||||
alias: "d",
|
||||
description: "Simulate the migration and return stats for the DB.",
|
||||
default: true,
|
||||
},
|
||||
emails: {
|
||||
type: "string",
|
||||
alias: "e",
|
||||
description: "Comma-separated list of user emails to migrate specific users",
|
||||
},
|
||||
},
|
||||
handler: async argv => {
|
||||
const dryRun = argv.dryRun as boolean;
|
||||
const emailsArg = argv.emails as string | undefined;
|
||||
const specifiedEmails = emailsArg
|
||||
? emailsArg.split(",").map(email => email.trim().toLowerCase())
|
||||
: null;
|
||||
|
||||
console.log("DRY RUN:", dryRun);
|
||||
if (specifiedEmails) {
|
||||
console.log("Migrating only specified emails:", specifiedEmails.join(", "));
|
||||
}
|
||||
|
||||
await runWithPlatform("Migrate users", async () => {
|
||||
await runWithLoggerLevel("info", async () => {
|
||||
const usersRepo = await globalResolver.database.getRepository<User>("user", User);
|
||||
|
||||
const allUsers = await (await usersRepo.find({})).getEntities();
|
||||
const migratedUsers = await (await usersRepo.find({ migrated: true })).getEntities();
|
||||
const pendingUsers = allUsers.filter(
|
||||
user => !migratedUsers.some(migratedUser => migratedUser.id === user.id),
|
||||
);
|
||||
|
||||
let usersToMigrate = pendingUsers;
|
||||
if (specifiedEmails) {
|
||||
usersToMigrate = pendingUsers.filter(user =>
|
||||
specifiedEmails.includes(user.email_canonical.toLowerCase()),
|
||||
);
|
||||
}
|
||||
|
||||
if (usersToMigrate.length === 0) {
|
||||
console.log("✅ No users to migrate.");
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`🚀 Users to migrate: ${usersToMigrate.length}`);
|
||||
|
||||
for (const user of usersToMigrate) {
|
||||
const userId = user.email_canonical.split("@")[0];
|
||||
const userObject = {
|
||||
_id: user.id,
|
||||
id: userId,
|
||||
email: user.email_canonical,
|
||||
name: `${user.first_name} ${user.last_name}`,
|
||||
};
|
||||
|
||||
if (!dryRun) {
|
||||
try {
|
||||
await createCozyInstance(userObject);
|
||||
} catch (error) {
|
||||
console.error(`❌ Failed to migrate user ${user.email_canonical}`, error);
|
||||
// Even if error, continue to next user
|
||||
}
|
||||
} else {
|
||||
console.log(`[DRY-RUN] Would create Cozy instance for user ${user.email_canonical}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
export default migrateUsersCommand;
|
||||
@@ -0,0 +1,167 @@
|
||||
/* eslint-disable prettier/prettier */
|
||||
import axios from "axios";
|
||||
import config from "config";
|
||||
|
||||
export const DEFAULT_COMPANY = config.get<string>("drive.defaultCompany");
|
||||
export const COZY_DOMAIN = config.get<string>("migration.cozyDomain");
|
||||
const COZY_MANAGER_URL = config.get<string>("migration.cozyManagerUrl");
|
||||
const COZY_MANAGER_TOKEN = config.get<string>("migration.cozyManagerToken");
|
||||
const POLL_INTERVAL_MS = config.get<number>("migration.pollInterval");
|
||||
const MAX_RETRIES = config.get<number>("migration.maxRetries");
|
||||
|
||||
function buildUploadUrl(baseUrl, params) {
|
||||
const searchParams = new URLSearchParams(params);
|
||||
return `${baseUrl}?${searchParams.toString()}`;
|
||||
}
|
||||
|
||||
export function nodeReadableToWebReadable(readable, onProgress?: (chunkSize: number) => void) {
|
||||
const reader = readable[Symbol.asyncIterator]();
|
||||
return new ReadableStream({
|
||||
async pull(controller) {
|
||||
const { value, done } = await reader.next();
|
||||
if (done) {
|
||||
controller.close();
|
||||
} else {
|
||||
if (onProgress) onProgress(value.length); // report progress
|
||||
controller.enqueue(value);
|
||||
}
|
||||
},
|
||||
cancel() {
|
||||
reader.return?.();
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function createCozyInstance(user: {
|
||||
id: string;
|
||||
email: string;
|
||||
name: string;
|
||||
_id: string;
|
||||
}) {
|
||||
console.log(`🚀 Creating Cozy instance for ${user.email}...`);
|
||||
|
||||
try {
|
||||
// Step 1: Create the instance
|
||||
const { data: createInstanceData } = await axios.post(
|
||||
`${COZY_MANAGER_URL}/instances`,
|
||||
{
|
||||
offer: "twake",
|
||||
slug: user.id,
|
||||
domain: COZY_DOMAIN,
|
||||
email: user.email,
|
||||
public_name: user.name,
|
||||
locale: "fr",
|
||||
oidc: user.id,
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Bearer ${COZY_MANAGER_TOKEN}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const workflowId = createInstanceData.workflow;
|
||||
console.log(`✅ Created instance for ${user.email}, workflow ID: ${workflowId}`);
|
||||
|
||||
// Step 2: Check workflow status
|
||||
const checkWorkflowStatus = async () => {
|
||||
const { data } = await axios.get(`${COZY_MANAGER_URL}/workflows/${workflowId}`, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${COZY_MANAGER_TOKEN}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
return data.status;
|
||||
};
|
||||
|
||||
let attempt = 0;
|
||||
let finished = false;
|
||||
|
||||
while (attempt < MAX_RETRIES && !finished) {
|
||||
attempt++;
|
||||
console.log(
|
||||
`🔄 Checking workflow status for ${user.email} (attempt ${attempt}/${MAX_RETRIES})...`,
|
||||
);
|
||||
|
||||
const status = await checkWorkflowStatus();
|
||||
|
||||
if (status === "finished") {
|
||||
console.log(`🎉 Workflow finished for ${user.email}`);
|
||||
finished = true;
|
||||
} else {
|
||||
console.log(
|
||||
`⏳ Workflow not finished yet for ${user.email}, waiting ${POLL_INTERVAL_MS / 1000}s...`,
|
||||
);
|
||||
await new Promise(resolve => setTimeout(resolve, POLL_INTERVAL_MS));
|
||||
}
|
||||
}
|
||||
|
||||
if (!finished) {
|
||||
console.warn(
|
||||
`⚠️ Workflow for ${user.email} did not finish after ${MAX_RETRIES} tries. Moving on.`,
|
||||
);
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.response) {
|
||||
console.error(
|
||||
`❌ Error migrating ${user.email}:`,
|
||||
error.response.status,
|
||||
error.response.data,
|
||||
);
|
||||
} else {
|
||||
console.error(`❌ Error migrating ${user.email}:`, error.message);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getDriveToken(slugDomain: string): Promise<{ token: string }> {
|
||||
const url = `${COZY_MANAGER_URL}/instances/${slugDomain}/drive_token`;
|
||||
|
||||
try {
|
||||
const response = await axios.post(url, null, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${COZY_MANAGER_TOKEN}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
return response.data; // Should be { token: "..." }
|
||||
} catch (error: any) {
|
||||
console.error(
|
||||
`Failed to get drive token for ${slugDomain}`,
|
||||
error.response?.data || error.message,
|
||||
url,
|
||||
);
|
||||
throw new Error(`Could not retrieve drive token for ${slugDomain}`);
|
||||
}
|
||||
}
|
||||
|
||||
export async function uploadFile(
|
||||
fileName: string,
|
||||
userId: string,
|
||||
fileDirPath: string,
|
||||
userToken: string,
|
||||
fileReadable: ReadableStream<any>,
|
||||
) {
|
||||
const baseUrl = `https://${userId}.${COZY_DOMAIN}/files/${fileDirPath}`;
|
||||
const params = {
|
||||
Name: fileName,
|
||||
Type: "file",
|
||||
Executable: false,
|
||||
Encrypted: false,
|
||||
Size: "",
|
||||
};
|
||||
const uploadUrl = buildUploadUrl(baseUrl, params);
|
||||
const resp = await fetch(uploadUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Bearer ${userToken}`,
|
||||
"Content-Type": "application/octet-stream",
|
||||
},
|
||||
duplex: "half",
|
||||
body: fileReadable,
|
||||
} as RequestInit);
|
||||
return resp;
|
||||
}
|
||||
@@ -109,6 +109,12 @@ export class DriveFile {
|
||||
@Column("access_info", "encoded_json")
|
||||
access_info: AccessInformation;
|
||||
|
||||
@Column("migrated", "tdrive_boolean")
|
||||
migrated: boolean;
|
||||
|
||||
@Column("migration_date", "number")
|
||||
migration_date: number;
|
||||
|
||||
/**
|
||||
* If this field is non-null, then an editing session is in progress (probably in OnlyOffice).
|
||||
* Use {@see EditingSessionKeyFormat} to generate and interpret it.
|
||||
|
||||
@@ -82,6 +82,12 @@ export default class User {
|
||||
@Column("deleted", "tdrive_boolean")
|
||||
deleted: boolean;
|
||||
|
||||
@Column("migrated", "tdrive_boolean")
|
||||
migrated: boolean;
|
||||
|
||||
@Column("migration_date", "number")
|
||||
migration_date: number;
|
||||
|
||||
/**
|
||||
* If set, a restartable suppression process is currently incomplete.
|
||||
*/
|
||||
|
||||
@@ -129,4 +129,4 @@ services:
|
||||
- ./docker-data/letsencrypt/:/etc/letsencrypt/
|
||||
- ./frontend/:/tdrive-react/
|
||||
healthcheck:
|
||||
test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:80/.well-known/
|
||||
test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:80/.well-known/
|
||||
@@ -16,7 +16,7 @@ COPY backend/node/package*.json ./
|
||||
# Test Stage
|
||||
FROM node-base AS test
|
||||
|
||||
RUN npm install
|
||||
RUN npm install --legacy-peer-deps
|
||||
COPY backend/node/ .
|
||||
|
||||
# Add frontend Stage
|
||||
@@ -25,7 +25,7 @@ FROM node-base AS installed-libs
|
||||
COPY backend/node/ .
|
||||
#Install dev dependancies for build
|
||||
ENV NODE_ENV=development
|
||||
RUN npm ci
|
||||
RUN npm ci --legacy-peer-deps
|
||||
|
||||
#Build in production mode
|
||||
ENV NODE_ENV=production
|
||||
@@ -40,11 +40,11 @@ FROM installed-libs AS development
|
||||
ENV NODE_ENV=development
|
||||
RUN npm install -g pino-pretty && \
|
||||
npm install -g tsc-watch && \
|
||||
npm ci
|
||||
npm ci --legacy-peer-deps
|
||||
CMD ["npm", "run", "dev:debug"]
|
||||
|
||||
# Production Stage
|
||||
FROM installed-libs AS production
|
||||
|
||||
EXPOSE 4000
|
||||
CMD ["npm", "run", "serve"]
|
||||
CMD ["npm", "run", "serve"]
|
||||
Reference in New Issue
Block a user