From 8efcb2546c3eba82f8325b5b8b1842fa16a4f5af Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Fri, 28 Feb 2025 00:52:19 +0100 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20back=20cli:=20add=20user=20dump=20a?= =?UTF-8?q?nd=20list=20commands?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../node/src/cli/cmds/users_cmds/dump.ts | 116 ++++++++++++++ .../node/src/cli/cmds/users_cmds/list.ts | 69 ++++++++ .../node/src/cli/lib/run-with-platform.ts | 2 + .../node/src/cli/utils/print-entities.ts | 92 +++++++++++ .../src/core/platform/services/admin/utils.ts | 150 ++++++++++++++++++ tdrive/backend/node/src/utils/arrays.ts | 10 ++ 6 files changed, 439 insertions(+) create mode 100644 tdrive/backend/node/src/cli/cmds/users_cmds/dump.ts create mode 100644 tdrive/backend/node/src/cli/cmds/users_cmds/list.ts create mode 100644 tdrive/backend/node/src/cli/utils/print-entities.ts create mode 100644 tdrive/backend/node/src/core/platform/services/admin/utils.ts create mode 100644 tdrive/backend/node/src/utils/arrays.ts diff --git a/tdrive/backend/node/src/cli/cmds/users_cmds/dump.ts b/tdrive/backend/node/src/cli/cmds/users_cmds/dump.ts new file mode 100644 index 00000000..f8d70070 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/users_cmds/dump.ts @@ -0,0 +1,116 @@ +import yargs from "yargs"; + +import runWithPlatform from "../../lib/run-with-platform"; +import gr from "../../../services/global-resolver"; +import { messageQueueLogger, platformLogger } from "../../../core/platform/framework"; +import { + buildUserDeletionRepositories, + descendDriveItemsDepthFirstRandomOrder, + loadRawVersionsOfItemForDeletion, + type TUserDeletionRepos, +} from "../../../core/platform/services/admin/utils"; +import type User from "../../../services/user/entities/user"; +import { EntityByKind } from "../../utils/print-entities"; + +let globalArgv: { + verbose: boolean; + + user_id: string; + json_output: boolean; + with_storage_paths: boolean; + + user?: User; // Not an argument per say, hydrated in handler +}; + +const lengthOfLongestEntityKind = Object.keys(EntityByKind).reduce( + (acc, kind) => Math.max(acc, kind.length), + 0, +); + +const entityToHeaderString = ( + kind: K & string, + entity: Parameters<(typeof EntityByKind)[K]["headerOf"]>[0], + depth: number = 0, +): string => { + if (globalArgv.json_output) return kind + " " + JSON.stringify(entity); // Can't really print out json directly, valid json seems to get prettified by pino + // ts doesn't seem to infer entityHeaderPrinter[kind] correctly even with help + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return `${kind.padEnd(lengthOfLongestEntityKind)} ${entity.id} ${new Array(depth + 1).join( + " ", + )}${EntityByKind[kind].headerOf(entity as any)}`; +}; + +async function dumpAccount(repos: TUserDeletionRepos): Promise { + console.log(entityToHeaderString("user", globalArgv.user)); + await descendDriveItemsDepthFirstRandomOrder( + repos, + "user_" + globalArgv.user.id, + async (item, _children, parents) => { + const depth = parents.length + 1; + if (!item.is_directory) { + const versionsAssets = await loadRawVersionsOfItemForDeletion( + repos, + item, + globalArgv.with_storage_paths, + ); + for (const { version, file, paths } of versionsAssets) { + const outputPath = (path: string) => + console.log(entityToHeaderString("s3", { id: file.id, path }, depth + 2)); + if (globalArgv.json_output) { + for (const path of paths || []) outputPath(path); + } else if (paths) { + paths.sort((a, b) => a.localeCompare(b)); + if (paths.length > 0) outputPath(paths[0]); + if (paths.length == 3) outputPath(paths[1]); + else if (paths.length > 3) outputPath(`... skipping ${paths.length - 2} paths ...`); + if (paths.length > 1) outputPath(paths[paths.length - 1]); + } + file && console.log(entityToHeaderString("file", file, depth + 2)); + version && console.log(entityToHeaderString("version", version, depth + 1)); + } + } + console.log(entityToHeaderString("item", item, depth)); + }, + ); + return 0; +} + +export default { + command: "dump ", + describe: "Output a list of entities related to a user", + builder: { + user_id: { + demandOption: true, + type: "string", + describe: + "id of the user, or a search string for the email.\n(if the id doesn't match, will print the matching users and exit)", + }, + json_output: { + type: "boolean", + alias: "j", + describe: "Output in lines of '[kind] [json_object]'.", + }, + with_storage_paths: { + type: "boolean", + alias: "p", + describe: "If set, also query the storage to include all relevant paths in the output", + }, + }, + handler: async (argv: typeof globalArgv) => { + globalArgv = argv; + if (!argv.verbose) { + platformLogger.level = "warn"; + messageQueueLogger.level = "warn"; + } + await runWithPlatform("dump_account", async ({ spinner: _spinner, platform: _platform }) => { + const repos = await buildUserDeletionRepositories(gr.database, gr.platformServices.search); + argv.user = await repos.user.findOne({ id: argv.user_id }); + if (!argv.user) { + console.error(`Error, unknown user ${JSON.stringify(argv.user_id)}`); + return 1; + } + await dumpAccount(repos); + return 0; + }); + }, +} as yargs.CommandModule; diff --git a/tdrive/backend/node/src/cli/cmds/users_cmds/list.ts b/tdrive/backend/node/src/cli/cmds/users_cmds/list.ts new file mode 100644 index 00000000..65541c6d --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/users_cmds/list.ts @@ -0,0 +1,69 @@ +import yargs from "yargs"; + +import runWithPlatform from "../../lib/run-with-platform"; +import { messageQueueLogger, platformLogger } from "../../../core/platform/framework"; +import { buildUserDeletionRepositories } from "../../../core/platform/services/admin/utils"; +import type User from "../../../services/user/entities/user"; +import { EntityByKind } from "../../utils/print-entities"; + +let globalArgv: { + verbose: boolean; + + email_like: string; + json_output: boolean; + + user?: User; // Not an argument per say, hydrated in handler +}; + +const entityToHeaderString = ( + kind: K & string, + entity: Parameters<(typeof EntityByKind)[K]["headerOf"]>[0], +): string => { + if (globalArgv.json_output) return kind + " " + JSON.stringify(entity); // Can't really print out json directly, valid json seems to get prettified by pino + // ts doesn't seem to infer entityHeaderPrinter[kind] correctly even with help + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return `${entity.id} ${EntityByKind[kind].headerOf(entity as any)}`; +}; + +export default { + command: "list", + describe: "list users", + builder: { + email_like: { + alias: "e", + type: "string", + describe: + "id of the user, or a search string for the email.\n(if the id doesn't match, will print the matching users and exit)", + }, + json_output: { + type: "boolean", + alias: "j", + describe: "Output in lines of '[kind] [json_object]'.", + }, + }, + handler: async (argv: typeof globalArgv) => { + globalArgv = argv; + if (!argv.verbose) { + platformLogger.level = "warn"; + messageQueueLogger.level = "warn"; + } + await runWithPlatform("dump_account", async ({ spinner: _spinner, platform: _platform }) => { + const repos = await buildUserDeletionRepositories(); + console.log( + ( + await repos.user.find( + {}, + { + ...(argv.email_like ? { $like: [["email_canonical", argv.email_like]] } : {}), + sort: { email_canonical: "asc" }, + }, + ) + ) + .getEntities() + .map(user => entityToHeaderString("user", user)) + .join("\n"), + ); + return 0; + }); + }, +} as yargs.CommandModule; diff --git a/tdrive/backend/node/src/cli/lib/run-with-platform.ts b/tdrive/backend/node/src/cli/lib/run-with-platform.ts index 0b5a8874..4543b5f8 100644 --- a/tdrive/backend/node/src/cli/lib/run-with-platform.ts +++ b/tdrive/backend/node/src/cli/lib/run-with-platform.ts @@ -35,6 +35,8 @@ export default async function runWithPlatform( spinner.fail(err.stack || err); process.exitCode = 1; } + // Spinner seems to interrupt still buffered output otherwise + await new Promise(resolve => setTimeout(resolve, 200)); spinner.start("Platform: shutting down..."); await platform.stop(); spinner.succeed("Platform: shutdown"); diff --git a/tdrive/backend/node/src/cli/utils/print-entities.ts b/tdrive/backend/node/src/cli/utils/print-entities.ts new file mode 100644 index 00000000..130ab59b --- /dev/null +++ b/tdrive/backend/node/src/cli/utils/print-entities.ts @@ -0,0 +1,92 @@ +import type { EntityTarget } from "../../core/platform/services/search/api"; +import gr from "../../services/global-resolver"; + +import User, { TYPE as UserTYPE } from "../../services/user/entities/user"; +import { DriveFile, TYPE as DriveFileTYPE } from "../../services/documents/entities/drive-file"; +import { File } from "../../services/files/entities/file"; +import { + FileVersion, + TYPE as FileVersionTYPE, +} from "../../services/documents/entities/file-version"; +import CompanyUser, { TYPE as CompanyUserTYPE } from "../../services/user/entities/company_user"; +import { + MissedDriveFile, + TYPE as MissedDriveFileTYPE, +} from "../../services/documents/entities/missed-drive-file"; +import Company, { TYPE as CompanyTYPE } from "../../services/user/entities/company"; + +const FileTYPE = "files"; + +/** Fake entity to represent S3 path entries as similar resources */ +interface S3Object { + /** id of the File */ + id: string; + /** path in bucket of the file */ + path: string; +} + +const epochToISO = epoch => (epoch ? new Date(epoch).toISOString() : ""); +export const EntityKindToString = { + user: (entity: User): string => `ðŸĪš ${entity.email_canonical}`, + // extUser: (entity: ExternalUser): string => `ðŸĪš ${entity.user_id} <-> 🌍 ${entity.external_id}`, + compUser: (entity: CompanyUser): string => `ðŸĪš ${entity.user_id} <-> ðŸĶ ${entity.group_id}`, + // session: (entity: Session): string => `ðŸĪš ${entity.sub} <-> ðŸĶ ${entity.company_id}`, + company: (entity: Company): string => `ðŸĶ ${entity.name || entity.displayName}`, + item: (entity: DriveFile): string => + `${entity.is_in_trash ? "🗑ïļ " : ""}${entity.is_directory ? "📁" : "📄"}${ + entity.scope != "personal" ? "ðŸĒ " : "" + } ${entity.name}`, + version: (entity: FileVersion): string => + `🕰ïļ ${epochToISO(entity.date_added)} - ${entity.file_size}`, + file: (entity: File): string => `ðŸ’ŋ ${epochToISO(entity.updated_at)}`, + s3: (entity: S3Object): string => `☁ïļ ${entity.path}`, + missed: (entity: MissedDriveFile): string => + `😭 ${entity.is_in_trash ? "🗑ïļ " : ""} ${entity.name}`, +}; + +const makeEntityKind = ( + kind: keyof typeof EntityKindToString, + db?: { entity: EntityTarget; type: string }, +) => { + const headerToString = EntityKindToString[kind]; + return { + kind, + headerOf: headerToString, + db: db + ? { + getRepository: () => gr.database.getRepository(db.type, db.entity), + ...db, + } + : undefined, + }; +}; + +export type TEntityNames = keyof typeof Entities; +export const Entities = { + User: makeEntityKind("user", { entity: User, type: UserTYPE }), + DriveFile: makeEntityKind("item", { entity: DriveFile, type: DriveFileTYPE }), + File: makeEntityKind("file", { entity: File, type: FileTYPE }), + FileVersion: makeEntityKind("version", { + entity: FileVersion, + type: FileVersionTYPE, + }), + // ExternalUser: makeEntityKind("extUser", { + // entity: ExternalUser, + // type: ExternalUserTYPE, + // }), + CompanyUser: makeEntityKind("compUser", { + entity: CompanyUser, + type: CompanyUserTYPE, + }), + MissedDriveFile: makeEntityKind("missed", { + entity: MissedDriveFile, + type: MissedDriveFileTYPE, + }), + Company: makeEntityKind("company", { entity: Company, type: CompanyTYPE }), + // Session: makeEntityKind("session", { entity: Session, type: SessionTYPE }), + $S3: makeEntityKind("s3"), +}; + +export const EntityByKind = Object.fromEntries( + Object.entries(Entities).map(([_, def]) => [def.kind, def]), +); diff --git a/tdrive/backend/node/src/core/platform/services/admin/utils.ts b/tdrive/backend/node/src/core/platform/services/admin/utils.ts new file mode 100644 index 00000000..07d2a9ba --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/admin/utils.ts @@ -0,0 +1,150 @@ +import { getLogger } from "../../framework"; + +import type { DatabaseServiceAPI } from "../database/api"; +import type { SearchServiceAPI } from "../search/api"; + +import gr from "../../../../services/global-resolver"; +import type Repository from "../database/services/orm/repository/repository"; + +import User, { TYPE as UserTYPE } from "../../../../services/user/entities/user"; +import { + DriveFile, + TYPE as DriveFileTYPE, +} from "../../../../services/documents/entities/drive-file"; +import { File } from "../../../../services/files/entities/file"; +import { + FileVersion, + TYPE as FileVersionTYPE, +} from "../../../../services/documents/entities/file-version"; +import ExternalUser, { + TYPE as ExternalUserTYPE, +} from "../../../../services/user/entities/external_user"; +import CompanyUser, { + TYPE as CompanyUserType, +} from "../../../../services/user/entities/company_user"; +import { + MissedDriveFile, + TYPE as MissedDriveFileTYPE, +} from "../../../../services/documents/entities/missed-drive-file"; +import Session from "../../../../services/console/entities/session"; +import { fisherYattesShuffleInPlace } from "../../../../utils/arrays"; +import { getFilePath } from "../../../../services/files/services"; +import Company, { TYPE as CompanyType } from "../../../../services/user/entities/company"; + +const FileTYPE = "files"; + +export const adminLogger = getLogger("Admin"); + +/** Run the `map` function on sets of `batchSize` in `list`, sequentially between batches */ +async function runInBatches( + batchSize: number, + list: readonly T[], + map: (items: T[]) => Promise, +): Promise { + const listCopy = [...list]; + let batch; + const result = [] as R[]; + while ((batch = listCopy.splice(0, batchSize)).length) { + console.error("batch", batch); + result.push(await map(batch)); + } + return result; +} + +/** Call {@see runInBatches} expecting a boolean result from `map`, return `true` only if empty or all batches returned true */ +export async function runInBatchesAreAllTrue( + batchSize: number, + list: T[], + map: (items: T[]) => Promise, +): Promise { + return ( + await runInBatches(batchSize, list, async results => (await map(results)).every(x => !!x)) + ).every(x => !!x); +} +export type TUserDeletionRepos = Awaited>; + +/** + * Create all repositories required for deleting a user + * @deprecated Do not use this outside of this file, it is exported exclusively for e2e tests + */ +export async function buildUserDeletionRepositories( + db: DatabaseServiceAPI = gr.database, + search: SearchServiceAPI = gr.platformServices.search, +) { + return { + driveFile: await db.getRepository(DriveFileTYPE, DriveFile), + file: await db.getRepository(FileTYPE, File), + fileVersion: await db.getRepository(FileVersionTYPE, FileVersion), + missedDriveFile: await db.getRepository(MissedDriveFileTYPE, MissedDriveFile), + user: await db.getRepository(UserTYPE, User), + companyUser: await db.getRepository(CompanyUserType, CompanyUser), + externalUser: await db.getRepository(ExternalUserTYPE, ExternalUser), + company: await db.getRepository(CompanyType, Company), + // This one is not typical because it's only valid for remote account type: + session: gr.services.console.getSessionRepo() as Repository | null, + //TODO: checkout what to do with session and user_online (seen in prod) + + search: { + driveFile: await search.getRepository(DriveFileTYPE, DriveFile), + user: await search.getRepository(UserTYPE, User), + }, + }; +} + +/** Return versions, sorted by oldest first, and related DB and S3 entries for a given DriveFile */ +export async function loadRawVersionsOfItemForDeletion( + repos: TUserDeletionRepos, + item: DriveFile, + loadStoragePaths = false, +): Promise< + { + version: FileVersion; + file?: File; + paths?: string[]; + }[] +> { + return Promise.all( + (await repos.fileVersion.find({ drive_item_id: item.id }, { sort: { date_added: "asc" } })) + .getEntities() + .map(async version => { + const file = await repos.file.findOne({ id: version.file_metadata.external_id }); + if (!loadStoragePaths) return { version, file }; + const paths = file + ? await gr.platformServices.storage.enumeratePathsForFile(getFilePath(file)) + : []; + return { version, file, paths }; + }), + ); +} + +/** + * Recursively descends through drive items in a depth-first manner with a random order at each level. + * + * @warn No security or filtering is done (e.g., no rights are checked, items in trash are included, etc.). + * + * @param {string} parent_id - The ID of the parent drive item (or virtual root) to start the descent from. + * @param {(item: DriveFile, children: undefined | T[], parents: DriveFile[]) => Promise} map - The mapping function to process each drive item. + * - `item`: The current drive item being processed. + * - `children`: The results of processing the children of the current drive item, or `undefined` if the item is not a directory. + * - `parents`: An array of parent drive items leading to the current item. + * @returns {Promise} A promise that resolves to an array of results produced by the `map` function. + */ +export async function descendDriveItemsDepthFirstRandomOrder( + repos: TUserDeletionRepos, + parent_id: string, + map: (item: DriveFile, children: undefined | T[], parents: readonly DriveFile[]) => Promise, +) { + async function descend(parent_id: string, parents: readonly DriveFile[]): Promise { + const items = (await repos.driveFile.find({ parent_id })).getEntities(); + fisherYattesShuffleInPlace(items); + const result = new Array(items.length); + for (const [index, item] of items.entries()) { + const children = item.is_directory + ? await descend(item.id, parents.concat([item])) + : undefined; + result[index] = await map(item, children, parents); + } + return result; + } + return descend(parent_id, []); +} diff --git a/tdrive/backend/node/src/utils/arrays.ts b/tdrive/backend/node/src/utils/arrays.ts new file mode 100644 index 00000000..2226134d --- /dev/null +++ b/tdrive/backend/node/src/utils/arrays.ts @@ -0,0 +1,10 @@ +/** Shuffle an array in place, returns its parameter for convenience */ +export function fisherYattesShuffleInPlace(list: T[]): T[] { + let index = list.length; + while (index) { + const randomIndex = Math.floor(Math.random() * index); + index--; + [list[index], list[randomIndex]] = [list[randomIndex], list[index]]; + } + return list; +}