From 7856bf509683d7dfbde95c2115119d1d8d9d27d7 Mon Sep 17 00:00:00 2001 From: Eric Doughty-Papassideris Date: Fri, 28 Feb 2025 00:57:06 +0100 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A7=20back:=20user=20FS=20deletion=20s?= =?UTF-8?q?ervice=20updated?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/delete-user-controller.ts | 78 ++++------ .../services/admin/service-provider.ts | 9 +- .../core/platform/services/admin/service.ts | 144 ++++++++++++++++++ .../node/src/services/global-resolver.ts | 3 + 4 files changed, 184 insertions(+), 50 deletions(-) diff --git a/tdrive/backend/node/src/core/platform/services/admin/controller/delete-user-controller.ts b/tdrive/backend/node/src/core/platform/services/admin/controller/delete-user-controller.ts index c578e382..9d93e5f4 100644 --- a/tdrive/backend/node/src/core/platform/services/admin/controller/delete-user-controller.ts +++ b/tdrive/backend/node/src/core/platform/services/admin/controller/delete-user-controller.ts @@ -1,58 +1,37 @@ import gr from "../../../../../services/global-resolver"; -import { getLogger } from "../../../../../core/platform/framework"; -import User, { TYPE as UserType } from "../../../../../services/user/entities/user"; -import type { DatabaseServiceAPI } from "../../database/api"; + import type { ExecutionContext } from "../../../../platform/framework/api/crud-service"; -import type { SearchServiceAPI } from "../../search/api"; -import { - DriveFile, - TYPE as DriveFileType, -} from "../../../../../services/documents/entities/drive-file"; -import { File } from "../../../../../services/files/entities/file"; -import { - FileVersion, - TYPE as FileVersionType, -} from "../../../../../services/documents/entities/file-version"; -import ExternalUser, { - TYPE as ExternalUserType, -} from "../../../../../services/user/entities/external_user"; -import CompanyUser, { - TYPE as CompanyUserType, -} from "../../../../../services/user/entities/company_user"; -const FileType = "files"; -const logger = getLogger("AdminDeleteUserController"); +import { adminLogger, buildUserDeletionRepositories } from "../utils"; -/** - * Create all repositories required for deleting a user - * @deprecated Do not use this outside of this file, it is exported exclusively for e2e tests - */ -export async function buildUserDeletionRepositories( - db: DatabaseServiceAPI, - search: SearchServiceAPI, -) { - return { - driveFile: await db.getRepository(DriveFileType, DriveFile), - file: await db.getRepository(FileType, File), - fileVersion: await db.getRepository(FileVersionType, FileVersion), +// /** +// * Create all repositories required for deleting a user +// * @deprecated Do not use this outside of this file, it is exported exclusively for e2e tests +// */ +// export async function buildUserDeletionRepositories( +// db: DatabaseServiceAPI, +// search: SearchServiceAPI, +// ) { +// return { +// driveFile: await db.getRepository(DriveFileType, DriveFile), +// file: await db.getRepository(FileType, File), +// fileVersion: await db.getRepository(FileVersionType, FileVersion), - user: await db.getRepository(UserType, User), - companyUser: await db.getRepository(CompanyUserType, CompanyUser), - externalUser: await db.getRepository(ExternalUserType, ExternalUser), +// user: await db.getRepository(UserType, User), +// companyUser: await db.getRepository(CompanyUserType, CompanyUser), +// externalUser: await db.getRepository(ExternalUserType, ExternalUser), - // group_entity - // group_user - // missed_drive_files - // session - // user - // user_online +// // company: await db.getRepository(CompanyType, Company), +// // missed_drive_files +// // session +// // user_online - search: { - driveFile: await search.getRepository(DriveFileType, DriveFile), - user: await search.getRepository(UserType, User), - }, - }; -} +// search: { +// driveFile: await search.getRepository(DriveFileType, DriveFile), +// user: await search.getRepository(UserType, User), +// }, +// }; +// } export class AdminDeleteUserController { private constructor( @@ -77,7 +56,8 @@ export class AdminDeleteUserController { if (existingUser.delete_process_started_epoch > 0) return "deleting"; } } catch (err) { - logger.error({ err, userId }, "User deletion error"); + adminLogger.error({ err, userId }, "User deletion error"); + console.log(err); //TODO: NONONO return "failed"; } return "done"; diff --git a/tdrive/backend/node/src/core/platform/services/admin/service-provider.ts b/tdrive/backend/node/src/core/platform/services/admin/service-provider.ts index e317d897..2fe15e9b 100644 --- a/tdrive/backend/node/src/core/platform/services/admin/service-provider.ts +++ b/tdrive/backend/node/src/core/platform/services/admin/service-provider.ts @@ -1,5 +1,12 @@ +import type User from "../../../../services/user/entities/user"; import type { TdriveServiceProvider } from "../../framework"; -type AdminServiceAPI = TdriveServiceProvider; +interface AdminServiceAPI extends TdriveServiceProvider { + /** + * Delete all data relative to provided user. Can be called repeatedly in case of timeout/intermittent s3, should get there. + * @warn Irreversible, and no security checks + */ + deleteUser(user: User): Promise; +} export default AdminServiceAPI; diff --git a/tdrive/backend/node/src/core/platform/services/admin/service.ts b/tdrive/backend/node/src/core/platform/services/admin/service.ts index 5e000b06..b20f5bda 100644 --- a/tdrive/backend/node/src/core/platform/services/admin/service.ts +++ b/tdrive/backend/node/src/core/platform/services/admin/service.ts @@ -1,5 +1,149 @@ +import type pino from "pino"; +import gr from "../../../../services/global-resolver"; +import type User from "../../../../services/user/entities/user"; import AdminServiceAPI from "./service-provider"; +import { + adminLogger, + buildUserDeletionRepositories, + descendDriveItemsDepthFirstRandomOrder, + loadRawVersionsOfItemForDeletion, + runInBatchesAreAllTrue, + TUserDeletionRepos, +} from "./utils"; export default class AdminServiceImpl implements AdminServiceAPI { version: "1"; + + private _repos; + private get repos(): TUserDeletionRepos { + return (this._repos ||= buildUserDeletionRepositories()); + } + + /** + * Delete all the provided storage path data in batches. + * Returns `true` if all (or empty) operations returned `true`. + * Ignores (logs) errors, but cause a return of `false`. + * + * @warn No business rules applied, this is unsafe with sync, use only for final deletion. + */ + private async deleteS3Paths( + logger: pino.Logger, + paths: string[], + batchSize = 5, + ): Promise { + return await runInBatchesAreAllTrue(batchSize, paths, async paths => + Promise.all( + paths.map(path => { + try { + return gr.platformServices.storage.remove(path); + } catch (err) { + logger.error({ err, path }, "Error deleting storage item"); + return false; + } + }), + ), + ); + } + + public async deleteUser(user: User): Promise { + /* + //TODO: NONONO + +// todo: do test with wrong s3 delete path to ignore specifically only non existing errors + + Phases: + - Recurse drive item + - if file + - Each version + - delete all s3 + - then file + - then the version + - delete search document + - delete item + - if folder - delete if empty + - search items by creator + - search files by userid + - versions ? + - search leftover s3 + + // search for file by user id after deletion + // search s3 for prefix company/userid + */ + const deleteUserLogger = adminLogger.child({ adminOp: "DeleteUser", user }); + const result = await descendDriveItemsDepthFirstRandomOrder( + this.repos, + "user_" + user.id, + async (item, children, _parents) => { + let canDeleteItem = true; + if (!item.is_directory) { + const versionsAssets = await loadRawVersionsOfItemForDeletion(this.repos, item, true); + for (const { version, file, paths } of versionsAssets) { + if (paths.length > 0 && !(await this.deleteS3Paths(deleteUserLogger, paths))) { + deleteUserLogger.error({ paths }, "Failed to delete paths"); + canDeleteItem = false; + } else { + try { + if (file) { + const result = await this.repos.file.remove(file); + if (!result) + // No error but nothing deleted, just move on + deleteUserLogger.warn({ file, result }, "Failed to delete file"); + } + } catch (err) { + canDeleteItem = false; + deleteUserLogger.error({ err, file }, "Error deleting file"); + } + if (canDeleteItem && version) + try { + if (version) { + const result = await this.repos.fileVersion.remove(version); + if (!result) + // No error but nothing deleted, just move on + deleteUserLogger.warn({ version, result }, "Failed to delete version"); + } + } catch (err) { + canDeleteItem = false; + deleteUserLogger.error({ err, version }, "Error deleting version"); + } + } + } + } + if (canDeleteItem) { + if (children.every(x => !!x)) { + if (item.is_directory && children.length == 0) + deleteUserLogger.warn({ item }, "Deleting empty directory"); + try { + await this.repos.search.driveFile.service.remove([item as never]); + } catch (err) { + canDeleteItem = false; + deleteUserLogger.error({ err, item }, "Error deleting drive item search entry"); + } + if (canDeleteItem) + try { + const result = await this.repos.driveFile.remove(item); + if (!result) + // No error but nothing deleted, just move on + deleteUserLogger.warn({ item, result }, "Failed to delete drive item"); + } catch (err) { + canDeleteItem = false; + deleteUserLogger.error({ err, item }, "Error deleting drive item"); + } + } else { + deleteUserLogger.error( + { item }, + "Not deleting directory as at least one child failed to delete", + ); + canDeleteItem = false; + } + } + return canDeleteItem; + }, + ); + if (!result.every(x => !!x)) return false; + //TODO: error checking and such + await this.repos.search.user.service.remove([user as never]); + user.delete_process_started_epoch = 0; + await this.repos.user.save(user); + return true; + } } diff --git a/tdrive/backend/node/src/services/global-resolver.ts b/tdrive/backend/node/src/services/global-resolver.ts index 42088f6e..0a60990d 100644 --- a/tdrive/backend/node/src/services/global-resolver.ts +++ b/tdrive/backend/node/src/services/global-resolver.ts @@ -35,8 +35,10 @@ import { AVServiceImpl } from "./av/service"; import { PreviewEngine } from "./previews/services/files/engine"; import { I18nService } from "./i18n"; import { getConfigOrDefault } from "../utils/get-config"; +import type AdminServiceAPI from "../core/platform/services/admin/service-provider"; type PlatformServices = { + admin: AdminServiceAPI; auth: AuthServiceAPI; counter: CounterAPI; cron: CronAPI; @@ -92,6 +94,7 @@ class GlobalResolver { this.database = platform.getProvider("database"); this.platformServices = { + admin: platform.getProvider("admin"), auth: platform.getProvider("auth"), counter: platform.getProvider("counter"), cron: platform.getProvider("cron"),