🚧 back: user FS deletion service updated

tdrive/backend/node/src/core/platform/services/admin/controller/delete-user-controller.ts

@@ -1,58 +1,37 @@
 import gr from "../../../../../services/global-resolver";
-import { getLogger } from "../../../../../core/platform/framework";
-import User, { TYPE as UserType } from "../../../../../services/user/entities/user";
-import type { DatabaseServiceAPI } from "../../database/api";
+
 import type { ExecutionContext } from "../../../../platform/framework/api/crud-service";
-import type { SearchServiceAPI } from "../../search/api";
-import {
-  DriveFile,
-  TYPE as DriveFileType,
-} from "../../../../../services/documents/entities/drive-file";
-import { File } from "../../../../../services/files/entities/file";
-import {
-  FileVersion,
-  TYPE as FileVersionType,
-} from "../../../../../services/documents/entities/file-version";
-import ExternalUser, {
-  TYPE as ExternalUserType,
-} from "../../../../../services/user/entities/external_user";
-import CompanyUser, {
-  TYPE as CompanyUserType,
-} from "../../../../../services/user/entities/company_user";
-const FileType = "files";
 
-const logger = getLogger("AdminDeleteUserController");
+import { adminLogger, buildUserDeletionRepositories } from "../utils";
 
-/**
- * Create all repositories required for deleting a user
- * @deprecated Do not use this outside of this file, it is exported exclusively for e2e tests
- */
-export async function buildUserDeletionRepositories(
-  db: DatabaseServiceAPI,
-  search: SearchServiceAPI,
-) {
-  return {
-    driveFile: await db.getRepository<DriveFile>(DriveFileType, DriveFile),
-    file: await db.getRepository<File>(FileType, File),
-    fileVersion: await db.getRepository<FileVersion>(FileVersionType, FileVersion),
+// /**
+//  * Create all repositories required for deleting a user
+//  * @deprecated Do not use this outside of this file, it is exported exclusively for e2e tests
+//  */
+// export async function buildUserDeletionRepositories(
+//   db: DatabaseServiceAPI,
+//   search: SearchServiceAPI,
+// ) {
+//   return {
+//     driveFile: await db.getRepository<DriveFile>(DriveFileType, DriveFile),
+//     file: await db.getRepository<File>(FileType, File),
+//     fileVersion: await db.getRepository<FileVersion>(FileVersionType, FileVersion),
 
-    user: await db.getRepository<User>(UserType, User),
-    companyUser: await db.getRepository<CompanyUser>(CompanyUserType, CompanyUser),
-    externalUser: await db.getRepository<ExternalUser>(ExternalUserType, ExternalUser),
+//     user: await db.getRepository<User>(UserType, User),
+//     companyUser: await db.getRepository<CompanyUser>(CompanyUserType, CompanyUser),
+//     externalUser: await db.getRepository<ExternalUser>(ExternalUserType, ExternalUser),
 
-    // group_entity
-    // group_user
-    // missed_drive_files
-    // session
-    // user
-    // user_online
+//     // company: await db.getRepository<Company>(CompanyType, Company),
+//     // missed_drive_files
+//     // session
+//     // user_online
 
-    search: {
-      driveFile: await search.getRepository<DriveFile>(DriveFileType, DriveFile),
-      user: await search.getRepository<User>(UserType, User),
-    },
-  };
-}
+//     search: {
+//       driveFile: await search.getRepository<DriveFile>(DriveFileType, DriveFile),
+//       user: await search.getRepository<User>(UserType, User),
+//     },
+//   };
+// }
 
 export class AdminDeleteUserController {
   private constructor(
@@ -77,7 +56,8 @@ export class AdminDeleteUserController {
         if (existingUser.delete_process_started_epoch > 0) return "deleting";
       }
     } catch (err) {
-      logger.error({ err, userId }, "User deletion error");
+      adminLogger.error({ err, userId }, "User deletion error");
+      console.log(err); //TODO: NONONO
       return "failed";
     }
     return "done";

tdrive/backend/node/src/core/platform/services/admin/service-provider.ts

@@ -1,5 +1,12 @@
+import type User from "../../../../services/user/entities/user";
 import type { TdriveServiceProvider } from "../../framework";
 
-type AdminServiceAPI = TdriveServiceProvider;
+interface AdminServiceAPI extends TdriveServiceProvider {
+  /**
+   * Delete all data relative to provided user. Can be called repeatedly in case of timeout/intermittent s3, should get there.
+   * @warn Irreversible, and no security checks
+   */
+  deleteUser(user: User): Promise<boolean>;
+}
 
 export default AdminServiceAPI;

tdrive/backend/node/src/core/platform/services/admin/service.ts

@@ -1,5 +1,149 @@
+import type pino from "pino";
+import gr from "../../../../services/global-resolver";
+import type User from "../../../../services/user/entities/user";
 import AdminServiceAPI from "./service-provider";
+import {
+  adminLogger,
+  buildUserDeletionRepositories,
+  descendDriveItemsDepthFirstRandomOrder,
+  loadRawVersionsOfItemForDeletion,
+  runInBatchesAreAllTrue,
+  TUserDeletionRepos,
+} from "./utils";
 
 export default class AdminServiceImpl implements AdminServiceAPI {
   version: "1";
+
+  private _repos;
+  private get repos(): TUserDeletionRepos {
+    return (this._repos ||= buildUserDeletionRepositories());
+  }
+
+  /**
+   * Delete all the provided storage path data in batches.
+   * Returns `true` if all (or empty) operations returned `true`.
+   * Ignores (logs) errors, but cause a return of `false`.
+   *
+   * @warn No business rules applied, this is unsafe with sync, use only for final deletion.
+   */
+  private async deleteS3Paths(
+    logger: pino.Logger,
+    paths: string[],
+    batchSize = 5,
+  ): Promise<boolean> {
+    return await runInBatchesAreAllTrue(batchSize, paths, async paths =>
+      Promise.all(
+        paths.map(path => {
+          try {
+            return gr.platformServices.storage.remove(path);
+          } catch (err) {
+            logger.error({ err, path }, "Error deleting storage item");
+            return false;
+          }
+        }),
+      ),
+    );
+  }
+
+  public async deleteUser(user: User): Promise<boolean> {
+    /*
+    //TODO: NONONO
+
+// todo: do test with wrong s3 delete path to ignore specifically only non existing errors
+
+      Phases:
+        - Recurse drive item
+          - if file
+            - Each version
+              - delete all s3
+              - then file
+              - then the version
+            - delete search document
+            - delete item
+          - if folder - delete if empty
+        - search items by creator
+        - search files by userid
+        - versions ?
+        - search leftover s3
+        
+        // search for file by user id after deletion
+        // search s3 for prefix company/userid
+    */
+    const deleteUserLogger = adminLogger.child({ adminOp: "DeleteUser", user });
+    const result = await descendDriveItemsDepthFirstRandomOrder(
+      this.repos,
+      "user_" + user.id,
+      async (item, children, _parents) => {
+        let canDeleteItem = true;
+        if (!item.is_directory) {
+          const versionsAssets = await loadRawVersionsOfItemForDeletion(this.repos, item, true);
+          for (const { version, file, paths } of versionsAssets) {
+            if (paths.length > 0 && !(await this.deleteS3Paths(deleteUserLogger, paths))) {
+              deleteUserLogger.error({ paths }, "Failed to delete paths");
+              canDeleteItem = false;
+            } else {
+              try {
+                if (file) {
+                  const result = await this.repos.file.remove(file);
+                  if (!result)
+                    // No error but nothing deleted, just move on
+                    deleteUserLogger.warn({ file, result }, "Failed to delete file");
+                }
+              } catch (err) {
+                canDeleteItem = false;
+                deleteUserLogger.error({ err, file }, "Error deleting file");
+              }
+              if (canDeleteItem && version)
+                try {
+                  if (version) {
+                    const result = await this.repos.fileVersion.remove(version);
+                    if (!result)
+                      // No error but nothing deleted, just move on
+                      deleteUserLogger.warn({ version, result }, "Failed to delete version");
+                  }
+                } catch (err) {
+                  canDeleteItem = false;
+                  deleteUserLogger.error({ err, version }, "Error deleting version");
+                }
+            }
+          }
+        }
+        if (canDeleteItem) {
+          if (children.every(x => !!x)) {
+            if (item.is_directory && children.length == 0)
+              deleteUserLogger.warn({ item }, "Deleting empty directory");
+            try {
+              await this.repos.search.driveFile.service.remove([item as never]);
+            } catch (err) {
+              canDeleteItem = false;
+              deleteUserLogger.error({ err, item }, "Error deleting drive item search entry");
+            }
+            if (canDeleteItem)
+              try {
+                const result = await this.repos.driveFile.remove(item);
+                if (!result)
+                  // No error but nothing deleted, just move on
+                  deleteUserLogger.warn({ item, result }, "Failed to delete drive item");
+              } catch (err) {
+                canDeleteItem = false;
+                deleteUserLogger.error({ err, item }, "Error deleting drive item");
+              }
+          } else {
+            deleteUserLogger.error(
+              { item },
+              "Not deleting directory as at least one child failed to delete",
+            );
+            canDeleteItem = false;
+          }
+        }
+        return canDeleteItem;
+      },
+    );
+    if (!result.every(x => !!x)) return false;
+    //TODO: error checking and such
+    await this.repos.search.user.service.remove([user as never]);
+    user.delete_process_started_epoch = 0;
+    await this.repos.user.save(user);
+    return true;
+  }
 }

tdrive/backend/node/src/services/global-resolver.ts

@@ -35,8 +35,10 @@ import { AVServiceImpl } from "./av/service";
 import { PreviewEngine } from "./previews/services/files/engine";
 import { I18nService } from "./i18n";
 import { getConfigOrDefault } from "../utils/get-config";
+import type AdminServiceAPI from "../core/platform/services/admin/service-provider";
 
 type PlatformServices = {
+  admin: AdminServiceAPI;
   auth: AuthServiceAPI;
   counter: CounterAPI;
   cron: CronAPI;
@@ -92,6 +94,7 @@ class GlobalResolver {
     this.database = platform.getProvider<DatabaseServiceAPI>("database");
 
     this.platformServices = {
+      admin: platform.getProvider<AdminServiceAPI>("admin"),
       auth: platform.getProvider<AuthServiceAPI>("auth"),
       counter: platform.getProvider<CounterAPI>("counter"),
       cron: platform.getProvider<CronAPI>("cron"),