🚧 backend: wip initial setup for admin route to delete user (#799)

tdrive/backend/node/config/custom-environment-variables.json

@@ -16,6 +16,9 @@
   "logger": {
     "level": "LOG_LEVEL"
   },
+  "admin": {
+    "endpointSecret": "ADMIN_ENDPOINT_SECRET"
+  },
   "diagnostics": {
     "skipKeys": {
       "__name": "DIAG_SKIP_KEYS",

tdrive/backend/node/config/default.json

@@ -35,6 +35,9 @@
   "logger": {
     "level": "debug"
   },
+  "admin": {
+    "endpointSecret": ""
+  },
   "diagnostics": {
     "skipKeys": [],
     "probeSecret": "",
@@ -231,6 +234,7 @@
     ]
   },
   "services": [
+    "admin",
     "auth",
     "diagnostics",
     "push",

tdrive/backend/node/src/core/platform/framework/api/admin.ts

@@ -0,0 +1,13 @@
+import config from "../../../config";
+
+interface IAdminConfig {
+  // This secret must be provided to the administration endpoints
+  endpointSecret?: string;
+}
+
+export const getConfig = (): IAdminConfig => {
+  const configSection = config.get("admin") as IAdminConfig;
+  return {
+    ...configSection,
+  };
+};

tdrive/backend/node/src/core/platform/services/admin/controller/delete-user-controller.ts

@@ -0,0 +1,92 @@
+import gr from "../../../../../services/global-resolver";
+import { getLogger } from "../../../../../core/platform/framework";
+import User, { TYPE as UserType } from "../../../../../services/user/entities/user";
+import type { DatabaseServiceAPI } from "../../database/api";
+import type { ExecutionContext } from "../../../../platform/framework/api/crud-service";
+import type { SearchServiceAPI } from "../../search/api";
+import {
+  DriveFile,
+  TYPE as DriveFileType,
+} from "../../../../../services/documents/entities/drive-file";
+import { File } from "../../../../../services/files/entities/file";
+import {
+  FileVersion,
+  TYPE as FileVersionType,
+} from "../../../../../services/documents/entities/file-version";
+import ExternalUser, {
+  TYPE as ExternalUserType,
+} from "../../../../../services/user/entities/external_user";
+import CompanyUser, {
+  TYPE as CompanyUserType,
+} from "../../../../../services/user/entities/company_user";
+const FileType = "files";
+
+const logger = getLogger("AdminDeleteUserController");
+
+/**
+ * Create all repositories required for deleting a user
+ * @deprecated Do not use this outside of this file, it is exported exclusively for e2e tests
+ */
+export async function buildUserDeletionRepositories(
+  db: DatabaseServiceAPI,
+  search: SearchServiceAPI,
+) {
+  return {
+    driveFile: await db.getRepository<DriveFile>(DriveFileType, DriveFile),
+    file: await db.getRepository<File>(FileType, File),
+    fileVersion: await db.getRepository<FileVersion>(FileVersionType, FileVersion),
+
+    user: await db.getRepository<User>(UserType, User),
+    companyUser: await db.getRepository<CompanyUser>(CompanyUserType, CompanyUser),
+    externalUser: await db.getRepository<ExternalUser>(ExternalUserType, ExternalUser),
+
+    // group_entity
+    // group_user
+    // missed_drive_files
+    // session
+    // user
+    // user_online
+
+    search: {
+      driveFile: await search.getRepository<DriveFile>(DriveFileType, DriveFile),
+      user: await search.getRepository<User>(UserType, User),
+    },
+  };
+}
+
+export class AdminDeleteUserController {
+  private constructor(
+    private readonly repos: Awaited<ReturnType<typeof buildUserDeletionRepositories>>,
+  ) {}
+  public static async create() {
+    return new AdminDeleteUserController(
+      await buildUserDeletionRepositories(gr.database, gr.platformServices.search),
+    );
+  }
+  // fisherYattesShuffleInPlace
+
+  /** Begin or forward the deletion process of a user */
+  async deleteUser(userId: string): Promise<"failed" | "deleting" | "done"> {
+    try {
+      await gr.services.users.anonymizeAndDelete({ id: userId }, {
+        user: { server_request: true },
+        company: { id: "// TODO: REPLACE WITH COMPANY ID" },
+      } as unknown as ExecutionContext);
+      const existingUser = await this.repos.user.findOne({ id: userId });
+      if (existingUser?.deleted) {
+        if (existingUser.delete_process_started_epoch > 0) return "deleting";
+      }
+    } catch (err) {
+      logger.error({ err, userId }, "User deletion error");
+      return "failed";
+    }
+    return "done";
+  }
+
+  /** Get an array of user IDs that are incompletely deleted */
+  async listUsersPendingDeletion() {
+    return (await this.repos.user.find({}, { $gt: [["delete_process_started_epoch", 0]] }))
+      .getEntities()
+      .map(({ id }) => id);
+  }
+}

tdrive/backend/node/src/core/platform/services/admin/index.ts

@@ -0,0 +1,36 @@
+import { TdriveService, Consumes, Prefix, ServiceName } from "../../framework";
+import web from "./web";
+import AdminServiceAPI from "./service-provider";
+import AdminServiceImpl from "./service";
+import WebServerAPI from "../webserver/provider";
+
+/**
+ * The admin service exposes endpoint that are of use for operational reasons to administrators only, and should not be exposed.
+ */
+@Prefix("/admin")
+@Consumes(["webserver"])
+@ServiceName("admin")
+export default class AdminService extends TdriveService<AdminServiceAPI> {
+  name = "admin";
+  service: AdminServiceAPI;
+
+  api(): AdminServiceAPI {
+    return this.service;
+  }
+
+  public async doInit(): Promise<this> {
+    this.service = new AdminServiceImpl();
+    const fastify = this.context.getProvider<WebServerAPI>("webserver").getServer();
+
+    fastify.register((instance, _opts, next) => {
+      web(instance, { prefix: this.prefix });
+      next();
+    });
+
+    return this;
+  }
+
+  public async doStop(): Promise<this> {
+    return this;
+  }
+}

tdrive/backend/node/src/core/platform/services/admin/service-provider.ts

@@ -0,0 +1,5 @@
+import type { TdriveServiceProvider } from "../../framework";
+
+type AdminServiceAPI = TdriveServiceProvider;
+
+export default AdminServiceAPI;

tdrive/backend/node/src/core/platform/services/admin/service.ts

@@ -0,0 +1,5 @@
+import AdminServiceAPI from "./service-provider";
+
+export default class AdminServiceImpl implements AdminServiceAPI {
+  version: "1";
+}

tdrive/backend/node/src/core/platform/services/admin/web/delete-user-routes.ts

@@ -0,0 +1,46 @@
+import type { FastifyInstance, FastifyPluginCallback, FastifyReply, FastifyRequest } from "fastify";
+import { getConfig } from "../../../framework/api/admin";
+import { AdminDeleteUserController } from "../controller/delete-user-controller";
+
+const config = getConfig();
+
+type TQueryBody = { secret: string };
+function authenticateAdminQuery(request: FastifyRequest, reply: FastifyReply) {
+  const body = request.body as TQueryBody;
+  if (body?.secret !== config.endpointSecret) {
+    reply.status(403).send();
+    return false;
+  }
+  return true;
+}
+
+type TUserDeleteQueryBody = TQueryBody & { userId: string };
+function getUserIfValidQuery(request: FastifyRequest, reply: FastifyReply) {
+  if (!authenticateAdminQuery(request, reply)) return false;
+  const body = request.body as TUserDeleteQueryBody;
+  if (!body.userId?.length) {
+    reply.status(400).send();
+    return false;
+  }
+  return body.userId;
+}
+
+const routes: FastifyPluginCallback = async (fastify: FastifyInstance, _opts, next) => {
+  const config = getConfig();
+  const controller = await AdminDeleteUserController.create();
+  if (config?.endpointSecret?.length) {
+    fastify.post("/user/delete", async (request, reply) => {
+      const userId = getUserIfValidQuery(request, reply);
+      if (!userId) return false;
+      return reply.send({ status: await controller.deleteUser(userId) });
+    });
+
+    fastify.post("/user/delete/pending", async (request, reply) => {
+      if (!authenticateAdminQuery(request, reply)) return false;
+      return reply.send(await controller.listUsersPendingDeletion());
+    });
+  }
+  next();
+};
+
+export default routes;

tdrive/backend/node/src/core/platform/services/admin/web/index.ts

@@ -0,0 +1,9 @@
+import type { FastifyInstance, FastifyRegisterOptions } from "fastify";
+import deleteUserRoutes from "./delete-user-routes";
+
+export default (
+  fastify: FastifyInstance,
+  opts: FastifyRegisterOptions<{ prefix: string }>,
+): void => {
+  fastify.register(deleteUserRoutes, opts);
+};

tdrive/backend/node/src/services/user/entities/user.ts

@@ -78,6 +78,12 @@ export default class User {
   @Column("deleted", "tdrive_boolean")
   deleted: boolean;
 
+  /**
+   * If set, a restartable suppression process is currently incomplete.
+   */
+  @Column("delete_process_started_epoch", "number")
+  delete_process_started_epoch?: number;
+
   @Column("mail_verified", "tdrive_boolean")
   mail_verified: boolean;
 

tdrive/backend/node/src/services/user/services/companies.ts

@@ -310,7 +310,11 @@ export class CompanyServiceImpl {
       for (const company of companies) {
         logger.warn(`User ${userPk.id} is deleted so removed from company ${company.id}`);
         await this.removeUserFromCompany(company, user);
-        await gr.services.workspaces.ensureUserNotInCompanyIsNotInWorkspace(userPk, company.id);
+        try {
+          await gr.services.workspaces.ensureUserNotInCompanyIsNotInWorkspace(userPk, company.id);
+        } catch (err) {
+          logger.error({ err }, "Error removing user from company from workspace");
+        }
       }
     }
   }

tdrive/backend/node/src/services/user/services/users/service.ts

@@ -170,6 +170,7 @@ export class UserServiceImpl {
       user.thumbnail_id = null;
       user.status_icon = null;
       user.deleted = true;
+      user.delete_process_started_epoch = new Date().getTime();
 
       await this.save(user);
 

tdrive/backend/node/src/services/user/web/schemas.ts

@@ -15,6 +15,7 @@ export const userObjectSchema = {
     last_name: { type: "string" },
     created_at: { type: "number" },
     deleted: { type: "boolean" },
+    delete_process_started_epoch: { type: "number" },
 
     status: { type: "string" },
     last_activity: { type: "number" },

tdrive/backend/node/src/services/workspaces/services/workspace.ts

@@ -732,9 +732,11 @@ export class WorkspaceServiceImpl implements TdriveServiceProvider, Initializabl
         logger.warn(
           `User ${userPk.id} is not in company ${workspace.company_id} so removing from workspace ${workspace.id}`,
         );
-        this.removeUser({ workspaceId: workspace.id, userId: userPk.id }, companyId, context).then(
-          () => null,
-        );
+        await this.removeUser(
+          { workspaceId: workspace.id, userId: userPk.id },
+          companyId,
+          context,
+        ).then(() => null);
       }
     }
   }

tdrive/backend/node/src/utils/users.ts

@@ -30,6 +30,7 @@ export async function formatUser(
     full_name: [user.first_name, user.last_name].join(" "),
     created_at: user.creation_date,
     deleted: Boolean(user.deleted),
+    delete_process_started_epoch: user.delete_process_started_epoch,
     status: user.status_icon,
     last_activity: user.last_activity,
     cache: { companies: user.cache?.companies || [] },

tdrive/backend/node/test/e2e/common/user-api.ts

@@ -197,6 +197,22 @@ export default class UserApi {
     return this.platform.authService.sign(payload);
   }
 
+  public async getUser(userId?: string, expectedStatus?: undefined | 200): Promise<User>;
+  public async getUser(userId: string | undefined, expectedStatus: number): Promise<Response>;
+  public async getUser(userId: string = this.user.id, expectedStatus: number = 200): Promise<Response | User> {
+    const response = await this.platform.app.inject({
+      method: "GET",
+      url: `/internal/services/users/v1/users/${encodeURIComponent(userId)}`,
+      headers: {
+        authorization: `Bearer ${this.jwt}`,
+      },
+    });
+    expect(response.statusCode).toBe(expectedStatus);
+    if (expectedStatus === 200)
+      return response.json()["resource"];
+    return response;
+  }
+
   async uploadEicarTestFile(filename: string) {
     // EICAR test file content
     const eicarContent = "X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*";

tdrive/backend/node/test/e2e/setup/index.ts

@@ -32,6 +32,8 @@ export type User = {
   id: string;
   first_name?: string;
   isWorkspaceModerator?: boolean;
+  deleted?: boolean,
+  delete_process_started_epoch?: number,
   preferences?: {
     language?: string;
   };

tdrive/backend/node/test/e2e/users/users.deletion.spec.ts

@@ -0,0 +1,246 @@
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "@jest/globals";
+import { init, TestPlatform } from "../setup";
+import { TestDbService } from "../utils.prepare.db";
+import UserApi from "../common/user-api";
+import type { DriveExecutionContext } from "../../../src/services/documents/types";
+import { DriveFile, TYPE as DriveFileType } from "../../../src/services/documents/entities/drive-file";
+import { File } from "../../../src/services/files/entities/file";
+import { FileVersion, TYPE as FileVersionType } from "../../../src/services/documents/entities/file-version";
+import User, { TYPE as UserType } from "../../../src/services/user/entities/user";
+import { getFilePath } from "../../../src/services/files/services";
+import { buildUserDeletionRepositories } from "../../../src/core/platform/services/admin/controller/delete-user-controller";
+import { getConfig } from "../../../src/core/platform/framework/api/admin";
+
+const loadRepositories = async (platform: TestPlatform) => buildUserDeletionRepositories(platform.database, platform.search);
+
+describe("The users deletion API", () => {
+  const url = "/internal/services/users/v1";
+  let platform: TestPlatform;
+  let currentUser: UserApi;
+  let myUserId: string;
+  let myDriveId: string;
+  let myCompanyId: string;
+  let repos: Awaited<ReturnType<typeof loadRepositories>>;
+  const adminConfig = getConfig();
+
+  interface UserTreeEntry {
+    driveFile: DriveFile;
+    driveFileFromSearch?: DriveFile;
+    versions?: {
+      version: FileVersion,
+      file?: File,
+      storagePaths?: string[],
+    }[];
+    children?: UserTreeEntry[];
+  }
+  
+  async function hydrateDriveFile(context: UserTreeEntry) {
+    const { driveFile } = context;
+    context.driveFileFromSearch = (await repos.search.driveFile.search({}, { $text: { $search: driveFile.name } }, { user: { id: myUserId }, company: { id: myCompanyId } } as DriveExecutionContext)).getEntities()[0];
+    const versions = (await repos.fileVersion.find({ drive_item_id: driveFile.id })).getEntities();
+
+    context.versions = await Promise.all(versions.map(async version => {
+      const file = await repos.file.findOne({id: version.file_metadata.external_id});
+      const fileStoragePath = file && getFilePath(file);
+      return {
+        version, file,
+        storagePaths: fileStoragePath ? (await platform.storage.getConnector().enumeratePathsForFile(fileStoragePath)) : undefined,
+      };
+    }));
+  }
+  
+  async function loadDriveFiles(parent_id: string, parents: UserTreeEntry[] = []): Promise<UserTreeEntry[]> {
+    const driveFiles = (await repos.driveFile.find({ parent_id })).getEntities();
+    const tree = driveFiles.map(driveFile => ({ driveFile }) as UserTreeEntry);
+    for (const entry of tree) {
+      const { driveFile } = entry;
+      if (driveFile.is_directory)
+        entry.children = await loadDriveFiles(driveFile.id, parents.concat([entry]));
+      else
+        await hydrateDriveFile(entry);
+    }
+    return tree;
+  }
+
+  async function userTreeDescend(tree: UserTreeEntry[], map: (entry: UserTreeEntry, parents: UserTreeEntry[]) => Promise<void>, parents: UserTreeEntry[] = [], output: string[] = []) {
+    for (const entry of tree) {
+      await map(entry, parents);
+      if (entry.children)
+        await userTreeDescend(entry.children, map, parents.concat([entry]), output);
+    }
+  }
+
+  async function userTreeToString(tree: UserTreeEntry[]) {
+    const oneIndent = "    ";
+    const output: string[] = [];
+    await userTreeDescend(tree, async (entry, parents) => {
+      const indent = parents.map(() => oneIndent).join("");
+      output.push(indent + (entry.driveFile.is_directory ? "📂" : "📄") + " " + entry.driveFile.name);
+      output.push(indent + oneIndent + " search: " + entry.driveFileFromSearch);
+      for (const version of entry.versions ?? []) {
+        output.push(indent + oneIndent + "version: " + JSON.stringify(version.version?.id));
+        output.push(indent + oneIndent + "   file: " + JSON.stringify(version.file?.id));
+        for (const path of version.storagePaths ?? [])
+          output.push(indent + oneIndent + "   path: " + JSON.stringify(path));
+      }
+    });
+    return output.join("\n");
+  }
+
+  async function createFakeFiles() {
+    const rootFolder = await currentUser.createDirectory(myDriveId, { name: "root_folder" });
+    const subRootFolder = await currentUser.createDirectory(rootFolder.id, { name: "sub_root_folder" });
+    return {
+      rootFolder,
+      subRootFolder,
+      fileAtRoot: await currentUser.uploadRandomFileAndCreateDocument(myDriveId),
+      fileInRootFolder: await currentUser.uploadRandomFileAndCreateDocument(rootFolder.id),
+      fileInSubRoot: await currentUser.uploadRandomFileAndCreateDocument(subRootFolder.id),
+    };
+  }
+
+  function findEntriesInUserTree(tree: UserTreeEntry[]): { [EntryKey in keyof Awaited<ReturnType<typeof createFakeFiles>>]: UserTreeEntry | undefined } {
+    const rootFolder = tree.find(({driveFile}) => driveFile.is_directory);
+    const subRootFolder = rootFolder?.children!.find(({driveFile}) => driveFile.is_directory);
+    return {
+      rootFolder,
+      subRootFolder,
+      fileAtRoot: tree.find(({driveFile}) => !driveFile.is_directory),
+      fileInRootFolder: rootFolder?.children?.find(({driveFile}) => !driveFile.is_directory),
+      fileInSubRoot: subRootFolder?.children?.find(({driveFile}) => !driveFile.is_directory),
+    }
+  }
+
+  beforeAll(async () => {
+    platform = await init({
+      services: [
+        "admin",
+        "database",
+        "search",
+        "message-queue",
+        "applications",
+        "webserver",
+        "user",
+        "auth",
+        "storage",
+        "counter",
+        "console",
+        "workspaces",
+        "statistics",
+        "platform-services",
+      ],
+    });
+
+    currentUser = await UserApi.getInstance(platform);
+    myUserId = currentUser.user.id;
+    myDriveId = "user_" + currentUser.user.id;
+    myCompanyId = currentUser.workspace.company_id;
+    repos = await loadRepositories(platform);
+  });
+
+  afterAll(async () => {
+    await platform.tearDown();
+    platform = null;
+  });
+
+  async function sendDeleteUser(secret: string, userId: string, expected: number = 200) {
+    const response = await platform.app.inject({
+      method: "POST",
+      url: `/admin/user/delete`,
+      body: { secret, userId },
+    });
+    expect(response.statusCode).toBe(expected);
+    if (expected === 200) return JSON.parse(response.body);
+    return response;
+  }
+
+  async function requestPendingUserDeletions(secret: string, expected: number = 200) {
+    const response = await platform.app.inject({
+      method: "POST",
+      url: `/admin/user/delete/pending`,
+      body: { secret },
+    });
+    expect(response.statusCode).toBe(expected);
+    if (expected === 200) return JSON.parse(response.body);
+    return response;
+  }
+
+  describe("The DELETE /users/:id route", () => {
+    it("should have a secret setup or none of this test will work and we should know", () => {
+      expect(adminConfig.endpointSecret).toBeTruthy();
+    });
+
+    //TODO: NONONONO
+    it("should start with a non deleted user with some files", async () => {
+      const user = await currentUser.getUser();
+      expect(user).toMatchObject({
+        id: myUserId,
+        deleted: false,
+        delete_process_started_epoch: 0,
+      });
+
+      const initialFakeFiles = await createFakeFiles();
+      expect(initialFakeFiles.rootFolder.id).toBeTruthy();
+      expect(initialFakeFiles.subRootFolder.id).toBeTruthy();
+      expect(initialFakeFiles.fileAtRoot.id).toBeTruthy();
+      expect(initialFakeFiles.fileInRootFolder.id).toBeTruthy();
+      expect(initialFakeFiles.fileInSubRoot.id).toBeTruthy();
+
+      // const docs = await currentUser.browseDocuments(myDriveId, {});
+      // console.error(JSON.stringify(docs, null, 2)); // works, but projected values
+      // const docs = await platform.documentService.browse(myDriveId, { }, { user: { id: myUserId }, company: { id: "string" } } as DriveExecutionContext);
+      // console.error(JSON.stringifya(docs, null, 2)); // doesn't work, empty children
+
+      // Wait for indexing, inspired by tdrive/backend/node/test/e2e/documents/documents-search.spec.ts
+      await new Promise(resolve => setTimeout(resolve, 5000)); // TODO: search doesn't work anyway
+
+      const tree = await loadDriveFiles(myDriveId);
+      console.log((await userTreeToString(tree)));
+
+      console.log((await repos.search.driveFile.search({}, { $text: { $search: "" } }, undefined)).getEntities());
+      console.log((await repos.search.user.search({}, { $text: { $search: "" } }, undefined)).getEntities());
+
+      const foundEntries = findEntriesInUserTree(tree);
+      expect(foundEntries.rootFolder?.driveFile).toMatchObject({ id: initialFakeFiles.rootFolder.id, parent_id: myDriveId, name: "root_folder" });
+      expect(foundEntries.subRootFolder?.driveFile).toMatchObject({ id: initialFakeFiles.subRootFolder.id, parent_id: initialFakeFiles.rootFolder.id, name: "sub_root_folder" });
+      expect(foundEntries.fileAtRoot?.driveFile).toMatchObject({ id: initialFakeFiles.fileAtRoot.id });
+      expect(foundEntries.fileInRootFolder?.driveFile).toMatchObject({ id: initialFakeFiles.fileInRootFolder.id });
+      expect(foundEntries.fileInSubRoot?.driveFile).toMatchObject({ id: initialFakeFiles.fileInSubRoot.id });
+    });
+
+    it("should reject queries without valid secret", async () => {
+      await sendDeleteUser(adminConfig.endpointSecret + "x", myUserId, 403);
+      await requestPendingUserDeletions(adminConfig.endpointSecret + "x", 403);
+    });
+   
+    it("should reject queries without valid user ID", async () => {
+      await sendDeleteUser(adminConfig.endpointSecret!, "", 400);
+    });
+
+    it("should start with an empty pending list", async () => {
+      const responseBody = await requestPendingUserDeletions(adminConfig.endpointSecret!, 200);
+      expect(responseBody).toStrictEqual([]);
+    });
+
+    it("should be immediately listed after deletion", async () => {
+      await sendDeleteUser(adminConfig.endpointSecret!, myUserId, 200);
+      // TODO: wait for message queue stuff ?
+      const responseBody = await requestPendingUserDeletions(adminConfig.endpointSecret!, 200);
+      expect(responseBody).toStrictEqual([myUserId]);
+    });
+
+    it("the user should be marked at least as pending deletion", async () => {
+      const user = await currentUser.getUser();
+      expect(user).toMatchObject({
+        id: myUserId,
+        deleted: true,
+      });
+      expect((user as unknown as User).delete_process_started_epoch).toBeGreaterThan(0);
+    });
+
+    it("the user should not be able to login", async () => {
+      expect((await currentUser.login()).statusCode).toBe(401);
+    });
+
+  });
+});

tdrive/backend/node/test/e2e/utils.prepare.db.ts

@@ -131,6 +131,8 @@ export class TestDbService {
       identity_provider?: string;
       type?: UserType;
       preferences?: User["preferences"];
+      deleted?: boolean;
+      delete_process_started_epoch?: number;
     } = {},
     id: string = uuidv1(),
   ): Promise<User> {
@@ -149,6 +151,8 @@ export class TestDbService {
       locale: "en",
       timezone: 0,
     };
+    user.deleted = options.deleted ?? false;
+    user.delete_process_started_epoch = options.delete_process_started_epoch;
 
     //Fixme this is cheating, we should correctly set the cache in internal mode in the code
     user.cache.companies = [