📁 Changed TDrive root folder (#16)

📁 Changed TDrive root folder

tdrive/backend/node/src/cli/cmds/applications.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Manage Tdrive Applications",
+  command: "applications <command>",
+  builder: yargs =>
+    yargs.commandDir("applications_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/export.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Export Tdrive data",
+  command: "export <command>",
+  builder: yargs =>
+    yargs.commandDir("export_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/export_cmds/export_company.ts

@@ -0,0 +1,197 @@
+import yargs from "yargs";
+import tdrive from "../../../tdrive";
+import { mkdirSync, writeFileSync } from "fs";
+import { Pagination } from "../../../core/platform/framework/api/crud-service";
+import WorkspaceUser from "../../../services/workspaces/entities/workspace_user";
+import { ChannelVisibility } from "../../../services/channels/types";
+import { Channel, ChannelMember } from "../../../services/channels/entities";
+import { formatCompany } from "../../../services/user/utils";
+import gr from "../../../services/global-resolver";
+import { formatUser } from "../../../utils/users";
+
+/**
+ * Merge command parameters. Check the builder definition below for more details.
+ */
+type CLIArgs = {
+  id: string;
+};
+
+const services = [
+  "auth",
+  "storage",
+  "counter",
+  "message-queue",
+  "user",
+  "files",
+  "messages",
+  "workspaces",
+  "platform-services",
+  "console",
+  "applications",
+  "search",
+  "database",
+  "webserver",
+  "channels",
+  "statistics",
+];
+
+const command: yargs.CommandModule<unknown, CLIArgs> = {
+  command: "company",
+  describe:
+    "command to export everything inside a company (publicly data only available to a new member)",
+  builder: {
+    id: {
+      default: "",
+      type: "string",
+      description: "Company ID",
+    },
+    output: {
+      default: "",
+      type: "string",
+      description: "Folder containing the exported data",
+    },
+  },
+  handler: async argv => {
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+
+    const company = await gr.services.companies.getCompany({ id: argv.id });
+
+    if (!company) {
+      return "No such company";
+    }
+
+    console.log(`Start export for company ${company.id}`);
+
+    const output = (argv.output as string) || `export-${company.id}`;
+    mkdirSync(output, { recursive: true });
+
+    //Company
+    console.log("- Create company json file");
+    writeFileSync(`${output}/company.json`, JSON.stringify(formatCompany(company)));
+
+    //Workspaces
+    console.log("- Create workspaces json file");
+    const workspaces = await gr.services.workspaces.getAllForCompany(company.id);
+    writeFileSync(`${output}/workspaces.json`, JSON.stringify(workspaces));
+
+    //Users
+    console.log("- Create users json file");
+    const users = [];
+    for (const workspace of workspaces) {
+      const workspace_users = [];
+      let workspaceUsers: WorkspaceUser[] = [];
+      let pagination = new Pagination();
+      do {
+        const res = await gr.services.workspaces.getUsers(
+          { workspaceId: workspace.id },
+          pagination,
+        );
+        workspaceUsers = [...workspaceUsers, ...res.getEntities()];
+        pagination = res.nextPage as Pagination;
+      } while (pagination.page_token);
+      for (const workspaceUser of workspaceUsers) {
+        const user = await gr.services.users.get({ id: workspaceUser.userId });
+        if (user) {
+          users.push(await formatUser(user));
+          workspace_users.push({ ...workspaceUser, user });
+        }
+      }
+      mkdirSync(`${output}/workspaces/${workspace.id}`, { recursive: true });
+      writeFileSync(
+        `${output}/workspaces/${workspace.id}/users.json`,
+        JSON.stringify(workspace_users),
+      );
+    }
+    writeFileSync(`${output}/users.json`, JSON.stringify(users));
+
+    //Channels
+    console.log("- Create channels json file");
+    const directChannels: Channel[] = [];
+    let allPublicChannels: Channel[] = [];
+
+    let pagination = new Pagination();
+    do {
+      const page = await gr.services.channels.channels.getDirectChannelsInCompany(
+        pagination,
+        company.id,
+        undefined,
+      );
+      for (const channel of page.getEntities()) {
+        const channelDetail = await gr.services.channels.channels.get(
+          {
+            company_id: channel.company_id,
+            workspace_id: "direct",
+            id: channel.id,
+          },
+          undefined,
+        );
+        directChannels.push(channelDetail);
+      }
+      pagination = page.nextPage as Pagination;
+    } while (pagination.page_token);
+
+    for (const workspace of workspaces) {
+      let pagination = new Pagination();
+
+      let publicChannels: Channel[] = [];
+      pagination = new Pagination();
+      do {
+        const page = await gr.services.channels.channels.list(
+          pagination,
+          {},
+          {
+            user: { id: "", server_request: true },
+            workspace: { workspace_id: workspace.id, company_id: company.id },
+          },
+        );
+        const chans = page.getEntities().filter(c => c.visibility == ChannelVisibility.PUBLIC);
+        allPublicChannels = [...allPublicChannels, ...chans];
+        publicChannels = [...publicChannels, ...chans];
+        pagination = page.nextPage as Pagination;
+      } while (pagination.page_token);
+
+      mkdirSync(`${output}/workspaces/${workspace.id}`, { recursive: true });
+      writeFileSync(
+        `${output}/workspaces/${workspace.id}/channels.json`,
+        JSON.stringify(publicChannels),
+      );
+    }
+    writeFileSync(`${output}/direct_channels.json`, JSON.stringify(directChannels));
+
+    //Channels users
+    console.log("- Create channels users json file");
+    for (const channel of [...allPublicChannels /*, ...directChannels*/]) {
+      let members: ChannelMember[] = [];
+      let pagination = new Pagination();
+      do {
+        const page = await gr.services.channels.members.list(
+          pagination,
+          {},
+          {
+            user: { id: "", server_request: true },
+            channel: {
+              company_id: channel.company_id,
+              workspace_id: channel.workspace_id,
+              id: channel.id,
+            },
+          },
+        );
+        members = [...members, ...page.getEntities()] as ChannelMember[];
+        pagination = page.nextPage as Pagination;
+      } while (pagination.page_token);
+
+      mkdirSync(`${output}/workspaces/${channel.workspace_id}/channels/${channel.id}`, {
+        recursive: true,
+      });
+      writeFileSync(
+        `${output}/workspaces/${channel.workspace_id}/channels/${channel.id}/members.json`,
+        JSON.stringify(members),
+      );
+    }
+
+    await platform.stop();
+  },
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/generate.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Generate Tdrive data",
+  command: "generate <command>",
+  builder: yargs =>
+    yargs.commandDir("generate_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/generate_cmds/users-and-companies.ts

@@ -0,0 +1,147 @@
+import passwordGenerator from "generate-password";
+import { from, pipe } from "rxjs";
+import { bufferCount, first, map, mergeMap, switchMap, tap } from "rxjs/operators";
+import { v1 as uuid } from "uuid";
+import yargs from "yargs";
+
+import Company, {
+  getInstance as getCompanyInstance,
+} from "../../../services/user/entities/company";
+import CompanyUser from "../../../services/user/entities/company_user";
+import tdrive from "../../../tdrive";
+import User, { getInstance as getUserInstance } from "../../../services/user/entities/user";
+import gr from "../../../services/global-resolver";
+
+type CLIArgs = {
+  company: number;
+  user: number;
+  concurrent: number;
+};
+
+const services = [
+  "storage",
+  "counter",
+  "applications",
+  "statistics",
+  "auth",
+  "realtime",
+  "push",
+  "platform-services",
+  "user",
+  "search",
+  "channels",
+  "notifications",
+  "database",
+  "webserver",
+  "message-queue",
+];
+
+// eslint-disable-next-line @typescript-eslint/ban-types
+const command: yargs.CommandModule<{}, CLIArgs> = {
+  command: "users",
+  describe: "Generate users and companies",
+  builder: {
+    company: {
+      alias: "c",
+      default: 3,
+      type: "number",
+      description: "Number of companies to generate",
+    },
+    user: {
+      alias: "u",
+      default: 5,
+      type: "number",
+      description: "Number of users to generate in the company",
+    },
+    concurrent: {
+      default: 1,
+      type: "number",
+      description: "Number of concurrent creation tasks",
+    },
+  },
+  handler: async argv => {
+    const concurrentTasks = argv.concurrent;
+    const nbUsersPerCompany = argv.user;
+    const nbCompanies = argv.company;
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+    const companies = getCompanies(nbCompanies);
+    const createUser = async (userInCompany: {
+      user: User;
+      company: Company;
+    }): Promise<CompanyUser> => {
+      console.log("Creating user", userInCompany);
+      const created = await gr.services.users.create(getUserInstance(userInCompany.user));
+
+      return (await gr.services.companies.setUserRole(userInCompany.company.id, created.entity.id))
+        ?.entity;
+    };
+    const createCompany = (company: Company) => {
+      console.log("Creating company", company);
+      return gr.services.companies.createCompany(company);
+    };
+
+    const obsv$ = from(companies).pipe(
+      // Create companies sequentially
+      mergeMap(company => createCompany(company), concurrentTasks),
+      // until we create enough companies
+      bufferCount(companies.length),
+      tap(companies => console.log("Created companies", companies.length)),
+      // for each created company
+      switchMap(companies => from(companies)),
+      // generate a set of user for each company
+      map(company => getUsersForCompany(company, nbUsersPerCompany)),
+      // Create users sequentially
+      pipe(
+        switchMap(userCompanies => from(userCompanies)),
+        mergeMap(userCompany => createUser(userCompany), concurrentTasks),
+        // until we reach the number of users in the company
+        bufferCount(nbUsersPerCompany),
+      ),
+      // until we reach the number of companies
+      bufferCount(companies.length),
+      first(),
+    );
+
+    return obsv$
+      .toPromise()
+      .then(() => platform.stop())
+      .finally(() => console.log("✅ Company users are now created"));
+  },
+};
+
+const getCompanies = (size: number = 10): Company[] => {
+  return [...Array(size).keys()].map(i =>
+    getCompanyInstance({
+      id: uuid(),
+      name: `tdrive${i}.app`,
+      displayName: `My Tdrive Company #${i}`,
+    }),
+  );
+};
+
+const getUsersForCompany = (company: Company, size: number = 10) => {
+  return getUsers(company, size).map(user => ({
+    user,
+    company,
+  }));
+};
+
+const getUsers = (company: Company, size: number = 100): User[] => {
+  return [...Array(size).keys()].map(i => getUser(company, i));
+};
+
+const getUser = (company: Company, id: number): User => {
+  return getUserInstance({
+    id: uuid(),
+    first_name: "John",
+    last_name: `Doe${id}`,
+    email_canonical: `user${id}@${company.name}`,
+    password: passwordGenerator.generate({
+      length: 10,
+      numbers: true,
+    }),
+  });
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/migration.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Migrate your php message to node",
+  command: "migration <command>",
+  builder: yargs =>
+    yargs.commandDir("migration_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/search.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Work with search middleware",
+  command: "search <command>",
+  builder: yargs =>
+    yargs.commandDir("search_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/search_cmds/index_all.ts

@@ -0,0 +1,143 @@
+import yargs from "yargs";
+import tdrive from "../../../tdrive";
+import ora from "ora";
+import { TdrivePlatform } from "../../../core/platform/platform";
+import { DatabaseServiceAPI } from "../../../core/platform/services/database/api";
+import { Pagination } from "../../../core/platform/framework/api/crud-service";
+
+import User, { TYPE as UserTYPE } from "../../../services/user/entities/user";
+import { Channel } from "../../../services/channels/entities";
+import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
+import { SearchServiceAPI } from "../../../core/platform/services/search/api";
+import CompanyUser, { TYPE as CompanyUserTYPE } from "../../../services/user/entities/company_user";
+import gr from "../../../services/global-resolver";
+
+type Options = {
+  repository?: string;
+  repairEntities?: boolean;
+};
+
+class SearchIndexAll {
+  database: DatabaseServiceAPI;
+  search: SearchServiceAPI;
+
+  constructor(readonly platform: TdrivePlatform) {
+    this.database = this.platform.getProvider<DatabaseServiceAPI>("database");
+    this.search = this.platform.getProvider<SearchServiceAPI>("search");
+  }
+
+  public async run(options: Options = {}): Promise<void> {
+    const repositories: Map<string, Repository<any>> = new Map();
+    repositories.set("users", await this.database.getRepository(UserTYPE, User));
+    repositories.set("channels", await this.database.getRepository("channels", Channel));
+
+    const repository = repositories.get(options.repository);
+    if (!repository) {
+      throw (
+        "No such repository ready for indexation, available are: " +
+        Array.from(repositories.keys()).join(", ")
+      );
+    }
+
+    // Complete user with companies in cache
+    if (options.repository === "users" && options.repairEntities) {
+      console.log("Complete user with companies in cache");
+      const companiesUsersRepository = await this.database.getRepository(
+        CompanyUserTYPE,
+        CompanyUser,
+      );
+      const userRepository = await this.database.getRepository(UserTYPE, User);
+      let page: Pagination = { limitStr: "100" };
+      // For each rows
+      do {
+        const list = await userRepository.find({}, { pagination: page }, undefined);
+
+        for (const user of list.getEntities()) {
+          const companies = await companiesUsersRepository.find(
+            { user_id: user.id },
+            {},
+            undefined,
+          );
+
+          user.cache ||= { companies: [] };
+          user.cache.companies = companies.getEntities().map(company => company.group_id);
+          await repositories.get("users").save(user, undefined);
+        }
+
+        page = list.nextPage as Pagination;
+        await new Promise(r => setTimeout(r, 200));
+      } while (page.page_token);
+    }
+
+    console.log("Start indexing...");
+    let count = 0;
+    // Get all items
+    let page: Pagination = { limitStr: "100" };
+    do {
+      console.log("Indexed " + count + " items...");
+      const list = await repository.find({}, { pagination: page }, undefined);
+      page = list.nextPage as Pagination;
+      await this.search.upsert(list.getEntities());
+      count += list.getEntities().length;
+      await new Promise(r => setTimeout(r, 200));
+    } while (page.page_token);
+
+    console.log("Emptying flush (10s)...");
+    await new Promise(r => setTimeout(r, 10000));
+
+    console.log("Done!");
+  }
+}
+
+const services = [
+  "search",
+  "database",
+  "webserver",
+  "auth",
+  "counter",
+  "cron",
+  "message-queue",
+  "push",
+  "realtime",
+  "storage",
+  "tracker",
+  "websocket",
+];
+
+const command: yargs.CommandModule<unknown, unknown> = {
+  command: "index",
+  describe: "command to reindex search middleware from db entities",
+  builder: {
+    repository: {
+      default: "",
+      type: "string",
+      description: "Choose a repository to reindex",
+    },
+    repairEntities: {
+      default: false,
+      type: "boolean",
+      description: "Choose to repair entities too when possible",
+    },
+  },
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  handler: async argv => {
+    const spinner = ora({ text: "Reindex repository - " }).start();
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+    const migrator = new SearchIndexAll(platform);
+
+    const repository = (argv.repository || "") as string;
+
+    if (!repository) {
+      throw "No repository was set.";
+    }
+
+    await migrator.run({
+      repository,
+    });
+
+    return spinner.stop();
+  },
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/users.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Manage Tdrive Users",
+  command: "users <command>",
+  builder: yargs =>
+    yargs.commandDir("users_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/users_cmds/remove.ts

@@ -0,0 +1,85 @@
+import yargs from "yargs";
+import ora from "ora";
+import tdrive from "../../../tdrive";
+import Table from "cli-table";
+import { exit } from "process";
+import gr from "../../../services/global-resolver";
+
+/**
+ * Merge command parameters. Check the builder definition below for more details.
+ */
+type CLIArgs = {
+  id: string;
+};
+
+const services = [
+  "storage",
+  "counter",
+  "message-queue",
+  "platform-services",
+  "user",
+  "search",
+  "database",
+  "webserver",
+  "statistics",
+  "applications",
+  "auth",
+  "realtime",
+  "websocket",
+];
+
+const command: yargs.CommandModule<unknown, CLIArgs> = {
+  command: "remove",
+  describe: "command that allow you to remove one user",
+  builder: {
+    id: {
+      default: "",
+      type: "string",
+      description: "User ID",
+    },
+  },
+  handler: async argv => {
+    const tableBefore = new Table({
+      head: ["User ID", "Username", "Deleted"],
+      colWidths: [40, 40, 10],
+    });
+    const tableAfter = new Table({
+      head: ["User ID", "Username", "Deleted"],
+      colWidths: [40, 40, 10],
+    });
+    const spinner = ora({ text: "Retrieving user" }).start();
+
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+
+    const user = await gr.services.users.get({ id: argv.id });
+
+    if (!user) {
+      console.error("Error: You need to provide User ID");
+      return spinner.stop();
+    }
+
+    if (user) {
+      // Table before
+      tableBefore.push([user.id, user.username_canonical, user.deleted]);
+
+      await gr.services.users.anonymizeAndDelete(
+        { id: user.id },
+        {
+          user: { id: user.id, server_request: true },
+        },
+      );
+
+      const finalUser = await gr.services.users.get({ id: argv.id });
+
+      // Table after
+      tableAfter.push([finalUser.id, finalUser.username_canonical, finalUser.deleted]);
+
+      spinner.stop();
+    }
+
+    exit();
+  },
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/workspace.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Manage Tdrive Workspaces",
+  command: "workspace <command>",
+  builder: yargs =>
+    yargs.commandDir("workspace_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/workspace_cmds/list.ts

@@ -0,0 +1,47 @@
+import yargs from "yargs";
+import ora from "ora";
+import Table from "cli-table";
+import tdrive from "../../../tdrive";
+import gr from "../../../services/global-resolver";
+/**
+ * Merge command parameters. Check the builder definition below for more details.
+ */
+type ListParams = {
+  size: string;
+};
+
+const services = [
+  "platform-services",
+  "user",
+  "search",
+  "channels",
+  "notifications",
+  "database",
+  "webserver",
+  "message-queue",
+];
+
+const command: yargs.CommandModule<ListParams, ListParams> = {
+  command: "list",
+  describe: "List Tdrive workspaces",
+  builder: {
+    size: {
+      default: "50",
+      type: "string",
+      description: "Number of workspaces to fetch",
+    },
+  },
+  handler: async argv => {
+    const table = new Table({ head: ["ID", "Name"], colWidths: [40, 50] });
+    const spinner = ora({ text: "List Tdrive workspaces" }).start();
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+    const workspaces = await gr.services.workspaces.list({ limitStr: argv.size });
+
+    spinner.stop();
+    workspaces.getEntities().forEach(ws => table.push([ws.id, ws.name]));
+    console.log(table.toString());
+  },
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/workspace_cmds/user.ts

@@ -0,0 +1,49 @@
+import yargs from "yargs";
+import ora from "ora";
+import Table from "cli-table";
+import tdrive from "../../../tdrive";
+import gr from "../../../services/global-resolver";
+/**
+ * Merge command parameters. Check the builder definition below for more details.
+ */
+type ListParams = {
+  id: string;
+};
+
+const services = [
+  "platform-services",
+  "user",
+  "search",
+  "channels",
+  "notifications",
+  "database",
+  "webserver",
+  "message-queue",
+];
+
+const command: yargs.CommandModule<ListParams, ListParams> = {
+  command: "user",
+  describe: "List workspace ers",
+  builder: {
+    id: {
+      default: "f339d54a-e833-11ea-92c3-0242ac120004",
+      type: "string",
+      description: "Workspace ID",
+    },
+  },
+  handler: async argv => {
+    const table = new Table({ head: ["user ID", "Date Added"], colWidths: [40, 40] });
+    const spinner = ora({ text: "Retrieving workspace users" }).start();
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+    const users = await gr.services.workspaces.getUsers({ workspaceId: argv.id });
+
+    spinner.stop();
+    users
+      .getEntities()
+      .forEach(u => table.push([u.id, new Date(u.dateAdded).toLocaleDateString()]));
+    console.log(table.toString());
+  },
+};
+
+export default command;