📁 Changed TDrive root folder (#16)

📁 Changed TDrive root folder

tdrive/backend/node/src/cli/cmds/applications.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Manage Tdrive Applications",
+  command: "applications <command>",
+  builder: yargs =>
+    yargs.commandDir("applications_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/export.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Export Tdrive data",
+  command: "export <command>",
+  builder: yargs =>
+    yargs.commandDir("export_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/export_cmds/export_company.ts

@@ -0,0 +1,197 @@
+import yargs from "yargs";
+import tdrive from "../../../tdrive";
+import { mkdirSync, writeFileSync } from "fs";
+import { Pagination } from "../../../core/platform/framework/api/crud-service";
+import WorkspaceUser from "../../../services/workspaces/entities/workspace_user";
+import { ChannelVisibility } from "../../../services/channels/types";
+import { Channel, ChannelMember } from "../../../services/channels/entities";
+import { formatCompany } from "../../../services/user/utils";
+import gr from "../../../services/global-resolver";
+import { formatUser } from "../../../utils/users";
+
+/**
+ * Merge command parameters. Check the builder definition below for more details.
+ */
+type CLIArgs = {
+  id: string;
+};
+
+const services = [
+  "auth",
+  "storage",
+  "counter",
+  "message-queue",
+  "user",
+  "files",
+  "messages",
+  "workspaces",
+  "platform-services",
+  "console",
+  "applications",
+  "search",
+  "database",
+  "webserver",
+  "channels",
+  "statistics",
+];
+
+const command: yargs.CommandModule<unknown, CLIArgs> = {
+  command: "company",
+  describe:
+    "command to export everything inside a company (publicly data only available to a new member)",
+  builder: {
+    id: {
+      default: "",
+      type: "string",
+      description: "Company ID",
+    },
+    output: {
+      default: "",
+      type: "string",
+      description: "Folder containing the exported data",
+    },
+  },
+  handler: async argv => {
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+
+    const company = await gr.services.companies.getCompany({ id: argv.id });
+
+    if (!company) {
+      return "No such company";
+    }
+
+    console.log(`Start export for company ${company.id}`);
+
+    const output = (argv.output as string) || `export-${company.id}`;
+    mkdirSync(output, { recursive: true });
+
+    //Company
+    console.log("- Create company json file");
+    writeFileSync(`${output}/company.json`, JSON.stringify(formatCompany(company)));
+
+    //Workspaces
+    console.log("- Create workspaces json file");
+    const workspaces = await gr.services.workspaces.getAllForCompany(company.id);
+    writeFileSync(`${output}/workspaces.json`, JSON.stringify(workspaces));
+
+    //Users
+    console.log("- Create users json file");
+    const users = [];
+    for (const workspace of workspaces) {
+      const workspace_users = [];
+      let workspaceUsers: WorkspaceUser[] = [];
+      let pagination = new Pagination();
+      do {
+        const res = await gr.services.workspaces.getUsers(
+          { workspaceId: workspace.id },
+          pagination,
+        );
+        workspaceUsers = [...workspaceUsers, ...res.getEntities()];
+        pagination = res.nextPage as Pagination;
+      } while (pagination.page_token);
+      for (const workspaceUser of workspaceUsers) {
+        const user = await gr.services.users.get({ id: workspaceUser.userId });
+        if (user) {
+          users.push(await formatUser(user));
+          workspace_users.push({ ...workspaceUser, user });
+        }
+      }
+      mkdirSync(`${output}/workspaces/${workspace.id}`, { recursive: true });
+      writeFileSync(
+        `${output}/workspaces/${workspace.id}/users.json`,
+        JSON.stringify(workspace_users),
+      );
+    }
+    writeFileSync(`${output}/users.json`, JSON.stringify(users));
+
+    //Channels
+    console.log("- Create channels json file");
+    const directChannels: Channel[] = [];
+    let allPublicChannels: Channel[] = [];
+
+    let pagination = new Pagination();
+    do {
+      const page = await gr.services.channels.channels.getDirectChannelsInCompany(
+        pagination,
+        company.id,
+        undefined,
+      );
+      for (const channel of page.getEntities()) {
+        const channelDetail = await gr.services.channels.channels.get(
+          {
+            company_id: channel.company_id,
+            workspace_id: "direct",
+            id: channel.id,
+          },
+          undefined,
+        );
+        directChannels.push(channelDetail);
+      }
+      pagination = page.nextPage as Pagination;
+    } while (pagination.page_token);
+
+    for (const workspace of workspaces) {
+      let pagination = new Pagination();
+
+      let publicChannels: Channel[] = [];
+      pagination = new Pagination();
+      do {
+        const page = await gr.services.channels.channels.list(
+          pagination,
+          {},
+          {
+            user: { id: "", server_request: true },
+            workspace: { workspace_id: workspace.id, company_id: company.id },
+          },
+        );
+        const chans = page.getEntities().filter(c => c.visibility == ChannelVisibility.PUBLIC);
+        allPublicChannels = [...allPublicChannels, ...chans];
+        publicChannels = [...publicChannels, ...chans];
+        pagination = page.nextPage as Pagination;
+      } while (pagination.page_token);
+
+      mkdirSync(`${output}/workspaces/${workspace.id}`, { recursive: true });
+      writeFileSync(
+        `${output}/workspaces/${workspace.id}/channels.json`,
+        JSON.stringify(publicChannels),
+      );
+    }
+    writeFileSync(`${output}/direct_channels.json`, JSON.stringify(directChannels));
+
+    //Channels users
+    console.log("- Create channels users json file");
+    for (const channel of [...allPublicChannels /*, ...directChannels*/]) {
+      let members: ChannelMember[] = [];
+      let pagination = new Pagination();
+      do {
+        const page = await gr.services.channels.members.list(
+          pagination,
+          {},
+          {
+            user: { id: "", server_request: true },
+            channel: {
+              company_id: channel.company_id,
+              workspace_id: channel.workspace_id,
+              id: channel.id,
+            },
+          },
+        );
+        members = [...members, ...page.getEntities()] as ChannelMember[];
+        pagination = page.nextPage as Pagination;
+      } while (pagination.page_token);
+
+      mkdirSync(`${output}/workspaces/${channel.workspace_id}/channels/${channel.id}`, {
+        recursive: true,
+      });
+      writeFileSync(
+        `${output}/workspaces/${channel.workspace_id}/channels/${channel.id}/members.json`,
+        JSON.stringify(members),
+      );
+    }
+
+    await platform.stop();
+  },
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/generate.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Generate Tdrive data",
+  command: "generate <command>",
+  builder: yargs =>
+    yargs.commandDir("generate_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/generate_cmds/users-and-companies.ts

@@ -0,0 +1,147 @@
+import passwordGenerator from "generate-password";
+import { from, pipe } from "rxjs";
+import { bufferCount, first, map, mergeMap, switchMap, tap } from "rxjs/operators";
+import { v1 as uuid } from "uuid";
+import yargs from "yargs";
+
+import Company, {
+  getInstance as getCompanyInstance,
+} from "../../../services/user/entities/company";
+import CompanyUser from "../../../services/user/entities/company_user";
+import tdrive from "../../../tdrive";
+import User, { getInstance as getUserInstance } from "../../../services/user/entities/user";
+import gr from "../../../services/global-resolver";
+
+type CLIArgs = {
+  company: number;
+  user: number;
+  concurrent: number;
+};
+
+const services = [
+  "storage",
+  "counter",
+  "applications",
+  "statistics",
+  "auth",
+  "realtime",
+  "push",
+  "platform-services",
+  "user",
+  "search",
+  "channels",
+  "notifications",
+  "database",
+  "webserver",
+  "message-queue",
+];
+
+// eslint-disable-next-line @typescript-eslint/ban-types
+const command: yargs.CommandModule<{}, CLIArgs> = {
+  command: "users",
+  describe: "Generate users and companies",
+  builder: {
+    company: {
+      alias: "c",
+      default: 3,
+      type: "number",
+      description: "Number of companies to generate",
+    },
+    user: {
+      alias: "u",
+      default: 5,
+      type: "number",
+      description: "Number of users to generate in the company",
+    },
+    concurrent: {
+      default: 1,
+      type: "number",
+      description: "Number of concurrent creation tasks",
+    },
+  },
+  handler: async argv => {
+    const concurrentTasks = argv.concurrent;
+    const nbUsersPerCompany = argv.user;
+    const nbCompanies = argv.company;
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+    const companies = getCompanies(nbCompanies);
+    const createUser = async (userInCompany: {
+      user: User;
+      company: Company;
+    }): Promise<CompanyUser> => {
+      console.log("Creating user", userInCompany);
+      const created = await gr.services.users.create(getUserInstance(userInCompany.user));
+
+      return (await gr.services.companies.setUserRole(userInCompany.company.id, created.entity.id))
+        ?.entity;
+    };
+    const createCompany = (company: Company) => {
+      console.log("Creating company", company);
+      return gr.services.companies.createCompany(company);
+    };
+
+    const obsv$ = from(companies).pipe(
+      // Create companies sequentially
+      mergeMap(company => createCompany(company), concurrentTasks),
+      // until we create enough companies
+      bufferCount(companies.length),
+      tap(companies => console.log("Created companies", companies.length)),
+      // for each created company
+      switchMap(companies => from(companies)),
+      // generate a set of user for each company
+      map(company => getUsersForCompany(company, nbUsersPerCompany)),
+      // Create users sequentially
+      pipe(
+        switchMap(userCompanies => from(userCompanies)),
+        mergeMap(userCompany => createUser(userCompany), concurrentTasks),
+        // until we reach the number of users in the company
+        bufferCount(nbUsersPerCompany),
+      ),
+      // until we reach the number of companies
+      bufferCount(companies.length),
+      first(),
+    );
+
+    return obsv$
+      .toPromise()
+      .then(() => platform.stop())
+      .finally(() => console.log("✅ Company users are now created"));
+  },
+};
+
+const getCompanies = (size: number = 10): Company[] => {
+  return [...Array(size).keys()].map(i =>
+    getCompanyInstance({
+      id: uuid(),
+      name: `tdrive${i}.app`,
+      displayName: `My Tdrive Company #${i}`,
+    }),
+  );
+};
+
+const getUsersForCompany = (company: Company, size: number = 10) => {
+  return getUsers(company, size).map(user => ({
+    user,
+    company,
+  }));
+};
+
+const getUsers = (company: Company, size: number = 100): User[] => {
+  return [...Array(size).keys()].map(i => getUser(company, i));
+};
+
+const getUser = (company: Company, id: number): User => {
+  return getUserInstance({
+    id: uuid(),
+    first_name: "John",
+    last_name: `Doe${id}`,
+    email_canonical: `user${id}@${company.name}`,
+    password: passwordGenerator.generate({
+      length: 10,
+      numbers: true,
+    }),
+  });
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/migration.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Migrate your php message to node",
+  command: "migration <command>",
+  builder: yargs =>
+    yargs.commandDir("migration_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/search.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Work with search middleware",
+  command: "search <command>",
+  builder: yargs =>
+    yargs.commandDir("search_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/search_cmds/index_all.ts

@@ -0,0 +1,143 @@
+import yargs from "yargs";
+import tdrive from "../../../tdrive";
+import ora from "ora";
+import { TdrivePlatform } from "../../../core/platform/platform";
+import { DatabaseServiceAPI } from "../../../core/platform/services/database/api";
+import { Pagination } from "../../../core/platform/framework/api/crud-service";
+
+import User, { TYPE as UserTYPE } from "../../../services/user/entities/user";
+import { Channel } from "../../../services/channels/entities";
+import Repository from "../../../core/platform/services/database/services/orm/repository/repository";
+import { SearchServiceAPI } from "../../../core/platform/services/search/api";
+import CompanyUser, { TYPE as CompanyUserTYPE } from "../../../services/user/entities/company_user";
+import gr from "../../../services/global-resolver";
+
+type Options = {
+  repository?: string;
+  repairEntities?: boolean;
+};
+
+class SearchIndexAll {
+  database: DatabaseServiceAPI;
+  search: SearchServiceAPI;
+
+  constructor(readonly platform: TdrivePlatform) {
+    this.database = this.platform.getProvider<DatabaseServiceAPI>("database");
+    this.search = this.platform.getProvider<SearchServiceAPI>("search");
+  }
+
+  public async run(options: Options = {}): Promise<void> {
+    const repositories: Map<string, Repository<any>> = new Map();
+    repositories.set("users", await this.database.getRepository(UserTYPE, User));
+    repositories.set("channels", await this.database.getRepository("channels", Channel));
+
+    const repository = repositories.get(options.repository);
+    if (!repository) {
+      throw (
+        "No such repository ready for indexation, available are: " +
+        Array.from(repositories.keys()).join(", ")
+      );
+    }
+
+    // Complete user with companies in cache
+    if (options.repository === "users" && options.repairEntities) {
+      console.log("Complete user with companies in cache");
+      const companiesUsersRepository = await this.database.getRepository(
+        CompanyUserTYPE,
+        CompanyUser,
+      );
+      const userRepository = await this.database.getRepository(UserTYPE, User);
+      let page: Pagination = { limitStr: "100" };
+      // For each rows
+      do {
+        const list = await userRepository.find({}, { pagination: page }, undefined);
+
+        for (const user of list.getEntities()) {
+          const companies = await companiesUsersRepository.find(
+            { user_id: user.id },
+            {},
+            undefined,
+          );
+
+          user.cache ||= { companies: [] };
+          user.cache.companies = companies.getEntities().map(company => company.group_id);
+          await repositories.get("users").save(user, undefined);
+        }
+
+        page = list.nextPage as Pagination;
+        await new Promise(r => setTimeout(r, 200));
+      } while (page.page_token);
+    }
+
+    console.log("Start indexing...");
+    let count = 0;
+    // Get all items
+    let page: Pagination = { limitStr: "100" };
+    do {
+      console.log("Indexed " + count + " items...");
+      const list = await repository.find({}, { pagination: page }, undefined);
+      page = list.nextPage as Pagination;
+      await this.search.upsert(list.getEntities());
+      count += list.getEntities().length;
+      await new Promise(r => setTimeout(r, 200));
+    } while (page.page_token);
+
+    console.log("Emptying flush (10s)...");
+    await new Promise(r => setTimeout(r, 10000));
+
+    console.log("Done!");
+  }
+}
+
+const services = [
+  "search",
+  "database",
+  "webserver",
+  "auth",
+  "counter",
+  "cron",
+  "message-queue",
+  "push",
+  "realtime",
+  "storage",
+  "tracker",
+  "websocket",
+];
+
+const command: yargs.CommandModule<unknown, unknown> = {
+  command: "index",
+  describe: "command to reindex search middleware from db entities",
+  builder: {
+    repository: {
+      default: "",
+      type: "string",
+      description: "Choose a repository to reindex",
+    },
+    repairEntities: {
+      default: false,
+      type: "boolean",
+      description: "Choose to repair entities too when possible",
+    },
+  },
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  handler: async argv => {
+    const spinner = ora({ text: "Reindex repository - " }).start();
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+    const migrator = new SearchIndexAll(platform);
+
+    const repository = (argv.repository || "") as string;
+
+    if (!repository) {
+      throw "No repository was set.";
+    }
+
+    await migrator.run({
+      repository,
+    });
+
+    return spinner.stop();
+  },
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/users.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Manage Tdrive Users",
+  command: "users <command>",
+  builder: yargs =>
+    yargs.commandDir("users_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/users_cmds/remove.ts

@@ -0,0 +1,85 @@
+import yargs from "yargs";
+import ora from "ora";
+import tdrive from "../../../tdrive";
+import Table from "cli-table";
+import { exit } from "process";
+import gr from "../../../services/global-resolver";
+
+/**
+ * Merge command parameters. Check the builder definition below for more details.
+ */
+type CLIArgs = {
+  id: string;
+};
+
+const services = [
+  "storage",
+  "counter",
+  "message-queue",
+  "platform-services",
+  "user",
+  "search",
+  "database",
+  "webserver",
+  "statistics",
+  "applications",
+  "auth",
+  "realtime",
+  "websocket",
+];
+
+const command: yargs.CommandModule<unknown, CLIArgs> = {
+  command: "remove",
+  describe: "command that allow you to remove one user",
+  builder: {
+    id: {
+      default: "",
+      type: "string",
+      description: "User ID",
+    },
+  },
+  handler: async argv => {
+    const tableBefore = new Table({
+      head: ["User ID", "Username", "Deleted"],
+      colWidths: [40, 40, 10],
+    });
+    const tableAfter = new Table({
+      head: ["User ID", "Username", "Deleted"],
+      colWidths: [40, 40, 10],
+    });
+    const spinner = ora({ text: "Retrieving user" }).start();
+
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+
+    const user = await gr.services.users.get({ id: argv.id });
+
+    if (!user) {
+      console.error("Error: You need to provide User ID");
+      return spinner.stop();
+    }
+
+    if (user) {
+      // Table before
+      tableBefore.push([user.id, user.username_canonical, user.deleted]);
+
+      await gr.services.users.anonymizeAndDelete(
+        { id: user.id },
+        {
+          user: { id: user.id, server_request: true },
+        },
+      );
+
+      const finalUser = await gr.services.users.get({ id: argv.id });
+
+      // Table after
+      tableAfter.push([finalUser.id, finalUser.username_canonical, finalUser.deleted]);
+
+      spinner.stop();
+    }
+
+    exit();
+  },
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/workspace.ts

@@ -0,0 +1,14 @@
+import { CommandModule } from "yargs";
+
+const command: CommandModule = {
+  describe: "Manage Tdrive Workspaces",
+  command: "workspace <command>",
+  builder: yargs =>
+    yargs.commandDir("workspace_cmds", {
+      visit: commandModule => commandModule.default,
+    }),
+  // eslint-disable-next-line @typescript-eslint/no-empty-function
+  handler: () => {},
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/workspace_cmds/list.ts

@@ -0,0 +1,47 @@
+import yargs from "yargs";
+import ora from "ora";
+import Table from "cli-table";
+import tdrive from "../../../tdrive";
+import gr from "../../../services/global-resolver";
+/**
+ * Merge command parameters. Check the builder definition below for more details.
+ */
+type ListParams = {
+  size: string;
+};
+
+const services = [
+  "platform-services",
+  "user",
+  "search",
+  "channels",
+  "notifications",
+  "database",
+  "webserver",
+  "message-queue",
+];
+
+const command: yargs.CommandModule<ListParams, ListParams> = {
+  command: "list",
+  describe: "List Tdrive workspaces",
+  builder: {
+    size: {
+      default: "50",
+      type: "string",
+      description: "Number of workspaces to fetch",
+    },
+  },
+  handler: async argv => {
+    const table = new Table({ head: ["ID", "Name"], colWidths: [40, 50] });
+    const spinner = ora({ text: "List Tdrive workspaces" }).start();
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+    const workspaces = await gr.services.workspaces.list({ limitStr: argv.size });
+
+    spinner.stop();
+    workspaces.getEntities().forEach(ws => table.push([ws.id, ws.name]));
+    console.log(table.toString());
+  },
+};
+
+export default command;

tdrive/backend/node/src/cli/cmds/workspace_cmds/user.ts

@@ -0,0 +1,49 @@
+import yargs from "yargs";
+import ora from "ora";
+import Table from "cli-table";
+import tdrive from "../../../tdrive";
+import gr from "../../../services/global-resolver";
+/**
+ * Merge command parameters. Check the builder definition below for more details.
+ */
+type ListParams = {
+  id: string;
+};
+
+const services = [
+  "platform-services",
+  "user",
+  "search",
+  "channels",
+  "notifications",
+  "database",
+  "webserver",
+  "message-queue",
+];
+
+const command: yargs.CommandModule<ListParams, ListParams> = {
+  command: "user",
+  describe: "List workspace ers",
+  builder: {
+    id: {
+      default: "f339d54a-e833-11ea-92c3-0242ac120004",
+      type: "string",
+      description: "Workspace ID",
+    },
+  },
+  handler: async argv => {
+    const table = new Table({ head: ["user ID", "Date Added"], colWidths: [40, 40] });
+    const spinner = ora({ text: "Retrieving workspace users" }).start();
+    const platform = await tdrive.run(services);
+    await gr.doInit(platform);
+    const users = await gr.services.workspaces.getUsers({ workspaceId: argv.id });
+
+    spinner.stop();
+    users
+      .getEntities()
+      .forEach(u => table.push([u.id, new Date(u.dateAdded).toLocaleDateString()]));
+    console.log(table.toString());
+  },
+};
+
+export default command;

tdrive/backend/node/src/cli/index.ts

@@ -0,0 +1,28 @@
+import yargs from "yargs";
+import { logger } from "../core/platform/framework/logger";
+
+process.env.NODE_ENV = "cli";
+
+yargs
+  .strict()
+  .usage("Usage: $0 <command> [options]")
+  .middleware([
+    argv => {
+      logger.level = argv.verbose ? "debug" : "fatal";
+    },
+  ])
+  .commandDir("cmds", {
+    visit: commandModule => commandModule.default,
+  })
+  .option("verbose", {
+    alias: "v",
+    default: false,
+    type: "boolean",
+    description: "Run with verbose logging",
+  })
+  .demandCommand(1, "Please supply a valid command")
+  .alias("help", "h")
+  .help("help")
+  .version()
+  .epilogue("for more information, go to https://tdrive.app")
+  .example("$0 <command> --help", "show help of the issue command").argv;

tdrive/backend/node/src/cli/utils/copy-cluster/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+yarn.lock

tdrive/backend/node/src/cli/utils/copy-cluster/README.md

@@ -0,0 +1,4 @@
+Edit the file to your needs
+
+yarn install
+node index.js

tdrive/backend/node/src/cli/utils/copy-cluster/index.js

@@ -0,0 +1,214 @@
+/* eslint-disable */
+var cassandra = require("cassandra-driver");
+
+var fromAuthProvider = new cassandra.auth.PlainTextAuthProvider("", "");
+var fromContactPoints = [""];
+var fromKeyspace = "tdrive";
+
+var toAuthProvider = new cassandra.auth.PlainTextAuthProvider("", "");
+var toContactPoints = [""];
+var toKeyspace = "tdrive";
+
+var forceUpdateAll = false;
+var ignoreTables = ["notification"];
+var countersTables = ["statistics", "stats_counter", "channel_counters", "scheduled_queue_counter"];
+
+var specialConversions = {
+  "tdrive.group_user.date_added": value => {
+    return new Date(value).getTime();
+  },
+};
+
+// -- start process
+
+var fromClient = new cassandra.Client({
+  localDataCenter: "datacenter1",
+  contactPoints: fromContactPoints,
+  authProvider: fromAuthProvider,
+  keyspace: fromKeyspace,
+});
+
+var toClient = new cassandra.Client({
+  localDataCenter: "datacenter1",
+  contactPoints: toContactPoints,
+  authProvider: toAuthProvider,
+  keyspace: toKeyspace,
+  queryOptions: {
+    consistency: cassandra.types.consistencies.quorum,
+  },
+});
+
+// -- Get all tables and copy schema
+
+async function client(origin, query, parameters, options) {
+  return await new Promise((resolve, reject) => {
+    origin.execute(query, [], {}, function (err, result) {
+      if (err) {
+        reject({ err, result });
+      } else {
+        resolve(result);
+      }
+    });
+  });
+}
+
+(async () => {
+  const result = await client(
+    fromClient,
+    "SELECT table_name from system_schema.tables WHERE keyspace_name = '" + fromKeyspace + "'",
+    [],
+    {},
+  );
+  for (row of result.rows) {
+    try {
+      const fromTable = fromKeyspace + "." + row.table_name;
+      const toTable = toKeyspace + "." + row.table_name;
+
+      if (ignoreTables.includes(row.table_name)) {
+        console.log(fromTable.padEnd(50) + " | " + "ignored".padEnd(20) + " | ⏺");
+        continue;
+      }
+
+      let fromCount = 0;
+
+      const destColumns = (
+        await client(
+          toClient,
+          "SELECT column_name from system_schema.columns where keyspace_name = '" +
+            toKeyspace +
+            "' and table_name = '" +
+            row.table_name +
+            "'",
+          [],
+          {},
+        )
+      ).rows.map(r => r.column_name);
+
+      try {
+        const fromResult = await client(
+          fromClient,
+          "SELECT count(*) from " + fromTable + "",
+          [],
+          {},
+        );
+        fromCount = parseInt(fromResult.rows[0].count);
+      } catch (err) {
+        fromCount = NaN;
+      }
+
+      if (destColumns.length === 0) {
+        if (fromCount > 0)
+          console.log(
+            fromTable.padEnd(50) + " | " + ("not_in_dest" + "/" + fromCount).padEnd(20) + " | ⏺",
+          );
+        continue;
+      }
+
+      try {
+        const toResult = await client(toClient, "SELECT count(*) from " + toTable + "", [], {});
+        const toCount = parseInt(toResult.rows[0].count);
+
+        console.log(
+          fromTable.padEnd(50) +
+            " | " +
+            (toCount + "/" + fromCount).padEnd(20) +
+            " | " +
+            (toCount >= fromCount ? "✅" : "❌"),
+        );
+
+        if (row.table_name.indexOf("counter") >= 0 || countersTables.includes(row.table_name)) {
+          console.log(
+            fromTable.padEnd(50) + " | " + ("counter_table" + "/" + fromCount).padEnd(20) + " | 🧮",
+          );
+          if (fromCount > toCount || !fromCount || forceUpdateAll) {
+            await new Promise(r => {
+              fromClient.eachRow(
+                "SELECT JSON * from " + fromTable,
+                [],
+                { prepare: true, fetchSize: 1000 },
+                async function (n, row) {
+                  //
+                  //TODO handle counters (it is special !)
+                },
+                async function (err, result) {
+                  if (result && result.nextPage) {
+                    await new Promise(r => setTimeout(r, 1000));
+                    result.nextPage();
+                  } else {
+                    r();
+                  }
+                },
+              );
+            });
+          }
+
+          continue;
+        }
+
+        if (fromCount > toCount || !fromCount || forceUpdateAll) {
+          await new Promise(r => {
+            fromClient.eachRow(
+              "SELECT JSON * from " + fromTable,
+              [],
+              { prepare: true, fetchSize: 1000 },
+              async function (n, row) {
+                try {
+                  const json = JSON.parse(row["[json]"]);
+
+                  //The from table can have additional depreciated fields, we need to remove them
+                  const filteredJson = {};
+                  for (const col of destColumns) {
+                    if (
+                      typeof json[col] == "string" &&
+                      (json[col] || "").match(
+                        /[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+/,
+                      )
+                    ) {
+                      json[col] = json[col].split(".")[0];
+                    }
+
+                    if (specialConversions[fromTable + "." + col]) {
+                      json[col] = specialConversions[fromTable + "." + col](json[col]);
+                    }
+
+                    if (json[col] !== undefined) filteredJson[col] = json[col];
+                  }
+
+                  await client(
+                    toClient,
+                    "INSERT INTO " +
+                      toTable +
+                      " JSON '" +
+                      JSON.stringify(filteredJson).replace(/'/g, "'$&") +
+                      "'",
+                    [],
+                    {},
+                  );
+                } catch (err) {
+                  console.log(err);
+                }
+              },
+              async function (err, result) {
+                if (result && result.nextPage) {
+                  await new Promise(r => setTimeout(r, 1000));
+                  result.nextPage();
+                } else {
+                  r();
+                }
+              },
+            );
+          });
+        }
+      } catch (err) {
+        console.log(
+          fromTable.padEnd(50) + " | " + ("error" + "/" + fromCount).padEnd(20) + " | ❌",
+        );
+      }
+    } catch (err) {
+      console.log(err);
+      continue;
+    }
+
+    //TODO copy content
+  }
+})();

tdrive/backend/node/src/cli/utils/copy-cluster/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "copy-cluster",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "cassandra-driver": "^4.6.3"
+  }
+}

tdrive/backend/node/src/cli/utils/copy-messages/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+yarn.lock

tdrive/backend/node/src/cli/utils/copy-messages/README.md

@@ -0,0 +1,4 @@
+Edit the file to your needs
+
+yarn install
+node index.js

tdrive/backend/node/src/cli/utils/copy-messages/index.js

@@ -0,0 +1,98 @@
+/* eslint-disable */
+var cassandra = require("cassandra-driver");
+
+var fromAuthProvider = new cassandra.auth.PlainTextAuthProvider("", "");
+var fromContactPoints = [""];
+
+var toAuthProvider = new cassandra.auth.PlainTextAuthProvider("", "");
+var toContactPoints = [""];
+
+// -- start process
+
+var fromClient = new cassandra.Client({
+  localDataCenter: "datacenter1",
+  contactPoints: fromContactPoints,
+  authProvider: fromAuthProvider,
+  keyspace: "tdrive",
+});
+
+var toClient = new cassandra.Client({
+  localDataCenter: "datacenter1",
+  contactPoints: toContactPoints,
+  authProvider: toAuthProvider,
+  keyspace: "tdrive",
+  queryOptions: {
+    consistency: cassandra.types.consistencies.quorum,
+  },
+});
+
+// -- Get all tables and copy schema
+
+async function client(origin, query, parameters, options) {
+  return await new Promise((resolve, reject) => {
+    origin.execute(query, [], {}, function (err, result) {
+      if (err) {
+        reject({ err, result });
+      } else {
+        resolve(result);
+      }
+    });
+  });
+}
+
+// Get all threads and then copy all messages in each threads
+
+let copiedMessages = 0;
+let copiedThreads = 0;
+
+(async () => {
+  await new Promise(r => {
+    fromClient.eachRow(
+      "SELECT id from tdrive.threads",
+      [],
+      { prepare: true, fetchSize: 50 },
+      async function (n, row) {
+        const threadId = row["id"];
+
+        console.log("Threads / Messages :", copiedThreads, "/", copiedMessages);
+
+        copiedThreads++;
+
+        await new Promise(r2 => {
+          fromClient.eachRow(
+            "SELECT JSON * from tdrive.messages where thread_id = ? order by id desc",
+            [threadId],
+            { prepare: true, fetchSize: 1000 },
+            async function (n, row) {
+              const message = row["[json]"];
+
+              copiedMessages++;
+
+              await toClient.execute(
+                "INSERT INTO tdrive.messages JSON '" + message.replace(/'/g, "'$&") + "'",
+                [],
+                { prepare: true },
+              );
+            },
+            async function (err, result) {
+              if (result && result.nextPage) {
+                await new Promise(r => setTimeout(r, 100));
+                result.nextPage();
+              } else {
+                r2();
+              }
+            },
+          );
+        });
+      },
+      async function (err, result) {
+        if (result && result.nextPage) {
+          await new Promise(r => setTimeout(r, 2000));
+          result.nextPage();
+        } else {
+          r();
+        }
+      },
+    );
+  });
+})();

tdrive/backend/node/src/cli/utils/copy-messages/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "copy-messages",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "cassandra-driver": "^4.6.3"
+  }
+}

tdrive/backend/node/src/cli/utils/copy-table/README.md

@@ -0,0 +1,4 @@
+Edit the file to your needs
+
+yarn install
+node index.js

tdrive/backend/node/src/cli/utils/copy-table/index.js

@@ -0,0 +1,88 @@
+/* eslint-disable */
+var cassandra = require("cassandra-driver");
+
+var authProvider = new cassandra.auth.PlainTextAuthProvider("", "");
+var contactPoints = [""];
+var from = "table_a";
+var to = "table_b";
+
+var client = new cassandra.Client({
+  localDataCenter: "datacenter1",
+  contactPoints: contactPoints,
+  authProvider: authProvider,
+  keyspace: "tdrive",
+});
+
+//Copy object to other table
+(async () => {
+  const query = "SELECT * FROM " + from;
+  const parameters = [];
+
+  let count = 0;
+  let max = 0;
+  let min = new Date().getTime();
+
+  let rows = [];
+
+  const options = { prepare: true, fetchSize: 1000 };
+
+  await new Promise(r => {
+    client.eachRow(
+      query,
+      parameters,
+      options,
+      function (n, row) {
+        min = Math.min(parseInt(row.created_at.toString()), min);
+        max = Math.max(parseInt(row.created_at.toString()), max);
+        count++;
+        if (count % 100 == 0) {
+          console.log(count);
+        }
+        rows.push(row);
+      },
+      function (err, result) {
+        if (result && result.nextPage) {
+          result.nextPage();
+        } else {
+          r();
+        }
+      },
+    );
+  });
+
+  console.log("Downloaded ", count, " rows of ", from, ". Starting copy...");
+
+  let copyCount = 0;
+
+  for (let i = 0; i < count; i++) {
+    copyCount++;
+    if (copyCount % 100 == 0) {
+      console.log(copyCount, "of", count);
+    }
+
+    const row = rows[i];
+
+    await new Promise(r => {
+      let query =
+        "UPDATE " +
+        to +
+        " SET answers=?, created_at=?, created_by=?, last_activity=?, participants=?, updated_at=? WHERE id=?";
+      client.execute(
+        query,
+        [
+          row.answers,
+          row.created_at,
+          row.created_by,
+          row.last_activity,
+          row.participants,
+          row.updated_at,
+          row.id,
+        ],
+        () => {
+          r();
+        },
+      );
+    });
+  }
+  console.log("Ended with ", count, "threads");
+})();

tdrive/backend/node/src/cli/utils/copy-table/package.json

@@ -0,0 +1,6 @@
+{
+  "name": "copy-table",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT"
+}

tdrive/backend/node/src/cli/utils/repair-threads/README.md

@@ -0,0 +1,4 @@
+Edit the file to your needs
+
+yarn install
+node index.js

tdrive/backend/node/src/cli/utils/repair-threads/index.js

@@ -0,0 +1,96 @@
+/* eslint-disable  */
+var cassandra = require("cassandra-driver");
+
+var authProvider = new cassandra.auth.PlainTextAuthProvider("", "");
+var contactPoints = [""];
+
+var client = new cassandra.Client({
+  localDataCenter: "datacenter1",
+  contactPoints: contactPoints,
+  authProvider: authProvider,
+  keyspace: "tdrive",
+});
+
+var threadsTable = "threads";
+var messagesTable = "messages";
+
+//Copy object to other table
+(async () => {
+  const query = "SELECT * FROM " + threadsTable + " where answers>1000 allow filtering";
+  const parameters = [];
+  const options = { prepare: true, fetchSize: 1000 };
+
+  let count = 0;
+
+  let threads = {};
+
+  await new Promise(r => {
+    client.eachRow(
+      query,
+      parameters,
+      options,
+      function (n, row) {
+        threads[row.id] ||= { answers: 0, id: row.id };
+        count++;
+      },
+      function (err, result) {
+        if (result && result.nextPage) {
+          result.nextPage();
+        } else {
+          r();
+        }
+      },
+    );
+  });
+
+  console.log(
+    "Downloaded ",
+    Object.keys(threads).length,
+    " threads of ",
+    messagesTable,
+    ". Starting copy...",
+  );
+
+  for (var key of Object.keys(threads)) {
+    await new Promise(r => {
+      const query = "SELECT id FROM " + messagesTable + " where thread_id=" + key;
+      const parameters = [];
+      const options = { prepare: true, fetchSize: 1000 };
+      client.eachRow(
+        query,
+        parameters,
+        options,
+        function (n, row) {
+          threads[key].answers++;
+        },
+        function (err, result) {
+          if (result && result.nextPage) {
+            result.nextPage();
+          } else {
+            r();
+          }
+        },
+      );
+      r();
+    });
+  }
+
+  var copyCount = 0;
+  for (var key of Object.keys(threads)) {
+    copyCount++;
+    if (copyCount % 100 == 0) {
+      console.log(copyCount, "of", Object.keys(threads));
+    }
+
+    const row = threads[key];
+
+    await new Promise(r => {
+      let query = "UPDATE " + threadsTable + " SET answers=? WHERE id=?";
+      console.log(query, [row.answers, row.id + ""]);
+      //client.execute(query, [row.answers, row.id], () => {
+      //  r();
+      //});
+    });
+  }
+  console.log("Ended with ", Object.keys(threads), "threads");
+})();

tdrive/backend/node/src/cli/utils/repair-threads/package.json

@@ -0,0 +1,9 @@
+{
+  "name": "repair-threads",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "cassandra-driver": "^4.6.3"
+  }
+}