🌟 Data migration from nextcloud (#302)

- Files upload to Twake Drive
- New logger

tdrive/backend/node/.nvmrc

@@ -1 +1 @@
-18
+20

tdrive/backend/node/package-lock.json

@@ -2311,6 +2311,11 @@
         "color-convert": "^1.9.0"
       }
     },
+    "any-base": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz",
+      "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg=="
+    },
     "anymatch": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
@@ -8982,6 +8987,15 @@
       "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
       "dev": true
     },
+    "short-uuid": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/short-uuid/-/short-uuid-4.2.2.tgz",
+      "integrity": "sha512-IE7hDSGV2U/VZoCsjctKX6l5t5ak2jE0+aeGJi3KtvjIUNuZVmHVYUjNBhmo369FIWGDtaieRaO8A83Lvwfpqw==",
+      "requires": {
+        "any-base": "^1.1.0",
+        "uuid": "^8.3.2"
+      }
+    },
     "signal-exit": {
       "version": "3.0.7",
       "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz",

tdrive/backend/node/src/services/applications-api/web/controllers/index.ts

@@ -21,6 +21,7 @@ import {
   ConfigureRequest,
 } from "../types";
 import { ConsoleHookUser } from "src/services/console/types";
+import User from "src/services/user/entities/user";
 
 export class ApplicationsApiController {
   async token(
@@ -174,9 +175,9 @@ export class ApplicationsApiController {
         last_name: string;
       };
     }>,
-  ): Promise<any> {
+  ): Promise<User> {
     try {
-      await gr.services.console.getClient().updateLocalUserFromConsole({
+      return await gr.services.console.getClient().updateLocalUserFromConsole({
         email: request.body.email.trim().toLocaleLowerCase(),
         name: request.body.first_name,
         surname: request.body.last_name,
@@ -185,7 +186,6 @@ export class ApplicationsApiController {
       logger.error(err);
       throw err;
     }
-    return {};
   }
 }
 

tdrive/backend/node/src/services/documents/services/access-check.ts

@@ -179,7 +179,7 @@ export const getAccessLevel = async (
   }
 
   if (id.startsWith("user_")) {
-    if (await isCompanyApplication(context)) return "read";
+    if (await isCompanyApplication(context)) return "manage";
   }
 
   if (context?.user?.id) {
@@ -432,7 +432,9 @@ export const getItemScope = async (
       {},
       context,
     );
-    scope = driveItemParent.scope;
+    //when application is creating document, there is not user in the context, so there is not scope as well
+    //TODO [ASH] do not forget tofix it when the context will be emulated during the application rework
+    scope = driveItemParent?.scope || item.scope;
   }
   return scope;
 };

tdrive/backend/utils/nextcloud-migration/package.json

@@ -17,8 +17,12 @@
   "dependencies": {
     "axios": "^1.4.0",
     "express": "^4.18.2",
+    "form-auto-content": "^3.2.1",
+    "formdata-node": "^6.0.3",
     "ldapjs": "^3.0.2",
-    "ldif": "^0.5.1"
+    "ldif": "^0.5.1",
+    "pino": "^8.17.1",
+    "pino-pretty": "^10.3.0"
   },
   "devDependencies": {
     "@jest/globals": "^29.7.0",

tdrive/backend/utils/nextcloud-migration/src/express_server.ts

@@ -1,5 +1,6 @@
 import express, { Express, Request, Response } from "express";
 import { NextcloudMigration, NextcloudMigrationConfiguration } from './nextcloud_migration.js';
+import { logger } from "./logger"
 
 const app: Express = express();
 const port = process.env.SERVER_PORT || 3000;
@@ -47,7 +48,7 @@ const nextcloud = new NextcloudMigration(config);
 
 app.post("/", async (req: Request, res: Response) => {
   const params = req.body;
-  console.log(`Got request for data synchronization with params: ${params}`)
+  logger.info(`Got request for data synchronization with params: ${params}`)
   if (!params || !params["username"] || !params.password) {
     res.status(400).send("Username and password for nextcloud are required");
   }
@@ -62,5 +63,5 @@ app.post("/", async (req: Request, res: Response) => {
 });
 
 app.listen(port, () => {
-  console.log(`[server]: Server is running at http://localhost:${port}`);
+  logger.info(`[server]: Server is running at http://localhost:${port}`);
 });

tdrive/backend/utils/nextcloud-migration/src/ldap_user.ts

@@ -1,4 +1,5 @@
 import ldap, { SearchEntry, SearchOptions } from 'ldapjs';
+import { logger } from "./logger"
 
 export type LdapConfiguration = {
   url: string,
@@ -29,7 +30,7 @@ export class LdapUser {
         if (error) {
           reject(new Error("Authentication error"));
         } else {
-          console.log("Successfully authenticated in LDAP")
+          logger.info("Successfully authenticated in LDAP")
           resolve(this.client);
         }
       });
@@ -44,11 +45,11 @@ export class LdapUser {
           reconnect: true
         });
         this.client.on('connect', (res) => {
-          console.log("Connected to LDAP")
+          logger.info("Connected to LDAP")
           resolve(this.auth("", ""));
         })
         this.client.on('connectionError', (error) => {
-          console.log("Error connecting to LDAP")
+          logger.info("Error connecting to LDAP")
           reject(error);
         });
       }
@@ -59,20 +60,20 @@ export class LdapUser {
     const search = await this.search(username);
     return new Promise<User>((resolve, reject) => {
       search.on('error', (err) => {
-        console.log("ERROR");
-        console.log(err);
+        logger.info("ERROR");
+        logger.info(err);
       });
       search.on('searchRequest', (searchRequest) => {
-        console.log('searchRequest: ', searchRequest.messageId);
+        logger.info('searchRequest: ', searchRequest.messageId);
       });
       search.on('searchEntry', (entry) => {
-        console.log('entry: ' + JSON.stringify(entry));
+        logger.info('entry: ' + JSON.stringify(entry));
       });
       search.on('searchReference', (referral) => {
-        console.log('referral: ' + referral.uris.join());
+        logger.info('referral: ' + referral.uris.join());
       });
       search.on('end', (err) => {
-        console.log("END");
+        logger.info("END");
       });
     });
   }
@@ -84,14 +85,14 @@ export class LdapUser {
         attributes: ['cn', 'sn'],
         scope: 'sub',
       } as SearchOptions;
-      console.log(`Search in ${this.config.baseDn} with options`);
+      logger.info(`Search in ${this.config.baseDn} with options`);
       // Perform search
       this.client?.search(this.config.baseDn, opts, (error, res) => {
         if (error) {
           console.error("Search error", error);
           reject(error)
         } else {
-          console.log("returning search callback");
+          logger.info("returning search callback");
           resolve(res);
         }
       });

tdrive/backend/utils/nextcloud-migration/src/logger.ts

@@ -0,0 +1,9 @@
+import pino from "pino";
+
+export type TDriveLogger = pino.Logger;
+
+export const logger = pino({
+  name: "TDriveNextCloudMigration",
+  level: "debug",
+  prettyPrint: false,
+} as pino.LoggerOptions);

tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts

@@ -1,8 +1,11 @@
 import { exec } from 'child_process';
+// @ts-ignore
 import fs from 'fs';
-import { LdapUser } from './shell_ldap_user.js';
+import { LdapUser } from './shell_ldap_user';
 import { User } from './ldap_user.js';
-import { TwakeDriveClient } from './twake_client.js';
+import { TwakeDriveClient, TwakeDriveUser } from './twake_client';
+import path from 'path';
+import { logger } from "./logger"
 
 export type NextcloudMigrationConfiguration = {
   ldap: {
@@ -26,7 +29,7 @@ export class NextcloudMigration {
 
   private ldap: LdapUser;
 
-  private driveClient: TwakeDriveClient;
+  driveClient: TwakeDriveClient;
 
   constructor(config: NextcloudMigrationConfiguration) {
     this.config = config;
@@ -37,11 +40,14 @@ export class NextcloudMigration {
   async migrate(username: string, password: string) {
     const dir = this.createTmpDir(username);
     try {
-      // await this.download(username, password, dir);
       const user = await this.getLDAPUser(username);
       //create user if needed Twake Drive
-      await this.driveClient.createUser(user);
+      const driveUser = await this.driveClient.createUser(user);
+      console.log(`Drive user ${driveUser.id} created`);
+      //download all files from nextcloud to tmp dir
+      await this.download(username, password, dir);
       //upload files to the Twake Drive
+      await this.upload(driveUser, dir);
     } catch (e) {
       console.error('Error downloading files from next cloud', e);
       throw e;
@@ -99,4 +105,46 @@ export class NextcloudMigration {
     console.log(`Directory ${dir} deleted`);
   }
 
+  async upload(user: TwakeDriveUser, sourceDirPath: string, parentDirId = "user_" + user.id) {
+    const dirsToUpload: Map<string, string> = new Map<string, string>();
+    const filesToUpload: string[] = [];
+
+    const parent = await this.driveClient.getDocument(parentDirId);
+
+    const exists = (filename: string) => {
+      return parent.children.filter(i => i.name.startsWith(path.parse(filename).name)).length > 0;
+    }
+
+    logger.debug(`Reading content of the directory ${sourceDirPath} ...`)
+    fs.readdirSync(sourceDirPath).forEach(function (name) {
+      const filePath = path.join(sourceDirPath, name);
+      const stat = fs.statSync(filePath);
+      if (exists(name)) {
+        logger.info(`File ${name} already exists`);
+      } else {
+        if (stat.isFile()) {
+          logger.info(`Add file for future upload ${filePath}`);
+          filesToUpload.push(filePath)
+        } else if (stat.isDirectory()) {
+          logger.info(`Add directory for the future upload ${filePath}`);
+          dirsToUpload.set(name, filePath);
+        }
+      }
+    });
+
+    //upload all files
+    logger.debug(`UPLOAD FILES FOR  ${sourceDirPath}`)
+    for (const file of filesToUpload) {
+      logger.debug(`Upload file ${file}`)
+      await this.driveClient.createFile(file, parentDirId);
+    }
+
+    logger.debug(`UPLOAD DIRS FOR  ${sourceDirPath}`)
+    for (const [name, path] of dirsToUpload) {
+      logger.info(`Create directory ${name}`);
+      const dir = await this.driveClient.createDirectory(name, parentDirId)
+      await this.upload(user, path, dir.id);
+    }
+  }
+
 }

tdrive/backend/utils/nextcloud-migration/src/shell_ldap_user.ts

@@ -1,6 +1,7 @@
 import { LdapConfiguration, User } from './ldap_user';
 import { exec } from 'child_process';
 import ldif from 'ldif';
+import { logger } from "./logger"
 
 export class LdapUser {
 
@@ -13,13 +14,13 @@ export class LdapUser {
   async find(username: string): Promise<User> {
     return new Promise((resolve, reject) => {
       let cmd = `ldapsearch -x -H ${this.config.url} -b '${this.config.baseDn}' '(uid=${username})'`;
-      console.log("Executing command to get data from LDAP for " + username);
+      logger.info("Executing command to get data from LDAP for " + username);
       exec(cmd, (error, stdout, stderr) => {
         if (stderr) {
-          console.log("ERROR: " + stderr);
+          logger.info("ERROR: " + stderr);
         }
         if (error) {
-          console.log(`ERROR running sync for the user: ${error.message}`);
+          logger.info(`ERROR running sync for the user: ${error.message}`);
           reject(new Error(error.message));
         } else {
           if (stdout) {
@@ -28,7 +29,7 @@ export class LdapUser {
                 stdout = stdout.substring(0, stdout.lastIndexOf("# search result"))
               }
               let obj =  ldif.parse(stdout).shift().toObject({});
-              console.log(obj);
+              logger.info(obj);
               resolve({
                 lastName: obj.attributes.sn,
                 firstName: obj.attributes.givenName,
@@ -39,7 +40,7 @@ export class LdapUser {
               resolve({ } as User);
             }
           } else {
-            console.log("No user");
+            logger.info("No user");
             resolve({ } as User);
           }
         }

tdrive/backend/utils/nextcloud-migration/src/twake_client.ts

@@ -1,15 +1,18 @@
-import axios, { AxiosError } from 'axios';
+import axios from 'axios';
 import { User } from './ldap_user';
+import FormData from 'form-data';
+// @ts-ignore
+import fs from 'fs';
+import { logger } from './logger';
 
 type TwakeClientConfiguration = {
   url: string,
-    credentials: {
+  credentials: {
     appId: string,
     secret: string,
   }
 }
 
-
 export class TwakeDriveClient {
 
   private config: TwakeClientConfiguration;
@@ -17,23 +20,105 @@ export class TwakeDriveClient {
   constructor(config: TwakeClientConfiguration) {
     this.config = config;
     //remove the trailing '/'
-    this.config.url = this.config.url.replace(/\/$/, '');
+    if (this.config && this.config.url) {
+      this.config.url = this.config.url.replace(/\/$/, '');
+    }
   }
 
-  async createUser(user: User) {
+  private async uploadFile(path: string) {
+    logger.info(`Upload ${path}`);
+    const client = await this.client();
+
+    const form = new FormData();
+    form.append('file', fs.createReadStream(path));
+
+    const response
+      = await client.post(`${this.config.url}/internal/services/files/v1/companies/00000000-0000-4000-0000-000000000000/files`, form);
+
+    logger.info(response.data);
+
+    return response.data?.resource;
+  }
+
+  async createFile(path: string, parent_id: string) {
+    const file = await this.uploadFile(path);
+    return this.createDocumentFromFile(file, parent_id);
+  }
+
+  async createDirectory(name: string, parent: string) {
+    return this.createDocument({
+      company_id: '00000000-0000-4000-0000-000000000000',
+      name: name,
+      parent_id: parent,
+      is_directory: true,
+    }, {});
+  }
+
+  private async createDocumentFromFile(file: any, parent_id: string) {
+    const item = {
+      name: file.metadata.name,
+      parent_id: parent_id,
+      company_id: file.company_id,
+    };
+
+    const version = {
+      file_metadata: {
+        name: file.metadata.name,
+        size: file.upload_data?.size,
+        thumbnails: [],
+        external_id: file.id,
+      },
+    };
+    return await this.createDocument(item, version);
+  };
+
+  private async createDocument(item: any, version: any) {
+    const response
+      = await (await this.client()).post(`${this.config.url}/internal/services/documents/v1/companies/00000000-0000-4000-0000-000000000000/item`, {
+      item,
+      version,
+    });
+    logger.info(response);
+    return response.data;
+  };
+
+  async getDocument(id: string) {
+    logger.info(`Get information for the doc ${id}`);
+    const response
+      = await (await this.client()).get(`${this.config.url}/internal/services/documents/v1/companies/00000000-0000-4000-0000-000000000000/item/${id}`, {
+    });
+    logger.info(response);
+    return response.data;
+  }
+
+  async getUser(id: string) {
     const client = await this.client();
     try {
-      const response = await client.post(this.config.url + "/api/sync",
+      let url = `${this.config.url}/internal/services/users/v1/users/${id}`;
+      logger.info(`Get user company with ${url}`);
+      const response = await client.get(url);
+      logger.info(response.data);
+      return response?.data?.resource as TwakeDriveUser;
+    } catch (e) {
+      logger.info(`Error for ${id}: ${e.message}, body: ${e.response?.data?.message}`);
+      throw e;
+    }
+  }
+
+  async createUser(user: User): Promise<TwakeDriveUser> {
+    const client = await this.client();
+    try {
+      const response = await client.post(this.config.url + '/api/sync',
         {
           first_name: user.firstName,
           last_name: user.lastName,
-          email: user.email
+          email: user.email,
         });
-      return response.data;
-    } catch(e){
-      console.log(`Error for ${JSON.stringify(user)}: ${e.message}, body: ${e.response?.data?.message}`);
+      return response.data as TwakeDriveUser;
+    } catch (e) {
+      logger.info(`Error for ${JSON.stringify(user)}: ${e.message}, body: ${e.response?.data?.message}`);
       throw e;
-    };
+    }
   }
 
   private async client() {
@@ -72,16 +157,18 @@ export class TwakeDriveClient {
       console.info('Using token ', this.config.credentials.appId, this.config.credentials.secret);
       console.info(`POST ${this.config.url}/api/console/v1/login`);
       console.info(`Basic ${Buffer.from(`${this.config.credentials.appId}:${this.config.credentials.secret}`).toString('base64')}`);
-      throw new Error("Unable to get access to token, see precious errors for details.");
+      throw new Error('Unable to get access to token, see precious errors for details.');
     }
   };
 
 }
 
-type TwakeDriveUser = {
+export type TwakeDriveUser = {
   first_name: string;
   last_name: string;
   email: string;
+  id: string,
+  email_canonical: string,
 }
 
 interface IApiServiceApplicationTokenRequestParams {

tdrive/backend/utils/nextcloud-migration/test/ldap.test.ts

@@ -2,7 +2,7 @@ import {describe, expect, test} from '@jest/globals';
 import { LdapUser } from '../src/shell_ldap_user';
 import { LdapConfiguration } from '../src/ldap_user';
 
-//integration debug test
+//FOR LOCAL DEBUG PURPOSE ONLY, ITS NOT A TEST
 describe.skip('ldap module', () => {
 
   test('ldap returns user info', async () => {

tdrive/backend/utils/nextcloud-migration/test/migration.test.ts

@@ -0,0 +1,26 @@
+import { expect, test } from '@jest/globals';
+import { TwakeDriveClient, TwakeDriveUser } from '../src/twake_client';
+import { NextcloudMigration, NextcloudMigrationConfiguration } from '../src/nextcloud_migration';
+
+//FOR LOCAL DEBUG PURPOSE ONLY, ITS NOT A TEST
+describe.skip('nextcloud migration module', () => {
+
+  test('upload directory', async () => {
+    //given
+    const subj = new NextcloudMigration({
+      drive: {
+        url: "http://localhost:4000",
+        credentials: {
+          appId: "tdrive_onlyoffice",
+          secret: "c1cc66db78e1d3bb4713c55d5ab2"
+        }
+      }
+    } as NextcloudMigrationConfiguration);
+
+    let user = await subj.driveClient.createUser({firstName: "DWHO", lastName: "DWHO", email: "dwho@example.com", uid: "test"});
+
+    //when
+    await subj.upload(user, "/Users/shepilov/WebstormProjects/TDrive/Documentation/docs/onprem");
+  });
+
+});

tdrive/backend/utils/nextcloud-migration/test/twake.test.ts

@@ -1,23 +1,34 @@
 import { expect, test } from '@jest/globals';
 import { TwakeDriveClient } from '../src/twake_client';
 
-//integration debug test
+//FOR LOCAL DEBUG PURPOSE ONLY, ITS NOT A TEST
 describe.skip('twake client module', () => {
 
-  test('twake creates new user', async () => {
-    //given
-    const subj = new TwakeDriveClient({
-        url: "https://tdrive.qa.lin-saas.com/",
-        credentials: {
-          appId: "tdrive_onlyoffice",
-          secret: "QWERTY"
-        }
-    })
+  const subj = new TwakeDriveClient({
+    url: "http://localhost:4000",
+    credentials: {
+      appId: "tdrive_onlyoffice",
+      secret: "c1cc66db78e1d3bb4713c55d5ab2"
+    }
+  })
 
+  test('twake creates new user and files for the user', async () => {
     //when
-    const response = await subj.createUser({firstName: "Test", lastName: "User", email: "test@example.com", uid: "test"});
-    expect(response).toBeDefined();
-    console.log(response);
+    let user = await subj.createUser({firstName: "DWHO", lastName: "DWHO", email: "dwho@example.com", uid: "test"});
+    expect(user).toBeDefined();
+    expect(user.id).toBeDefined();
+
+    user = await subj.getUser(user.id);
+    expect(user).toBeDefined();
+    expect(user.id).toBeDefined();
+
+    const dir = await subj.createDirectory("My DIR", "user_" + user.id)
+    const doc = await subj.createFile("/Users/shepilov/WebstormProjects/TDrive/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts", dir.id);
+    expect(doc).toBeDefined();
+    expect(doc.id).toBeDefined();
+
+    const item = await subj.getDocument("user_" + user.id);
+    expect(item.children.filter(i => i.name.startsWith("My DIR")).length).toBeGreaterThan(0);
   });
 
 });