diff --git a/tdrive/backend/node/.nvmrc b/tdrive/backend/node/.nvmrc index 3c032078..209e3ef4 100644 --- a/tdrive/backend/node/.nvmrc +++ b/tdrive/backend/node/.nvmrc @@ -1 +1 @@ -18 +20 diff --git a/tdrive/backend/node/package-lock.json b/tdrive/backend/node/package-lock.json index 3178bda6..902c5ca2 100644 --- a/tdrive/backend/node/package-lock.json +++ b/tdrive/backend/node/package-lock.json @@ -2311,6 +2311,11 @@ "color-convert": "^1.9.0" } }, + "any-base": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/any-base/-/any-base-1.1.0.tgz", + "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==" + }, "anymatch": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", @@ -8982,6 +8987,15 @@ "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", "dev": true }, + "short-uuid": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/short-uuid/-/short-uuid-4.2.2.tgz", + "integrity": "sha512-IE7hDSGV2U/VZoCsjctKX6l5t5ak2jE0+aeGJi3KtvjIUNuZVmHVYUjNBhmo369FIWGDtaieRaO8A83Lvwfpqw==", + "requires": { + "any-base": "^1.1.0", + "uuid": "^8.3.2" + } + }, "signal-exit": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", diff --git a/tdrive/backend/node/src/services/applications-api/web/controllers/index.ts b/tdrive/backend/node/src/services/applications-api/web/controllers/index.ts index d014ae91..e43a1610 100644 --- a/tdrive/backend/node/src/services/applications-api/web/controllers/index.ts +++ b/tdrive/backend/node/src/services/applications-api/web/controllers/index.ts @@ -21,6 +21,7 @@ import { ConfigureRequest, } from "../types"; import { ConsoleHookUser } from "src/services/console/types"; +import User from "src/services/user/entities/user"; export class ApplicationsApiController { async token( @@ -174,9 +175,9 @@ export class ApplicationsApiController { last_name: string; }; }>, - ): Promise { + ): Promise { try { - await gr.services.console.getClient().updateLocalUserFromConsole({ + return await gr.services.console.getClient().updateLocalUserFromConsole({ email: request.body.email.trim().toLocaleLowerCase(), name: request.body.first_name, surname: request.body.last_name, @@ -185,7 +186,6 @@ export class ApplicationsApiController { logger.error(err); throw err; } - return {}; } } diff --git a/tdrive/backend/node/src/services/documents/services/access-check.ts b/tdrive/backend/node/src/services/documents/services/access-check.ts index 2c9ebb47..d1ef44c8 100644 --- a/tdrive/backend/node/src/services/documents/services/access-check.ts +++ b/tdrive/backend/node/src/services/documents/services/access-check.ts @@ -179,7 +179,7 @@ export const getAccessLevel = async ( } if (id.startsWith("user_")) { - if (await isCompanyApplication(context)) return "read"; + if (await isCompanyApplication(context)) return "manage"; } if (context?.user?.id) { @@ -432,7 +432,9 @@ export const getItemScope = async ( {}, context, ); - scope = driveItemParent.scope; + //when application is creating document, there is not user in the context, so there is not scope as well + //TODO [ASH] do not forget tofix it when the context will be emulated during the application rework + scope = driveItemParent?.scope || item.scope; } return scope; }; diff --git a/tdrive/backend/utils/nextcloud-migration/package.json b/tdrive/backend/utils/nextcloud-migration/package.json index 6a52178a..05f6d382 100644 --- a/tdrive/backend/utils/nextcloud-migration/package.json +++ b/tdrive/backend/utils/nextcloud-migration/package.json @@ -17,8 +17,12 @@ "dependencies": { "axios": "^1.4.0", "express": "^4.18.2", + "form-auto-content": "^3.2.1", + "formdata-node": "^6.0.3", "ldapjs": "^3.0.2", - "ldif": "^0.5.1" + "ldif": "^0.5.1", + "pino": "^8.17.1", + "pino-pretty": "^10.3.0" }, "devDependencies": { "@jest/globals": "^29.7.0", diff --git a/tdrive/backend/utils/nextcloud-migration/src/express_server.ts b/tdrive/backend/utils/nextcloud-migration/src/express_server.ts index ca21e30f..730dc9ab 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/express_server.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/express_server.ts @@ -1,5 +1,6 @@ import express, { Express, Request, Response } from "express"; import { NextcloudMigration, NextcloudMigrationConfiguration } from './nextcloud_migration.js'; +import { logger } from "./logger" const app: Express = express(); const port = process.env.SERVER_PORT || 3000; @@ -47,7 +48,7 @@ const nextcloud = new NextcloudMigration(config); app.post("/", async (req: Request, res: Response) => { const params = req.body; - console.log(`Got request for data synchronization with params: ${params}`) + logger.info(`Got request for data synchronization with params: ${params}`) if (!params || !params["username"] || !params.password) { res.status(400).send("Username and password for nextcloud are required"); } @@ -62,5 +63,5 @@ app.post("/", async (req: Request, res: Response) => { }); app.listen(port, () => { - console.log(`[server]: Server is running at http://localhost:${port}`); + logger.info(`[server]: Server is running at http://localhost:${port}`); }); diff --git a/tdrive/backend/utils/nextcloud-migration/src/ldap_user.ts b/tdrive/backend/utils/nextcloud-migration/src/ldap_user.ts index 1749a626..55f123c8 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/ldap_user.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/ldap_user.ts @@ -1,4 +1,5 @@ import ldap, { SearchEntry, SearchOptions } from 'ldapjs'; +import { logger } from "./logger" export type LdapConfiguration = { url: string, @@ -29,7 +30,7 @@ export class LdapUser { if (error) { reject(new Error("Authentication error")); } else { - console.log("Successfully authenticated in LDAP") + logger.info("Successfully authenticated in LDAP") resolve(this.client); } }); @@ -44,11 +45,11 @@ export class LdapUser { reconnect: true }); this.client.on('connect', (res) => { - console.log("Connected to LDAP") + logger.info("Connected to LDAP") resolve(this.auth("", "")); }) this.client.on('connectionError', (error) => { - console.log("Error connecting to LDAP") + logger.info("Error connecting to LDAP") reject(error); }); } @@ -59,20 +60,20 @@ export class LdapUser { const search = await this.search(username); return new Promise((resolve, reject) => { search.on('error', (err) => { - console.log("ERROR"); - console.log(err); + logger.info("ERROR"); + logger.info(err); }); search.on('searchRequest', (searchRequest) => { - console.log('searchRequest: ', searchRequest.messageId); + logger.info('searchRequest: ', searchRequest.messageId); }); search.on('searchEntry', (entry) => { - console.log('entry: ' + JSON.stringify(entry)); + logger.info('entry: ' + JSON.stringify(entry)); }); search.on('searchReference', (referral) => { - console.log('referral: ' + referral.uris.join()); + logger.info('referral: ' + referral.uris.join()); }); search.on('end', (err) => { - console.log("END"); + logger.info("END"); }); }); } @@ -84,14 +85,14 @@ export class LdapUser { attributes: ['cn', 'sn'], scope: 'sub', } as SearchOptions; - console.log(`Search in ${this.config.baseDn} with options`); + logger.info(`Search in ${this.config.baseDn} with options`); // Perform search this.client?.search(this.config.baseDn, opts, (error, res) => { if (error) { console.error("Search error", error); reject(error) } else { - console.log("returning search callback"); + logger.info("returning search callback"); resolve(res); } }); diff --git a/tdrive/backend/utils/nextcloud-migration/src/logger.ts b/tdrive/backend/utils/nextcloud-migration/src/logger.ts new file mode 100644 index 00000000..b37badd1 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/src/logger.ts @@ -0,0 +1,9 @@ +import pino from "pino"; + +export type TDriveLogger = pino.Logger; + +export const logger = pino({ + name: "TDriveNextCloudMigration", + level: "debug", + prettyPrint: false, +} as pino.LoggerOptions); diff --git a/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts b/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts index aa46ab69..334a1fa0 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts @@ -1,8 +1,11 @@ import { exec } from 'child_process'; +// @ts-ignore import fs from 'fs'; -import { LdapUser } from './shell_ldap_user.js'; +import { LdapUser } from './shell_ldap_user'; import { User } from './ldap_user.js'; -import { TwakeDriveClient } from './twake_client.js'; +import { TwakeDriveClient, TwakeDriveUser } from './twake_client'; +import path from 'path'; +import { logger } from "./logger" export type NextcloudMigrationConfiguration = { ldap: { @@ -26,7 +29,7 @@ export class NextcloudMigration { private ldap: LdapUser; - private driveClient: TwakeDriveClient; + driveClient: TwakeDriveClient; constructor(config: NextcloudMigrationConfiguration) { this.config = config; @@ -37,11 +40,14 @@ export class NextcloudMigration { async migrate(username: string, password: string) { const dir = this.createTmpDir(username); try { - // await this.download(username, password, dir); const user = await this.getLDAPUser(username); //create user if needed Twake Drive - await this.driveClient.createUser(user); + const driveUser = await this.driveClient.createUser(user); + console.log(`Drive user ${driveUser.id} created`); + //download all files from nextcloud to tmp dir + await this.download(username, password, dir); //upload files to the Twake Drive + await this.upload(driveUser, dir); } catch (e) { console.error('Error downloading files from next cloud', e); throw e; @@ -99,4 +105,46 @@ export class NextcloudMigration { console.log(`Directory ${dir} deleted`); } + async upload(user: TwakeDriveUser, sourceDirPath: string, parentDirId = "user_" + user.id) { + const dirsToUpload: Map = new Map(); + const filesToUpload: string[] = []; + + const parent = await this.driveClient.getDocument(parentDirId); + + const exists = (filename: string) => { + return parent.children.filter(i => i.name.startsWith(path.parse(filename).name)).length > 0; + } + + logger.debug(`Reading content of the directory ${sourceDirPath} ...`) + fs.readdirSync(sourceDirPath).forEach(function (name) { + const filePath = path.join(sourceDirPath, name); + const stat = fs.statSync(filePath); + if (exists(name)) { + logger.info(`File ${name} already exists`); + } else { + if (stat.isFile()) { + logger.info(`Add file for future upload ${filePath}`); + filesToUpload.push(filePath) + } else if (stat.isDirectory()) { + logger.info(`Add directory for the future upload ${filePath}`); + dirsToUpload.set(name, filePath); + } + } + }); + + //upload all files + logger.debug(`UPLOAD FILES FOR ${sourceDirPath}`) + for (const file of filesToUpload) { + logger.debug(`Upload file ${file}`) + await this.driveClient.createFile(file, parentDirId); + } + + logger.debug(`UPLOAD DIRS FOR ${sourceDirPath}`) + for (const [name, path] of dirsToUpload) { + logger.info(`Create directory ${name}`); + const dir = await this.driveClient.createDirectory(name, parentDirId) + await this.upload(user, path, dir.id); + } + } + } \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/src/shell_ldap_user.ts b/tdrive/backend/utils/nextcloud-migration/src/shell_ldap_user.ts index 0c4f2e15..203c5fad 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/shell_ldap_user.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/shell_ldap_user.ts @@ -1,6 +1,7 @@ import { LdapConfiguration, User } from './ldap_user'; import { exec } from 'child_process'; import ldif from 'ldif'; +import { logger } from "./logger" export class LdapUser { @@ -13,13 +14,13 @@ export class LdapUser { async find(username: string): Promise { return new Promise((resolve, reject) => { let cmd = `ldapsearch -x -H ${this.config.url} -b '${this.config.baseDn}' '(uid=${username})'`; - console.log("Executing command to get data from LDAP for " + username); + logger.info("Executing command to get data from LDAP for " + username); exec(cmd, (error, stdout, stderr) => { if (stderr) { - console.log("ERROR: " + stderr); + logger.info("ERROR: " + stderr); } if (error) { - console.log(`ERROR running sync for the user: ${error.message}`); + logger.info(`ERROR running sync for the user: ${error.message}`); reject(new Error(error.message)); } else { if (stdout) { @@ -28,7 +29,7 @@ export class LdapUser { stdout = stdout.substring(0, stdout.lastIndexOf("# search result")) } let obj = ldif.parse(stdout).shift().toObject({}); - console.log(obj); + logger.info(obj); resolve({ lastName: obj.attributes.sn, firstName: obj.attributes.givenName, @@ -39,7 +40,7 @@ export class LdapUser { resolve({ } as User); } } else { - console.log("No user"); + logger.info("No user"); resolve({ } as User); } } diff --git a/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts b/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts index ccd1e9e1..5bb47646 100644 --- a/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts +++ b/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts @@ -1,15 +1,18 @@ -import axios, { AxiosError } from 'axios'; +import axios from 'axios'; import { User } from './ldap_user'; +import FormData from 'form-data'; +// @ts-ignore +import fs from 'fs'; +import { logger } from './logger'; type TwakeClientConfiguration = { url: string, - credentials: { + credentials: { appId: string, secret: string, } } - export class TwakeDriveClient { private config: TwakeClientConfiguration; @@ -17,23 +20,105 @@ export class TwakeDriveClient { constructor(config: TwakeClientConfiguration) { this.config = config; //remove the trailing '/' - this.config.url = this.config.url.replace(/\/$/, ''); + if (this.config && this.config.url) { + this.config.url = this.config.url.replace(/\/$/, ''); + } } - async createUser(user: User) { + private async uploadFile(path: string) { + logger.info(`Upload ${path}`); + const client = await this.client(); + + const form = new FormData(); + form.append('file', fs.createReadStream(path)); + + const response + = await client.post(`${this.config.url}/internal/services/files/v1/companies/00000000-0000-4000-0000-000000000000/files`, form); + + logger.info(response.data); + + return response.data?.resource; + } + + async createFile(path: string, parent_id: string) { + const file = await this.uploadFile(path); + return this.createDocumentFromFile(file, parent_id); + } + + async createDirectory(name: string, parent: string) { + return this.createDocument({ + company_id: '00000000-0000-4000-0000-000000000000', + name: name, + parent_id: parent, + is_directory: true, + }, {}); + } + + private async createDocumentFromFile(file: any, parent_id: string) { + const item = { + name: file.metadata.name, + parent_id: parent_id, + company_id: file.company_id, + }; + + const version = { + file_metadata: { + name: file.metadata.name, + size: file.upload_data?.size, + thumbnails: [], + external_id: file.id, + }, + }; + return await this.createDocument(item, version); + }; + + private async createDocument(item: any, version: any) { + const response + = await (await this.client()).post(`${this.config.url}/internal/services/documents/v1/companies/00000000-0000-4000-0000-000000000000/item`, { + item, + version, + }); + logger.info(response); + return response.data; + }; + + async getDocument(id: string) { + logger.info(`Get information for the doc ${id}`); + const response + = await (await this.client()).get(`${this.config.url}/internal/services/documents/v1/companies/00000000-0000-4000-0000-000000000000/item/${id}`, { + }); + logger.info(response); + return response.data; + } + + async getUser(id: string) { const client = await this.client(); try { - const response = await client.post(this.config.url + "/api/sync", + let url = `${this.config.url}/internal/services/users/v1/users/${id}`; + logger.info(`Get user company with ${url}`); + const response = await client.get(url); + logger.info(response.data); + return response?.data?.resource as TwakeDriveUser; + } catch (e) { + logger.info(`Error for ${id}: ${e.message}, body: ${e.response?.data?.message}`); + throw e; + } + } + + async createUser(user: User): Promise { + const client = await this.client(); + try { + const response = await client.post(this.config.url + '/api/sync', { first_name: user.firstName, last_name: user.lastName, - email: user.email + email: user.email, }); - return response.data; - } catch(e){ - console.log(`Error for ${JSON.stringify(user)}: ${e.message}, body: ${e.response?.data?.message}`); + return response.data as TwakeDriveUser; + } catch (e) { + logger.info(`Error for ${JSON.stringify(user)}: ${e.message}, body: ${e.response?.data?.message}`); throw e; - }; + } } private async client() { @@ -72,16 +157,18 @@ export class TwakeDriveClient { console.info('Using token ', this.config.credentials.appId, this.config.credentials.secret); console.info(`POST ${this.config.url}/api/console/v1/login`); console.info(`Basic ${Buffer.from(`${this.config.credentials.appId}:${this.config.credentials.secret}`).toString('base64')}`); - throw new Error("Unable to get access to token, see precious errors for details."); + throw new Error('Unable to get access to token, see precious errors for details.'); } }; } -type TwakeDriveUser = { +export type TwakeDriveUser = { first_name: string; last_name: string; email: string; + id: string, + email_canonical: string, } interface IApiServiceApplicationTokenRequestParams { diff --git a/tdrive/backend/utils/nextcloud-migration/test/ldap.test.ts b/tdrive/backend/utils/nextcloud-migration/test/ldap.test.ts index 83646845..3d26d71d 100644 --- a/tdrive/backend/utils/nextcloud-migration/test/ldap.test.ts +++ b/tdrive/backend/utils/nextcloud-migration/test/ldap.test.ts @@ -2,7 +2,7 @@ import {describe, expect, test} from '@jest/globals'; import { LdapUser } from '../src/shell_ldap_user'; import { LdapConfiguration } from '../src/ldap_user'; -//integration debug test +//FOR LOCAL DEBUG PURPOSE ONLY, ITS NOT A TEST describe.skip('ldap module', () => { test('ldap returns user info', async () => { diff --git a/tdrive/backend/utils/nextcloud-migration/test/migration.test.ts b/tdrive/backend/utils/nextcloud-migration/test/migration.test.ts new file mode 100644 index 00000000..21c3fda4 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/test/migration.test.ts @@ -0,0 +1,26 @@ +import { expect, test } from '@jest/globals'; +import { TwakeDriveClient, TwakeDriveUser } from '../src/twake_client'; +import { NextcloudMigration, NextcloudMigrationConfiguration } from '../src/nextcloud_migration'; + +//FOR LOCAL DEBUG PURPOSE ONLY, ITS NOT A TEST +describe.skip('nextcloud migration module', () => { + + test('upload directory', async () => { + //given + const subj = new NextcloudMigration({ + drive: { + url: "http://localhost:4000", + credentials: { + appId: "tdrive_onlyoffice", + secret: "c1cc66db78e1d3bb4713c55d5ab2" + } + } + } as NextcloudMigrationConfiguration); + + let user = await subj.driveClient.createUser({firstName: "DWHO", lastName: "DWHO", email: "dwho@example.com", uid: "test"}); + + //when + await subj.upload(user, "/Users/shepilov/WebstormProjects/TDrive/Documentation/docs/onprem"); + }); + +}); \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/test/twake.test.ts b/tdrive/backend/utils/nextcloud-migration/test/twake.test.ts index 6a34aae0..65aba406 100644 --- a/tdrive/backend/utils/nextcloud-migration/test/twake.test.ts +++ b/tdrive/backend/utils/nextcloud-migration/test/twake.test.ts @@ -1,23 +1,34 @@ import { expect, test } from '@jest/globals'; import { TwakeDriveClient } from '../src/twake_client'; -//integration debug test +//FOR LOCAL DEBUG PURPOSE ONLY, ITS NOT A TEST describe.skip('twake client module', () => { - test('twake creates new user', async () => { - //given - const subj = new TwakeDriveClient({ - url: "https://tdrive.qa.lin-saas.com/", - credentials: { - appId: "tdrive_onlyoffice", - secret: "QWERTY" - } - }) + const subj = new TwakeDriveClient({ + url: "http://localhost:4000", + credentials: { + appId: "tdrive_onlyoffice", + secret: "c1cc66db78e1d3bb4713c55d5ab2" + } + }) + test('twake creates new user and files for the user', async () => { //when - const response = await subj.createUser({firstName: "Test", lastName: "User", email: "test@example.com", uid: "test"}); - expect(response).toBeDefined(); - console.log(response); + let user = await subj.createUser({firstName: "DWHO", lastName: "DWHO", email: "dwho@example.com", uid: "test"}); + expect(user).toBeDefined(); + expect(user.id).toBeDefined(); + + user = await subj.getUser(user.id); + expect(user).toBeDefined(); + expect(user.id).toBeDefined(); + + const dir = await subj.createDirectory("My DIR", "user_" + user.id) + const doc = await subj.createFile("/Users/shepilov/WebstormProjects/TDrive/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts", dir.id); + expect(doc).toBeDefined(); + expect(doc.id).toBeDefined(); + + const item = await subj.getDocument("user_" + user.id); + expect(item.children.filter(i => i.name.startsWith("My DIR")).length).toBeGreaterThan(0); }); }); \ No newline at end of file