diff --git a/.github/workflows/ldap-sync.yml b/.github/workflows/ldap-sync.yml index 54088a89..88949816 100644 --- a/.github/workflows/ldap-sync.yml +++ b/.github/workflows/ldap-sync.yml @@ -4,7 +4,7 @@ on: pull_request: branches: [main] paths: - - "tdrive/backend/utils/**" + - "tdrive/backend/utils/ldap-sync/**" jobs: ldap-sync-build: @@ -12,4 +12,4 @@ jobs: steps: - uses: actions/checkout@v2 - name: Build ldap sync - run: cd tdrive/backend/utils/ldap-sync && npm i && npm run build \ No newline at end of file + run: cd tdrive/backend/utils/ldap-sync && npm i && npm run build diff --git a/.github/workflows/nextcloud-migration.yml b/.github/workflows/nextcloud-migration.yml new file mode 100644 index 00000000..156eb2f1 --- /dev/null +++ b/.github/workflows/nextcloud-migration.yml @@ -0,0 +1,15 @@ +name: nextcloud-migration-build + +on: + pull_request: + branches: [main] + paths: + - "tdrive/backend/utils/nextcloud-migration/**" + +jobs: + ldap-sync-build: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - name: Build ldap sync + run: cd tdrive/backend/utils/nextcloud-migration && npm i && npm run build diff --git a/.github/workflows/publish-nextcloud-migration.yml b/.github/workflows/publish-nextcloud-migration.yml new file mode 100644 index 00000000..bc9ba17d --- /dev/null +++ b/.github/workflows/publish-nextcloud-migration.yml @@ -0,0 +1,32 @@ +name: publish-nextcloud-migration + +on: + push: + branches: [main] + paths: + - "tdrive/backend/utils/nextcloud-migration/**" + - "tdrive/docker/tdrive-nextcloud-migration/**" + +jobs: + publish-node: + runs-on: ubuntu-20.04 + steps: + - name: Set env to production + if: endsWith(github.ref, '/main') + run: 'echo "DOCKERTAG=latest" >> $GITHUB_ENV' + - name: "Push to the registry following labels:" + run: | + echo "${{ env.DOCKERTAG }},${{ env.DOCKERTAGVERSION }}" + - uses: actions/checkout@v2 + - name: Publish to Registry + uses: elgohr/Publish-Docker-Github-Action@v5 + with: + name: tdrive/tdrive-nextcloud-migration + username: ${{ secrets.REGISTRY_USERNAME }} + password: ${{ secrets.REGISTRY_PASSWORD }} + workdir: tdrive + registry: docker-registry.linagora.com + context: . + target: production + buildoptions: "-t docker-registry.linagora.com/tdrive/tdrive-nextcloud-migration -f docker/tdrive-nextcloud-migration/Dockerfile" + tags: "${{ env.DOCKERTAG }},${{ env.DOCKERTAGVERSION }}" \ No newline at end of file diff --git a/tdrive/backend/node/config/custom-environment-variables.json b/tdrive/backend/node/config/custom-environment-variables.json index 75146a0b..e31478b5 100644 --- a/tdrive/backend/node/config/custom-environment-variables.json +++ b/tdrive/backend/node/config/custom-environment-variables.json @@ -63,6 +63,13 @@ "useAuth": "SEARCH_ES_USE_AUTH", "username": "SEARCH_ES_USERNAME", "password": "SEARCH_ES_PASSWORD" + }, + "opensearch": { + "endpoint": "SEARCH_OS_ENDPOINT", + "flushInterval": "SEARCH_OS_FLUSHINTERVAL", + "useAuth": "SEARCH_OS_USE_AUTH", + "username": "SEARCH_OS_USERNAME", + "password": "SEARCH_OS_PASSWORD" } }, "storage": { diff --git a/tdrive/backend/node/package-lock.json b/tdrive/backend/node/package-lock.json index 7a363890..3178bda6 100644 --- a/tdrive/backend/node/package-lock.json +++ b/tdrive/backend/node/package-lock.json @@ -1349,6 +1349,25 @@ "fastq": "^1.6.0" } }, + "@opensearch-project/opensearch": { + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-2.4.0.tgz", + "integrity": "sha512-r0ZNIlDxAua1ZecOBJ8qOXshf2ZQhNKmfly7o0aNuACf0pDa6Et/8mWMZuaFOu7xlNEeRNB7IjDQUYFy2SPElw==", + "requires": { + "aws4": "^1.11.0", + "debug": "^4.3.1", + "hpagent": "^1.2.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0" + }, + "dependencies": { + "hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==" + } + } + }, "@segment/loosely-validate-event": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@segment/loosely-validate-event/-/loosely-validate-event-2.0.0.tgz", @@ -2456,6 +2475,11 @@ "queue-microtask": "^1.1.2" } }, + "aws4": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", + "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==" + }, "axios": { "version": "0.21.4", "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz", diff --git a/tdrive/backend/node/package.json b/tdrive/backend/node/package.json index aaecc10c..95b47665 100644 --- a/tdrive/backend/node/package.json +++ b/tdrive/backend/node/package.json @@ -107,6 +107,7 @@ "@fastify/formbody": "^6.0.0", "@fastify/static": "^5.0.1", "@ffprobe-installer/ffprobe": "^1.4.1", + "@opensearch-project/opensearch": "^2.4.0", "@sentry/node": "^6.19.7", "@sentry/tracing": "^6.19.7", "@socket.io/redis-adapter": "^7.2.0", diff --git a/tdrive/backend/node/src/core/platform/services/search/adapters/opensearch/opensearch.ts b/tdrive/backend/node/src/core/platform/services/search/adapters/opensearch/opensearch.ts new file mode 100644 index 00000000..2237b2a4 --- /dev/null +++ b/tdrive/backend/node/src/core/platform/services/search/adapters/opensearch/opensearch.ts @@ -0,0 +1,328 @@ +import { logger } from "../../../../framework"; +import _ from "lodash"; +import { + ColumnDefinition, + EntityDefinition, + EntityTarget, + FindFilter, + FindOptions, + IndexedEntity, + SearchAdapterInterface, + SearchConfiguration, +} from "../../api"; +import { SearchAdapter } from "../abstract"; +import { DatabaseServiceAPI } from "../../../database/api"; +import { getEntityDefinition, unwrapPrimarykey } from "../../api"; +import { ListResult, Paginable, Pagination } from "../../../../framework/api/crud-service"; +import { asciiFold, parsePrimaryKey, stringifyPrimaryKey } from "../utils"; +import { buildSearchQuery } from "../elasticsearch/search"; +import { Client } from "@opensearch-project/opensearch"; + +type Operation = { + index?: { _index: string; _id: string }; + delete?: { _index: string; _id: string }; + [key: string]: any; +}; + +export default class OpenSearch extends SearchAdapter implements SearchAdapterInterface { + private client: Client; + private bulkReaders = 0; + private buffer: Operation[] = []; + private name = "ElasticSearch"; + + constructor( + readonly database: DatabaseServiceAPI, + readonly configuration: SearchConfiguration["elasticsearch"], + ) { + super(); + } + + public async connect() { + try { + const clientOptions: any = { + node: this.configuration.endpoint, + ssl: { + rejectUnauthorized: false, + }, + }; + + if (this.configuration.useAuth) { + logger.info("Using auth for ES client"); + clientOptions.auth = { + username: this.configuration.username, + password: this.configuration.password, + }; + } + + this.client = new Client(clientOptions); + } catch (e) { + logger.error( + `Unable to connect to ElasticSearch for options: ${JSON.stringify({ + node: this.configuration.endpoint, + auth: { + useAuth: this.configuration.useAuth, + username: this.configuration.username, + password: this.configuration.password, + }, + ssl: { + rejectUnauthorized: false, + }, + })} at: ${this.configuration.endpoint}`, + ); + } + this.startBulkReader(); + } + + private async createIndex( + entity: EntityDefinition, + _columns: { [name: string]: ColumnDefinition }, + ) { + if (!entity.options?.search) { + return; + } + + const name = entity.options?.search?.index || entity.name; + const mapping = entity.options?.search?.esMapping; + + try { + await this.client.indices.get({ + index: name, + }); + logger.info(`Index "${name}" already created`); + } catch (e) { + logger.info(`Create index ${name} with mapping %o`, mapping); + + const indice = { + index: name, + body: { + settings: { + analysis: { + analyzer: { + folding: { + tokenizer: "standard", + filter: ["lowercase", "asciifolding"], + }, + }, + }, + }, + mappings: { ...mapping, _source: { enabled: false } }, + }, + }; + + const rep = await this.client.indices.create(indice, { ignore: [400] }); + + if (rep.statusCode !== 200) { + logger.error(`${this.name} - ${JSON.stringify(rep.body)}`); + } + } + } + + public async upsert(entities: any[]) { + for (const entity of entities) { + const { entityDefinition, columnsDefinition } = getEntityDefinition(entity); + const pkColumns = unwrapPrimarykey(entityDefinition); + + await this.ensureIndex(entityDefinition, columnsDefinition, this.createIndex.bind(this)); + + if (!entityDefinition.options?.search) { + return; + } + + if ( + entityDefinition.options.search.shouldUpdate && + !entityDefinition.options.search.shouldUpdate(entity) + ) { + return; + } + + if (!entityDefinition.options?.search?.source) { + logger.info(`Unable to do operation upsert to elasticsearch for doc ${entity}`); + return; + } + + const body = { + ..._.pick(entity, ...pkColumns), + ...entityDefinition.options.search.source(entity), + }; + + Object.keys(entityDefinition.options?.search.esMapping?.properties || []).forEach( + (key: string) => { + const mapping: any = entityDefinition.options?.search?.esMapping?.properties[key]; + if (mapping.type === "text") { + body[key] = asciiFold(body[key]).toLocaleLowerCase(); + } + }, + ); + + const index = entityDefinition.options?.search?.index || entityDefinition.name; + + const record: Operation = { + index: { + _index: index, + _id: stringifyPrimaryKey(entity), + }, + ...body, + }; + + logger.info(`Add operation upsert to elasticsearch for doc ${record.id}`); + + this.buffer.push(record); + } + + this.startBulkReader(); + } + + public async remove(entities: any[]) { + for (const entity of entities) { + const { entityDefinition, columnsDefinition } = getEntityDefinition(entity); + + await this.ensureIndex(entityDefinition, columnsDefinition, this.createIndex.bind(this)); + + if (!entityDefinition.options?.search) { + return; + } + + const index = entityDefinition.options?.search?.index || entityDefinition.name; + + const record: Operation = { + delete: { + _index: index, + _id: stringifyPrimaryKey(entity), + }, + }; + + logger.info(`Add operation remove from elasticsearch for doc ${record.id}`); + + this.buffer.push(record); + } + + this.startBulkReader(); + } + + private async startBulkReader() { + if (this.bulkReaders > 0) { + return; + } + + logger.info("Start new Elasticsearch bulk reader."); + this.bulkReaders += 1; + + let buffer; + do { + await new Promise(r => + setTimeout(r, parseInt(`${this.configuration.flushInterval}`) || 3000), + ); + buffer = this.buffer; + } while (buffer.length === 0); + this.buffer = []; + + try { + await this.client.helpers.bulk({ + flushInterval: 1, + datasource: buffer, + onDocument: (doc: Operation) => { + if (doc.delete) { + logger.info( + `Operation ${"DELETE"} pushed to elasticsearch index ${doc.delete._index} (doc.id: ${ + doc.delete._id + })`, + ); + return { + delete: doc.delete, + }; + } + if (doc.index) { + logger.info( + `Operation ${"INDEX"} pushed to elasticsearch index ${doc.index._index} (doc.id: ${ + doc.index._id + })`, + ); + return { + index: doc.index, + ...doc.index, + }; + } + return null; + }, + onDrop: res => { + const doc = res.document; + logger.error( + `Operation ${ + doc.action + } was droped while pushing to elasticsearch index ${JSON.stringify( + doc.index, + )} (doc.id: ${doc.id})`, + ); + logger.error(res.error); + }, + }); + } catch (err) { + logger.error(`${this.name} - An error occured with the bulk reader`); + logger.error(err); + } + + logger.info("Elasticsearch bulk flushed."); + this.bulkReaders += -1; + + this.startBulkReader(); + } + + public async search( + _table: string, + entityType: EntityTarget, + filters: FindFilter, + options: FindOptions = {}, + ) { + const instance = new (entityType as any)(); + const { entityDefinition } = getEntityDefinition(instance); + + const { esParams, esOptions } = buildSearchQuery(entityType, filters, options); + const esParamsWithScroll = { + ...esParams, + size: parseInt(options.pagination.limitStr || "100"), + scroll: "1m", + }; + + let esResponse: any; + + if (options.pagination.page_token) { + esResponse = await this.client.scroll( + { + scroll_id: options.pagination.page_token, + }, + esOptions, + ); + } else { + esResponse = await this.client.search(esParamsWithScroll, esOptions); + } + + if (esResponse.statusCode !== 200) { + logger.error(`${this.name} - ${JSON.stringify(esResponse.body)}`); + } + + const nextToken = esResponse.body?._scroll_id || ""; + const hits = esResponse.body?.hits?.hits || []; + + logger.debug(`${this.name} got response: ${JSON.stringify(esResponse)}`); + + const entities: IndexedEntity[] = []; + for await (const hit of hits) { + try { + entities.push({ + primaryKey: parsePrimaryKey(entityDefinition, hit._id), + score: hit._score, + }); + } catch (err) { + logger.error( + `${this.name} failed to get entity from search result: ${JSON.stringify( + hit._id, + )}, ${JSON.stringify(err)}`, + ); + } + } + + const nextPage: Paginable = new Pagination(nextToken, options.pagination.limitStr || "100"); + + return new ListResult(entityDefinition.type, entities, nextPage); + } +} diff --git a/tdrive/backend/node/test/http/elasticsearch.http b/tdrive/backend/node/test/http/elasticsearch.http index bcc1eae8..280824a0 100644 --- a/tdrive/backend/node/test/http/elasticsearch.http +++ b/tdrive/backend/node/test/http/elasticsearch.http @@ -1,8 +1,3 @@ -# The task is: -# to find all the files that I havee access to, it shoud be a doc files, -# that was modified during the last week and share with my by Diana Potoking - - ### Drop index DELETE https://localhost:9200/tdrive_files_extended @@ -11,6 +6,7 @@ DELETE https://localhost:9200/drive_files ### Create index PUT https://localhost:9200/tdrive_files_extended +Authorization: Basic admin admin Content-Type: application/json { diff --git a/tdrive/backend/node/test/http/opensearch.http b/tdrive/backend/node/test/http/opensearch.http new file mode 100644 index 00000000..facded50 --- /dev/null +++ b/tdrive/backend/node/test/http/opensearch.http @@ -0,0 +1,248 @@ +### Drop index +DELETE https://localhost:9200/tdrive_files_extended +Authorization: Basic admin admin + +### Delete Index +DELETE https://localhost:9200/drive_files +Authorization: Basic admin admin + +### Create index +PUT https://localhost:9200/tdrive_files_extended?pretty +Authorization: Basic admin admin +Content-Type: application/json + +{ + "aliases": {}, + "mappings": { + "_source": { + "enabled": true + } + } +} + +### Check that it is created +GET https://localhost:9200/_cat/indices +Authorization: Basic admin admin + +### GET index info +GET https://localhost:9200/drive_files +Authorization: Basic admin admin + +### GET data to the index +GET https://localhost:9200/drive_files/_doc/8c03c5a1-0146-11ee-82d9-f503f8a58e9d +Authorization: Basic admin admin +Content-Type: application/json + +### Put data to the index +PUT https://localhost:9200/tdrive_files_extended/_doc/0001 +Content-Type: application/json + +{ + "file_name": "file0001.txt", + "access_info": [ + { + "type": "user", + "id": "user01", + "level": "read", + "grantor": "user02" + } + ] +} + +### +PUT https://localhost:9200/tdrive_files_extended/_doc/8c03c5a1-0146-11ee-82d9-f503f8a58e9d +Content-Type: application/json + +{ + "file_name": "file0002.png", + "access_info": [ + { + "type": "user", + "id": "user01", + "level": "read", + "grantor": "user03" + } + ] +} + +### Search all in the index by last_modified +GET https://localhost:9200/drive_files/_search +Content-Type: application/json + +{ + "query": { + + "bool": { + "boost": 1, + "must": [ + { + "range": + { + "last_modified": { + "gte": "0" + } + } + }, + { + "range": + { + "last_modified": { + "lte": "1685958426348" + } + } + }, + { + "bool": { + "should": [ + { + "match": { + "access_entities": { + "query": "edfb30e0-0385-11ee-80e8-41892804174d", + "operator": "AND" + } + } + }, + { + "match": { + "access_entities": { + "query": "edd447f1-0385-11ee-80e8-41892804174d", + "operator": "AND" + } + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "match": { + "company_id": { + "query": "edd447f1-0385-11ee-80e8-41892804174d", + "operator": "AND" + } + } + } + ], + "minimum_should_match": 1 + } + } + ] + } + } +} + + +### Search all in the index and order by name +GET https://localhost:9200/drive_files/_search +Content-Type: application/json + +{ + "query": { + "bool": { + "boost": 1, + "must": [ + { + "bool": { + "should": [ + { + "match": { + "access_entities": { + "query": "12fb6a40-03b3-11ee-af31-a569970b7f74", + "operator": "AND" + } + } + }, + { + "match": { + "access_entities": { + "query": "12f290a1-03b3-11ee-af31-a569970b7f74", + "operator": "AND" + } + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "match": { + "company_id": { + "query": "12f290a1-03b3-11ee-af31-a569970b7f74", + "operator": "AND" + } + } + } + ], + "minimum_should_match": 1 + } + } + ] + } + }, + "sort": [ + { + "name": "asc" + } + ] +} + +### Search all in the index and order by name +GET https://localhost:9200/drive_files/_search +Content-Type: application/json + +{ + "query": { + "bool": { + "boost": 1, + "must": [ + { + "bool": { + "should": [ + { + "match": { + "access_entities": { + "query": "4090dce0-03ba-11ee-97d7-fb8fa15d86a1", + "operator": "AND" + } + } + }, + { + "match": { + "access_entities": { + "query": "40808931-03ba-11ee-97d7-fb8fa15d86a1", + "operator": "AND" + } + } + } + ], + "minimum_should_match": 1 + } + }, + { + "bool": { + "should": [ + { + "match": { + "company_id": { + "query": "40808931-03ba-11ee-97d7-fb8fa15d86a1", + "operator": "AND" + } + } + } + ], + "minimum_should_match": 1 + } + } + ] + } + } +} + + + diff --git a/tdrive/backend/utils/ldap-sync/package.json b/tdrive/backend/utils/ldap-sync/package.json index 1ca7d880..fed1d741 100644 --- a/tdrive/backend/utils/ldap-sync/package.json +++ b/tdrive/backend/utils/ldap-sync/package.json @@ -9,7 +9,7 @@ "build:ts": "tsc", "build:clean": "rimraf ./dist", "sync": "node dist/index.js", - "test": "echo \"Error: no test specified\" && exit 1" + "test": "jest" }, "keywords": [], "author": "", diff --git a/tdrive/backend/utils/nextcloud-migration/.nvmrc b/tdrive/backend/utils/nextcloud-migration/.nvmrc new file mode 100644 index 00000000..25bf17fc --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/.nvmrc @@ -0,0 +1 @@ +18 \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/README.MD b/tdrive/backend/utils/nextcloud-migration/README.MD new file mode 100644 index 00000000..8ada8be0 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/README.MD @@ -0,0 +1,29 @@ + +## About + +This is a service for migrating date from Nextcloud to Twake Drive + +## Development + +1. To build image locally + ```sh + docker build -t twakedrive/tdrive-nextcloud-migration -f docker/tdrive-nextcloud-migration/Dockerfile . + ``` +2. Run it with Docker + ```sh + docker run -p 4001:4001 \ + -e SERVER_PORT=4001 \ + -e LDAP_BASE=dc=example,dc=com \ + -e LDAP_URL=ldap://ldap.twake.app:389 \ + -e TMP_DIR=/tmp \ + -e NEXTCLOUD_URL=https://nextcloud.fr \ + -e TWAKE_DRIVE_URL=https://tdrive.qa.lin-saas.com/ \ + -e TWAKE_DRIVE_APP_ID= \ + -e TWAKE_DRIVE_SECRET= \ + twakedrive/tdrive-nextcloud-migration + ``` + +2. Request example + ```sh + curl -X POST --data "username=luke&password=iamyourfather" http://localhost:4001/ + ``` diff --git a/tdrive/backend/utils/nextcloud-migration/package.json b/tdrive/backend/utils/nextcloud-migration/package.json new file mode 100644 index 00000000..6a52178a --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/package.json @@ -0,0 +1,34 @@ +{ + "name": "nextcloud_migration", + "version": "1.0.0", + "description": "", + "type": "module", + "scripts": { + "build": "npm run build:clean && npm run build:ts", + "build:ts": "tsc", + "build:clean": "rimraf ./dist", + "prestart": "npm run build", + "start": "node dist/express_server.js", + "test": "jest" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "axios": "^1.4.0", + "express": "^4.18.2", + "ldapjs": "^3.0.2", + "ldif": "^0.5.1" + }, + "devDependencies": { + "@jest/globals": "^29.7.0", + "@types/express": "^4.17.21", + "@types/jest": "^29.5.11", + "@types/ldapjs": "^2.2.5", + "jest": "^29.7.0", + "rimraf": "^3.0.2", + "ts-jest": "^29.1.1", + "ts-node": "^10.9.2", + "typescript": "^5.3.3" + } +} diff --git a/tdrive/backend/utils/nextcloud-migration/src/express_server.ts b/tdrive/backend/utils/nextcloud-migration/src/express_server.ts new file mode 100644 index 00000000..ca21e30f --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/src/express_server.ts @@ -0,0 +1,66 @@ +import express, { Express, Request, Response } from "express"; +import { NextcloudMigration, NextcloudMigrationConfiguration } from './nextcloud_migration.js'; + +const app: Express = express(); +const port = process.env.SERVER_PORT || 3000; + +app.use(express.json()); +app.use(express.urlencoded()); + +const config: NextcloudMigrationConfiguration = { + ldap: { + baseDn: process.env.LDAP_BASE!, + url: process.env.LDAP_URL!, + }, + tmpDir: process.env.TMP_DIR || '/tmp', + nextcloudUrl: process.env.NEXTCLOUD_URL!, + drive: { + url: process.env.TWAKE_DRIVE_URL!, + credentials: { + appId: process.env.TWAKE_DRIVE_APP_ID!, + secret: process.env.TWAKE_DRIVE_SECRET!, + } + } +} + +if (!config.ldap.baseDn) { + throw new Error("LDAP base has to be set") +} +if (!config.ldap.url) { + throw new Error("LDAP url has to be set") +} +if (!config.drive.url) { + throw new Error("Twake Drive url host has to be set") +} +if (!config.drive.credentials.appId) { + throw new Error("Twake Drive application identifier host has to be set") +} +if (!config.nextcloudUrl) { + throw new Error("Nextcloud url has to be set") +} + +app.get("/", (req: Request, res: Response) => { + res.send("Hello, to run the the migration process you should send post request."); +}); + +const nextcloud = new NextcloudMigration(config); + +app.post("/", async (req: Request, res: Response) => { + const params = req.body; + console.log(`Got request for data synchronization with params: ${params}`) + if (!params || !params["username"] || !params.password) { + res.status(400).send("Username and password for nextcloud are required"); + } + try { + await nextcloud.migrate(params.username, params.password); + res.status(200).send("Sync DONE ✅"); + } catch (e) { + console.error(e) + res.status(500).send("Error during synchronization:: " + e.message) + } + +}); + +app.listen(port, () => { + console.log(`[server]: Server is running at http://localhost:${port}`); +}); diff --git a/tdrive/backend/utils/nextcloud-migration/src/ldap_user.ts b/tdrive/backend/utils/nextcloud-migration/src/ldap_user.ts new file mode 100644 index 00000000..1749a626 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/src/ldap_user.ts @@ -0,0 +1,101 @@ +import ldap, { SearchEntry, SearchOptions } from 'ldapjs'; + +export type LdapConfiguration = { + url: string, + baseDn: string, +} + +export type User = { + firstName: string, + lastName: string, + email: string, + uid: string +} + +// Doesn't work, fix it later, somehow none of the events is called for the search request +export class LdapUser { + + private config: LdapConfiguration; + + private client?: ldap.Client; + + constructor(config: LdapConfiguration) { + this.config = config; + } + + async auth(username: string, password: string) { + return new Promise((resolve, reject) => { + this.client?.bind(username, password, (error) => { + if (error) { + reject(new Error("Authentication error")); + } else { + console.log("Successfully authenticated in LDAP") + resolve(this.client); + } + }); + }); + } + + async connect() { + return new Promise((resolve, reject) => { + if (!this.client) { + this.client = ldap.createClient({ + url: this.config.url, + reconnect: true + }); + this.client.on('connect', (res) => { + console.log("Connected to LDAP") + resolve(this.auth("", "")); + }) + this.client.on('connectionError', (error) => { + console.log("Error connecting to LDAP") + reject(error); + }); + } + }); + } + + async find(username: string): Promise { + const search = await this.search(username); + return new Promise((resolve, reject) => { + search.on('error', (err) => { + console.log("ERROR"); + console.log(err); + }); + search.on('searchRequest', (searchRequest) => { + console.log('searchRequest: ', searchRequest.messageId); + }); + search.on('searchEntry', (entry) => { + console.log('entry: ' + JSON.stringify(entry)); + }); + search.on('searchReference', (referral) => { + console.log('referral: ' + referral.uris.join()); + }); + search.on('end', (err) => { + console.log("END"); + }); + }); + } + + async search(username: string): Promise { + return new Promise((resolve, reject) => { + const opts = { + filter: `(objectClass=*)`, + attributes: ['cn', 'sn'], + scope: 'sub', + } as SearchOptions; + console.log(`Search in ${this.config.baseDn} with options`); + // Perform search + this.client?.search(this.config.baseDn, opts, (error, res) => { + if (error) { + console.error("Search error", error); + reject(error) + } else { + console.log("returning search callback"); + resolve(res); + } + }); + }); + } + +} \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts b/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts new file mode 100644 index 00000000..aa46ab69 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/src/nextcloud_migration.ts @@ -0,0 +1,102 @@ +import { exec } from 'child_process'; +import fs from 'fs'; +import { LdapUser } from './shell_ldap_user.js'; +import { User } from './ldap_user.js'; +import { TwakeDriveClient } from './twake_client.js'; + +export type NextcloudMigrationConfiguration = { + ldap: { + baseDn: string, + url: string, + }, + drive: { + url: string, + credentials: { + appId: string, + secret: string, + } + }, + tmpDir: string, + nextcloudUrl: string +} + +export class NextcloudMigration { + + private config: NextcloudMigrationConfiguration; + + private ldap: LdapUser; + + private driveClient: TwakeDriveClient; + + constructor(config: NextcloudMigrationConfiguration) { + this.config = config; + this.ldap = new LdapUser(config.ldap); + this.driveClient = new TwakeDriveClient(this.config.drive); + } + + async migrate(username: string, password: string) { + const dir = this.createTmpDir(username); + try { + // await this.download(username, password, dir); + const user = await this.getLDAPUser(username); + //create user if needed Twake Drive + await this.driveClient.createUser(user); + //upload files to the Twake Drive + } catch (e) { + console.error('Error downloading files from next cloud', e); + throw e; + } finally { + this.deleteDir(dir); + } + } + + async download(username: string, password: string, dir: string) { + return new Promise((resolve, reject) => { + let cmd = `nextcloudcmd -s --non-interactive -u '${username}' -p '${password}' ${dir} ${this.config.nextcloudUrl}`; + console.log('Start downloading data from Nextcloud'); + exec(cmd, (error, stdout, stderr) => { + if (stderr) { + console.log('ERROR: ' + stderr); + } + if (stdout) { + console.log('OUT: ' + stdout); + } + if (error) { + console.log(`ERROR running sync for the user: ${error.message}`); + reject(error.message); + } else { + console.log('Download finished'); + resolve(''); + } + }); + }); + } + + async getLDAPUser(username: string): Promise { + const user = await this.ldap.find(username); + if (!user.email) { + throw new Error(`User ${username} not found`); + } + return user; + } + + createTmpDir(username: string) { + console.log('Creating tmp directory for the user data'); + const dir = this.config.tmpDir + '/' + username + '_' + new Date().getTime(); + if (!fs.existsSync(dir)) { + console.log(`Creating directory ${dir} ...`); + fs.mkdirSync(dir); + console.log(`Directory ${dir} created`); + } else { + this.deleteDir(dir); + } + return dir; + } + + deleteDir(dir: string) { + console.log(`Deleting directory ${dir} ...`); + fs.rmSync(dir, { recursive: true, force: true }); + console.log(`Directory ${dir} deleted`); + } + +} \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/src/shell_ldap_user.ts b/tdrive/backend/utils/nextcloud-migration/src/shell_ldap_user.ts new file mode 100644 index 00000000..0c4f2e15 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/src/shell_ldap_user.ts @@ -0,0 +1,50 @@ +import { LdapConfiguration, User } from './ldap_user'; +import { exec } from 'child_process'; +import ldif from 'ldif'; + +export class LdapUser { + + private config: LdapConfiguration; + + constructor(config: LdapConfiguration) { + this.config = config; + } + + async find(username: string): Promise { + return new Promise((resolve, reject) => { + let cmd = `ldapsearch -x -H ${this.config.url} -b '${this.config.baseDn}' '(uid=${username})'`; + console.log("Executing command to get data from LDAP for " + username); + exec(cmd, (error, stdout, stderr) => { + if (stderr) { + console.log("ERROR: " + stderr); + } + if (error) { + console.log(`ERROR running sync for the user: ${error.message}`); + reject(new Error(error.message)); + } else { + if (stdout) { + try { + if (stdout.lastIndexOf("# search result") > 0) { + stdout = stdout.substring(0, stdout.lastIndexOf("# search result")) + } + let obj = ldif.parse(stdout).shift().toObject({}); + console.log(obj); + resolve({ + lastName: obj.attributes.sn, + firstName: obj.attributes.givenName, + email: obj.attributes.mail, + uid: obj.attributes.uid} as User); + } catch (e) { + console.error(e) + resolve({ } as User); + } + } else { + console.log("No user"); + resolve({ } as User); + } + } + }); + }); + } + +} \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts b/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts new file mode 100644 index 00000000..ccd1e9e1 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/src/twake_client.ts @@ -0,0 +1,101 @@ +import axios, { AxiosError } from 'axios'; +import { User } from './ldap_user'; + +type TwakeClientConfiguration = { + url: string, + credentials: { + appId: string, + secret: string, + } +} + + +export class TwakeDriveClient { + + private config: TwakeClientConfiguration; + + constructor(config: TwakeClientConfiguration) { + this.config = config; + //remove the trailing '/' + this.config.url = this.config.url.replace(/\/$/, ''); + } + + async createUser(user: User) { + const client = await this.client(); + try { + const response = await client.post(this.config.url + "/api/sync", + { + first_name: user.firstName, + last_name: user.lastName, + email: user.email + }); + return response.data; + } catch(e){ + console.log(`Error for ${JSON.stringify(user)}: ${e.message}, body: ${e.response?.data?.message}`); + throw e; + }; + } + + private async client() { + return axios.create({ + baseURL: this.config.url, + headers: { + Authorization: `Bearer ${await (this.accessToken())}`, + }, + }); + } + + private async accessToken(): Promise { + try { + const response = await axios.post( + `${this.config.url}/api/console/v1/login`, + { + id: this.config.credentials.appId, + secret: this.config.credentials.secret, + }, + { + headers: { + Authorization: `Basic ${Buffer.from(`${this.config.credentials.appId}:${this.config.credentials.secret}`).toString('base64')}`, + }, + }, + ); + + const { + resource: { + access_token: { value }, + }, + } = response.data; + + return value; + } catch (error) { + console.error('failed to get application token', error); + console.info('Using token ', this.config.credentials.appId, this.config.credentials.secret); + console.info(`POST ${this.config.url}/api/console/v1/login`); + console.info(`Basic ${Buffer.from(`${this.config.credentials.appId}:${this.config.credentials.secret}`).toString('base64')}`); + throw new Error("Unable to get access to token, see precious errors for details."); + } + }; + +} + +type TwakeDriveUser = { + first_name: string; + last_name: string; + email: string; +} + +interface IApiServiceApplicationTokenRequestParams { + id: string; + secret: string; +} + +interface IApiServiceApplicationTokenResponse { + resource: { + access_token: { + time: number; + expiration: number; + value: string; + type: string; + }; + }; +} \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/test/ldap.test.ts b/tdrive/backend/utils/nextcloud-migration/test/ldap.test.ts new file mode 100644 index 00000000..83646845 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/test/ldap.test.ts @@ -0,0 +1,29 @@ +import {describe, expect, test} from '@jest/globals'; +import { LdapUser } from '../src/shell_ldap_user'; +import { LdapConfiguration } from '../src/ldap_user'; + +//integration debug test +describe.skip('ldap module', () => { + + test('ldap returns user info', async () => { + const ldap = new LdapUser({ + url: "ldap://auth.poc-mail-avocat.fr:389", + baseDn: "ou=users,dc=example,dc=com", + } as LdapConfiguration); + const user = await ldap.find("999248"); + expect(user.firstName).toBe("Xavier"); + expect(user.lastName).toBe("GUIMARD"); + expect(user.email).toBe("xguimard@avocat.fr"); + expect(user.uid).toBe("999248"); + }); + + test('ldap returns nothing', async () => { + const ldap = new LdapUser({ + url: "ldap://auth.poc-mail-avocat.fr:389", + baseDn: "ou=users,dc=example,dc=com", + } as LdapConfiguration); + const user = await ldap.find("NOTHING_HERE"); + expect(user.email).toBeUndefined(); + }); + +}); \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/test/twake.test.ts b/tdrive/backend/utils/nextcloud-migration/test/twake.test.ts new file mode 100644 index 00000000..6a34aae0 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/test/twake.test.ts @@ -0,0 +1,23 @@ +import { expect, test } from '@jest/globals'; +import { TwakeDriveClient } from '../src/twake_client'; + +//integration debug test +describe.skip('twake client module', () => { + + test('twake creates new user', async () => { + //given + const subj = new TwakeDriveClient({ + url: "https://tdrive.qa.lin-saas.com/", + credentials: { + appId: "tdrive_onlyoffice", + secret: "QWERTY" + } + }) + + //when + const response = await subj.createUser({firstName: "Test", lastName: "User", email: "test@example.com", uid: "test"}); + expect(response).toBeDefined(); + console.log(response); + }); + +}); \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/tsconfig.json b/tdrive/backend/utils/nextcloud-migration/tsconfig.json new file mode 100644 index 00000000..7c205f45 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/tsconfig.json @@ -0,0 +1,18 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "esnext", + "moduleResolution": "node", + "outDir": "dist", + "strict": true, + "esModuleInterop": true, + "useUnknownInCatchVariables": false, + "typeRoots": [ + "./typings", + "./node_modules/@types/" + ], + "noImplicitAny": false + }, + "include": ["src"] +} + \ No newline at end of file diff --git a/tdrive/backend/utils/nextcloud-migration/typings/index.d.ts b/tdrive/backend/utils/nextcloud-migration/typings/index.d.ts new file mode 100644 index 00000000..3d8142c9 --- /dev/null +++ b/tdrive/backend/utils/nextcloud-migration/typings/index.d.ts @@ -0,0 +1 @@ +declare module 'ldif'; \ No newline at end of file diff --git a/tdrive/docker-compose.dev.tests.opensearch.yml b/tdrive/docker-compose.dev.tests.opensearch.yml new file mode 100644 index 00000000..c08e4496 --- /dev/null +++ b/tdrive/docker-compose.dev.tests.opensearch.yml @@ -0,0 +1,60 @@ +version: "3.4" + +# docker-compose -f docker-compose.dev.tests.mongo.yml stop mongo; rm -R docker-data/mongo/; docker-compose -f docker-compose.dev.tests.mongo.yml run -e SEARCH_DRIVER=mongodb -e DB_DRIVER=mongodb node npm run test:e2e + +services: + + opensearch-node1: + image: opensearchproject/opensearch:latest # Specifying the latest available image - modify if you want a specific version + container_name: opensearch-node1 + environment: + - cluster.name=opensearch-cluster # Name the cluster + - node.name=opensearch-node1 # Name the node that will run in this container + - discovery.seed_hosts=opensearch-node1 # Nodes to look for when discovering the cluster + - cluster.initial_cluster_manager_nodes=opensearch-node1 # Nodes eligible to serve as cluster manager + - bootstrap.memory_lock=true # Disable JVM heap memory swapping + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # Set min and max JVM heap sizes to at least 50% of system RAM + ulimits: + memlock: + soft: -1 # Set memlock to unlimited (no soft or hard limit) + hard: -1 + nofile: + soft: 65536 # Maximum number of open files for the opensearch user - set to at least 65536 + hard: 65536 +# volumes: +# - opensearch-data1:/usr/share/opensearch/data # Creates volume called opensearch-data1 and mounts it to the container + ports: + - 9200:9200 # REST API + - 9600:9600 # Performance Analyzer + # networks: + # - opensearch-net # All of the containers will join the same Docker bridge network + + mongo: + container_name: mongo + image: mongo + volumes: + - ./docker-data/mongo:/data/db + ports: + - 27017:27017 + + node: + image: tdrive/tdrive-node:test + container_name: node-test + build: + context: . + dockerfile: docker/tdrive-node/Dockerfile + target: test + environment: + - NODE_ENV=test + - DB_DRIVER + #- PUBSUB_URLS=amqp://guest:guest@rabbitmq:5672 + - SEARCH_ES_ENDPOINT=http://elasticsearch:9200 + - SEARCH_ES_FLUSHINTERVAL=1 + volumes: + - ./backend/node:/usr/src/app + - ./docker-data/documents/:/storage/ + depends_on: + - mongo + - opensearch-node1 + links: + - mongo diff --git a/tdrive/docker/tdrive-nextcloud-migration/Dockerfile b/tdrive/docker/tdrive-nextcloud-migration/Dockerfile new file mode 100644 index 00000000..908b90f2 --- /dev/null +++ b/tdrive/docker/tdrive-nextcloud-migration/Dockerfile @@ -0,0 +1,27 @@ +# Use an official Node.js runtime as the base image +FROM node:bullseye + +RUN echo "deb http://ppa.launchpad.net/nextcloud-devs/client/ubuntu zesty main" >> /etc/apt/sources.list.d/nextcloud-client.list +RUN echo "deb-src http://ppa.launchpad.net/nextcloud-devs/client/ubuntu zesty main" >> /etc/apt/sources.list.d/nextcloud-client.list + +RUN apt install dirmngr +RUN apt-key adv --recv-key --keyserver keyserver.ubuntu.com AD3DD469 + +RUN apt update + +RUN apt install --assume-yes nextcloud-desktop-cmd + +RUN apt install ldap-utils + +# Set the working directory inside the container +WORKDIR /usr/src/app + +# Copy app +COPY backend/utils/nextcloud-migration/src/** ./src/ +COPY backend/utils/nextcloud-migration/.nvmrc ./ +COPY backend/utils/nextcloud-migration/*.json ./ + +RUN npm i && npm run build + +# Run the Node.js application +CMD ["npm", "run", "start"]