diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e16787ce..ac8a99fd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -68,12 +68,6 @@ jobs: docker-compose -f docker-compose.dev.tests.opensearch.yml logs docker-compose -f docker-compose.dev.tests.opensearch.yml run -e NODE_OPTIONS=--unhandled-rejections=warn -e SEARCH_DRIVER=opensearch -e DB_DRIVER=postgres -e PUBSUB_TYPE=local node npm run test:all docker-compose -f docker-compose.dev.tests.opensearch.yml down - - name: e2e-cassandra-test - run: | - cd tdrive - docker-compose -f docker-compose.tests.yml up -d scylladb elasticsearch rabbitmq - sleep 60 - docker-compose -f docker-compose.tests.yml run -e NODE_OPTIONS=--unhandled-rejections=warn -e SEARCH_DRIVER=elasticsearch -e DB_DRIVER=cassandra node npm run test:all - name: coverage uses: adRise/jest-cov-reporter@e13d7638888cad7788298f4022b203b31a10c7c8 with: diff --git a/tdrive/backend/node/config/custom-environment-variables.json b/tdrive/backend/node/config/custom-environment-variables.json index 0c5afee3..66c7ec6f 100644 --- a/tdrive/backend/node/config/custom-environment-variables.json +++ b/tdrive/backend/node/config/custom-environment-variables.json @@ -43,11 +43,6 @@ "uri": "DB_MONGO_URI", "database": "DB_MONGO_DATABASE" }, - "cassandra": { - "contactPoints": "DB_CASSANDRA_URI", - "localDataCenter": "DB_CASSANDRA_LOCALDATACENTER", - "keyspace": "DB_CASSANDRA_KEYSPACE" - }, "postgres": { "database": "DB_POSTGRES_DBNAME", "host": "DB_POSTGRES_HOST", diff --git a/tdrive/backend/node/config/default.json b/tdrive/backend/node/config/default.json index 10c85580..3820a1f5 100644 --- a/tdrive/backend/node/config/default.json +++ b/tdrive/backend/node/config/default.json @@ -63,20 +63,12 @@ }, "database": { "secret": "", - "type": "cassandra", + "type": "mongodb", "encryption": "aes-256-cbc", "mongodb": { "uri": "mongodb://mongo:27017", "database": "tdrive" }, - "cassandra": { - "contactPoints": ["scylladb:9042"], - "localDataCenter": "datacenter1", - "keyspace": "tdrive", - "wait": false, - "retries": 10, - "delay": 200 - } }, "message-queue": { "// possible 'type' values are": "'amqp' or 'local'", diff --git a/tdrive/backend/node/config/test.json b/tdrive/backend/node/config/test.json index d6462f97..4b44351c 100644 --- a/tdrive/backend/node/config/test.json +++ b/tdrive/backend/node/config/test.json @@ -8,15 +8,6 @@ "secret": "", "mongodb": { "uri": "mongodb://mongo:27017" - }, - "cassandra": { - "contactPoints": ["scylladb:9042"], - "wait": true, - "retries": 50, - "delay": 500, - "queryOptions": { - "consistency": 1 - } } }, "message-queue": { diff --git a/tdrive/backend/node/package.json b/tdrive/backend/node/package.json index 44bd8bca..8c7dac2d 100644 --- a/tdrive/backend/node/package.json +++ b/tdrive/backend/node/package.json @@ -129,7 +129,6 @@ "archiver": "^5.3.1", "axios": "^1.6.8", "bcrypt": "^5.0.1", - "cassandra-driver": "^4.6.0", "class-transformer": "^0.3.1", "cli-table": "^0.3.6", "config": "^3.3.2", diff --git a/tdrive/backend/node/src/cli/lib/print-config-summary.ts b/tdrive/backend/node/src/cli/lib/print-config-summary.ts index 02045b69..d382540b 100644 --- a/tdrive/backend/node/src/cli/lib/print-config-summary.ts +++ b/tdrive/backend/node/src/cli/lib/print-config-summary.ts @@ -29,14 +29,6 @@ export default function printConfigSummary(useIcons: boolean = !!process.env.HAV "user", user, ], - cassandra: ({ contactPoints, localDataCenter, keyspace }) => [ - "host", - contactPoints, - "datacenter", - localDataCenter, - "keyspace", - keyspace, - ], S3: ({ endPoint, port, accessKey, bucket }) => [ "host", [endPoint, port], @@ -62,7 +54,6 @@ export default function printConfigSummary(useIcons: boolean = !!process.env.HAV elasticsearch: "ES", opensearch: "OS", mongodb: "mongo", - cassandra: "Cass.", }; if (useIcons) { const makeLyingLengthIcon = str => ({ length: 1, toString: _ => str } as string); diff --git a/tdrive/backend/node/src/cli/utils/copy-cluster/.gitignore b/tdrive/backend/node/src/cli/utils/copy-cluster/.gitignore deleted file mode 100644 index 97008e5b..00000000 --- a/tdrive/backend/node/src/cli/utils/copy-cluster/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -node_modules -yarn.lock \ No newline at end of file diff --git a/tdrive/backend/node/src/cli/utils/copy-cluster/README.md b/tdrive/backend/node/src/cli/utils/copy-cluster/README.md deleted file mode 100644 index 02fe7cc4..00000000 --- a/tdrive/backend/node/src/cli/utils/copy-cluster/README.md +++ /dev/null @@ -1,4 +0,0 @@ -Edit the file to your needs - -yarn install -node index.js diff --git a/tdrive/backend/node/src/cli/utils/copy-cluster/index.js b/tdrive/backend/node/src/cli/utils/copy-cluster/index.js deleted file mode 100644 index 1c4d5716..00000000 --- a/tdrive/backend/node/src/cli/utils/copy-cluster/index.js +++ /dev/null @@ -1,214 +0,0 @@ -/* eslint-disable */ -var cassandra = require("cassandra-driver"); - -var fromAuthProvider = new cassandra.auth.PlainTextAuthProvider("", ""); -var fromContactPoints = [""]; -var fromKeyspace = "tdrive"; - -var toAuthProvider = new cassandra.auth.PlainTextAuthProvider("", ""); -var toContactPoints = [""]; -var toKeyspace = "tdrive"; - -var forceUpdateAll = false; -var ignoreTables = ["notification"]; -var countersTables = ["statistics", "stats_counter", "channel_counters", "scheduled_queue_counter"]; - -var specialConversions = { - "tdrive.group_user.date_added": value => { - return new Date(value).getTime(); - }, -}; - -// -- start process - -var fromClient = new cassandra.Client({ - localDataCenter: "datacenter1", - contactPoints: fromContactPoints, - authProvider: fromAuthProvider, - keyspace: fromKeyspace, -}); - -var toClient = new cassandra.Client({ - localDataCenter: "datacenter1", - contactPoints: toContactPoints, - authProvider: toAuthProvider, - keyspace: toKeyspace, - queryOptions: { - consistency: cassandra.types.consistencies.quorum, - }, -}); - -// -- Get all tables and copy schema - -async function client(origin, query, parameters, options) { - return await new Promise((resolve, reject) => { - origin.execute(query, [], {}, function (err, result) { - if (err) { - reject({ err, result }); - } else { - resolve(result); - } - }); - }); -} - -(async () => { - const result = await client( - fromClient, - "SELECT table_name from system_schema.tables WHERE keyspace_name = '" + fromKeyspace + "'", - [], - {}, - ); - for (row of result.rows) { - try { - const fromTable = fromKeyspace + "." + row.table_name; - const toTable = toKeyspace + "." + row.table_name; - - if (ignoreTables.includes(row.table_name)) { - console.log(fromTable.padEnd(50) + " | " + "ignored".padEnd(20) + " | ⏺"); - continue; - } - - let fromCount = 0; - - const destColumns = ( - await client( - toClient, - "SELECT column_name from system_schema.columns where keyspace_name = '" + - toKeyspace + - "' and table_name = '" + - row.table_name + - "'", - [], - {}, - ) - ).rows.map(r => r.column_name); - - try { - const fromResult = await client( - fromClient, - "SELECT count(*) from " + fromTable + "", - [], - {}, - ); - fromCount = parseInt(fromResult.rows[0].count); - } catch (err) { - fromCount = NaN; - } - - if (destColumns.length === 0) { - if (fromCount > 0) - console.log( - fromTable.padEnd(50) + " | " + ("not_in_dest" + "/" + fromCount).padEnd(20) + " | ⏺", - ); - continue; - } - - try { - const toResult = await client(toClient, "SELECT count(*) from " + toTable + "", [], {}); - const toCount = parseInt(toResult.rows[0].count); - - console.log( - fromTable.padEnd(50) + - " | " + - (toCount + "/" + fromCount).padEnd(20) + - " | " + - (toCount >= fromCount ? "✅" : "❌"), - ); - - if (row.table_name.indexOf("counter") >= 0 || countersTables.includes(row.table_name)) { - console.log( - fromTable.padEnd(50) + " | " + ("counter_table" + "/" + fromCount).padEnd(20) + " | 🧮", - ); - if (fromCount > toCount || !fromCount || forceUpdateAll) { - await new Promise(r => { - fromClient.eachRow( - "SELECT JSON * from " + fromTable, - [], - { prepare: true, fetchSize: 1000 }, - async function (n, row) { - // - //TODO handle counters (it is special !) - }, - async function (err, result) { - if (result && result.nextPage) { - await new Promise(r => setTimeout(r, 1000)); - result.nextPage(); - } else { - r(); - } - }, - ); - }); - } - - continue; - } - - if (fromCount > toCount || !fromCount || forceUpdateAll) { - await new Promise(r => { - fromClient.eachRow( - "SELECT JSON * from " + fromTable, - [], - { prepare: true, fetchSize: 1000 }, - async function (n, row) { - try { - const json = JSON.parse(row["[json]"]); - - //The from table can have additional depreciated fields, we need to remove them - const filteredJson = {}; - for (const col of destColumns) { - if ( - typeof json[col] == "string" && - (json[col] || "").match( - /[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+/, - ) - ) { - json[col] = json[col].split(".")[0]; - } - - if (specialConversions[fromTable + "." + col]) { - json[col] = specialConversions[fromTable + "." + col](json[col]); - } - - if (json[col] !== undefined) filteredJson[col] = json[col]; - } - - await client( - toClient, - "INSERT INTO " + - toTable + - " JSON '" + - JSON.stringify(filteredJson).replace(/'/g, "'$&") + - "'", - [], - {}, - ); - } catch (err) { - console.log(err); - } - }, - async function (err, result) { - if (result && result.nextPage) { - await new Promise(r => setTimeout(r, 1000)); - result.nextPage(); - } else { - r(); - } - }, - ); - }); - } - } catch (err) { - console.log( - fromTable.padEnd(50) + " | " + ("error" + "/" + fromCount).padEnd(20) + " | ❌", - ); - } - } catch (err) { - console.log(err); - continue; - } - - //TODO copy content - } -})(); diff --git a/tdrive/backend/node/src/cli/utils/copy-cluster/package.json b/tdrive/backend/node/src/cli/utils/copy-cluster/package.json deleted file mode 100644 index 4a306878..00000000 --- a/tdrive/backend/node/src/cli/utils/copy-cluster/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "copy-cluster", - "version": "1.0.0", - "main": "index.js", - "license": "MIT", - "dependencies": { - "cassandra-driver": "^4.6.3" - } -} diff --git a/tdrive/backend/node/src/cli/utils/copy-table/README.md b/tdrive/backend/node/src/cli/utils/copy-table/README.md deleted file mode 100644 index 02fe7cc4..00000000 --- a/tdrive/backend/node/src/cli/utils/copy-table/README.md +++ /dev/null @@ -1,4 +0,0 @@ -Edit the file to your needs - -yarn install -node index.js diff --git a/tdrive/backend/node/src/cli/utils/copy-table/index.js b/tdrive/backend/node/src/cli/utils/copy-table/index.js deleted file mode 100644 index 50ed0db1..00000000 --- a/tdrive/backend/node/src/cli/utils/copy-table/index.js +++ /dev/null @@ -1,88 +0,0 @@ -/* eslint-disable */ -var cassandra = require("cassandra-driver"); - -var authProvider = new cassandra.auth.PlainTextAuthProvider("", ""); -var contactPoints = [""]; -var from = "table_a"; -var to = "table_b"; - -var client = new cassandra.Client({ - localDataCenter: "datacenter1", - contactPoints: contactPoints, - authProvider: authProvider, - keyspace: "tdrive", -}); - -//Copy object to other table -(async () => { - const query = "SELECT * FROM " + from; - const parameters = []; - - let count = 0; - let max = 0; - let min = new Date().getTime(); - - let rows = []; - - const options = { prepare: true, fetchSize: 1000 }; - - await new Promise(r => { - client.eachRow( - query, - parameters, - options, - function (n, row) { - min = Math.min(parseInt(row.created_at.toString()), min); - max = Math.max(parseInt(row.created_at.toString()), max); - count++; - if (count % 100 == 0) { - console.log(count); - } - rows.push(row); - }, - function (err, result) { - if (result && result.nextPage) { - result.nextPage(); - } else { - r(); - } - }, - ); - }); - - console.log("Downloaded ", count, " rows of ", from, ". Starting copy..."); - - let copyCount = 0; - - for (let i = 0; i < count; i++) { - copyCount++; - if (copyCount % 100 == 0) { - console.log(copyCount, "of", count); - } - - const row = rows[i]; - - await new Promise(r => { - let query = - "UPDATE " + - to + - " SET answers=?, created_at=?, created_by=?, last_activity=?, participants=?, updated_at=? WHERE id=?"; - client.execute( - query, - [ - row.answers, - row.created_at, - row.created_by, - row.last_activity, - row.participants, - row.updated_at, - row.id, - ], - () => { - r(); - }, - ); - }); - } - console.log("Ended with ", count, "threads"); -})(); diff --git a/tdrive/backend/node/src/cli/utils/copy-table/package.json b/tdrive/backend/node/src/cli/utils/copy-table/package.json deleted file mode 100644 index f3c1b49b..00000000 --- a/tdrive/backend/node/src/cli/utils/copy-table/package.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "copy-table", - "version": "1.0.0", - "main": "index.js", - "license": "MIT" -} diff --git a/tdrive/backend/node/src/core/platform/services/database/services/connector-factory.ts b/tdrive/backend/node/src/core/platform/services/database/services/connector-factory.ts index 933f0e12..cdba0d00 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/connector-factory.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/connector-factory.ts @@ -1,17 +1,11 @@ import { DatabaseType } from "."; import { ConnectionOptions, Connector } from "./orm/connectors"; -import { - CassandraConnectionOptions, - CassandraConnector, -} from "./orm/connectors/cassandra/cassandra"; import { MongoConnectionOptions, MongoConnector } from "./orm/connectors/mongodb/mongodb"; import { PostgresConnectionOptions, PostgresConnector } from "./orm/connectors/postgres/postgres"; export class ConnectorFactory { public create(type: DatabaseType, options: ConnectionOptions, secret: string): Connector { switch (type) { - case "cassandra": - return new CassandraConnector(type, options as CassandraConnectionOptions, secret); case "mongodb": return new MongoConnector(type, options as MongoConnectionOptions, secret); case "postgres": diff --git a/tdrive/backend/node/src/core/platform/services/database/services/index.ts b/tdrive/backend/node/src/core/platform/services/database/services/index.ts index 3bb13f8b..d389d621 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/index.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/index.ts @@ -3,7 +3,6 @@ import { ConnectorFactory } from "./connector-factory"; import { Connector } from "./orm/connectors"; import Manager from "./orm/manager"; import Repository from "./orm/repository/repository"; -import { CassandraConnectionOptions } from "./orm/connectors/cassandra/cassandra"; import { MongoConnectionOptions } from "./orm/connectors/mongodb/mongodb"; import { EntityTarget } from "./orm/types"; import { RepositoryManager } from "./orm/repository/manager"; @@ -48,9 +47,6 @@ export default class DatabaseService implements DatabaseServiceAPI { } } -export declare type ConnectionOptions = - | MongoConnectionOptions - | CassandraConnectionOptions - | PostgresConnectionOptions; +export declare type ConnectionOptions = MongoConnectionOptions | PostgresConnectionOptions; -export declare type DatabaseType = "mongodb" | "cassandra" | "postgres"; +export declare type DatabaseType = "mongodb" | "postgres"; diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/cassandra.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/cassandra.ts deleted file mode 100644 index 956f4112..00000000 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/cassandra.ts +++ /dev/null @@ -1,565 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types */ -import cassandra, { types } from "cassandra-driver"; -import { md5 } from "../../../../../../../crypto"; -import { defer, Subject, throwError, timer } from "rxjs"; -import { concat, delayWhen, retryWhen, take, tap } from "rxjs/operators"; -import { UpsertOptions } from ".."; -import { logger } from "../../../../../../framework"; -import { getEntityDefinition, unwrapPrimarykey } from "../../utils"; -import { EntityDefinition, ColumnDefinition, ObjectType } from "../../types"; -import { AbstractConnector } from "../abstract-connector"; -import { - transformValueToDbString, - cassandraType, - transformValueFromDbString, -} from "./typeTransforms"; -import { FindOptions } from "../../repository/repository"; -import { ListResult, Pagination } from "../../../../../../framework/api/crud-service"; -import { Paginable } from "../../../../../../framework/api/crud-service"; -import { buildSelectQuery } from "./query-builder"; - -export { CassandraPagination } from "./pagination"; - -export interface CassandraConnectionOptions { - contactPoints: string[]; - localDataCenter: string; - username: string; - password: string; - keyspace: string; - - /** - * Consistency level - */ - queryOptions: { consistency: number }; - - /** - * Wait for keyspace and tables to be created at init - */ - wait: boolean; - - /** - * When wait = true, retry to get the resources N times where N = retries - */ - retries?: number; - - /** - * Delay in ms between the retries. The delay is growing each time a retry fails like delay = retryCount * delay - */ - delay?: number; -} - -export class CassandraConnector extends AbstractConnector { - private client: cassandra.Client; - private keyspaceExists = false; - - getClient(): cassandra.Client { - return this.client; - } - - async init(): Promise { - if (!this.client) { - await this.connect(); - } - - try { - await this.createKeyspace(); - } catch (err) { - logger.warn("services.database.orm.cassandra - Keyspace can not be created", err); - } - - if (this.options.wait) { - await this.waitForKeyspace(this.options.delay, this.options.retries); - } - - return this; - } - - createKeyspace(): Promise { - const query = `CREATE KEYSPACE IF NOT EXISTS ${this.options.keyspace} WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1': '2'} AND durable_writes = true;`; - logger.info(query); - return this.client.execute(query); - } - - async isKeyspaceCreated(): Promise { - let result; - - if (this.keyspaceExists) { - return true; - } - - try { - result = await this.client.execute( - `SELECT * FROM system_schema.keyspaces where keyspace_name = '${this.options.keyspace}'`, - ); - - if (!result) { - throw new Error("No result for keyspace query"); - } - } catch (err) { - throw new Error("Keyspace query error"); - } - - if (result) { - await new Promise(resolve => setTimeout(resolve, 2000)); - this.keyspaceExists = true; - logger.info(`Keyspace '${this.options.keyspace}' found.`); - } - - return result ? Promise.resolve(true) : Promise.reject(new Error("Keyspace not found")); - } - - waitForKeyspace(delay: number = 500, retries: number = 10): Promise { - const subject = new Subject(); - const obs$ = defer(() => this.isKeyspaceCreated()); - - obs$ - .pipe( - retryWhen(errors => - errors.pipe( - delayWhen((_, i) => timer(i * delay)), - //delay(1000), if we want fixed delay - tap(() => logger.debug("services.database.orm.cassandra - Retrying...")), - take(retries), - concat(throwError("Maximum number of retries reached")), - ), - ), - ) - .subscribe( - () => logger.debug("services.database.orm.cassandra - Keyspace has been found"), - err => { - logger.error( - { err }, - "services.database.orm.cassandra - Error while getting keyspace information", - ); - subject.error(new Error("Can not find keyspace information")); - }, - () => subject.complete(), - ); - - return subject.toPromise(); - } - - async drop(): Promise { - logger.info("Drop database is not implemented for security reasons."); - return this; - } - - async connect(): Promise { - if (this.client) { - return this; - } - - // Environment variable format is comma separated string - const contactPoints = - typeof this.options.contactPoints === "string" - ? (this.options.contactPoints as string).split(",") - : this.options.contactPoints; - - const cassandraOptions: cassandra.DseClientOptions = { - contactPoints: contactPoints, - localDataCenter: this.options.localDataCenter, - queryOptions: {}, - }; - - if (this.options.username && this.options.password) { - cassandraOptions.authProvider = new cassandra.auth.PlainTextAuthProvider( - this.options.username, - this.options.password, - ); - } - - //Set default consistency level to quorum - cassandraOptions.queryOptions.consistency = - this.options?.queryOptions?.consistency || types.consistencies.quorum; - - this.client = new cassandra.Client(cassandraOptions); - await this.client.connect(); - - return this; - } - - async disconnect(): Promise { - if (this.client) await this.client.shutdown(); - this.client = null; - return this; - } - - async getTableDefinition(name: string): Promise { - let result: string[]; - - try { - const query = `SELECT column_name from system_schema.columns WHERE keyspace_name = '${this.options.keyspace}' AND table_name='${name}';`; - const dbResult = await this.client.execute(query); - result = dbResult.rows.map(row => row.values()[0]); - } catch (err) { - throw "Table query error"; - } - - return result ? Promise.resolve(result || []) : Promise.resolve([]); - } - - async createTable( - entity: EntityDefinition, - columns: { [name: string]: ColumnDefinition }, - ): Promise { - await this.waitForKeyspace(this.options.delay, this.options.retries); - - let result = true; - - // --- Generate column and key definition --- // - const primaryKey = entity.options.primaryKey || []; - - if (primaryKey.length === 0) { - logger.error( - `services.database.orm.cassandra - Primary key was not defined for table ${entity.name}`, - ); - return false; - } - - const partitionKeyPart = primaryKey.shift(); - const partitionKey: string[] = - typeof partitionKeyPart === "string" ? [partitionKeyPart] : partitionKeyPart; - const clusteringKeys: string[] = primaryKey as string[]; - if ([...partitionKey, ...clusteringKeys].some(key => columns[key] === undefined)) { - logger.error( - `services.database.orm.cassandra - One primary key item doesn't exists in entity columns for table ${entity.name}`, - ); - return false; - } - - const clusteringOrderBy = clusteringKeys.map(key => { - const direction: "ASC" | "DESC" = columns[key].options.order || "ASC"; - return `${key} ${direction}`; - }); - const clusteringOrderByString = - clusteringOrderBy.length > 0 - ? `WITH CLUSTERING ORDER BY (${clusteringOrderBy.join(", ")})` - : ""; - - const allKeys = [`(${partitionKey.join(", ")})`, ...clusteringKeys]; - const primaryKeyString = `(${allKeys.join(", ")})`; - - const columnsString = Object.keys(columns) - .map(colName => { - const definition = columns[colName]; - return `${colName} ${cassandraType[definition.type]}`; - }) - .join(",\n"); - - // --- Generate final create table query --- // - let query = ` - CREATE TABLE IF NOT EXISTS ${this.options.keyspace}.${entity.name} - ( - ${columnsString}, - PRIMARY KEY ${primaryKeyString} - ) ${clusteringOrderByString};`; - - // --- Alter table if not up to date --- // - const existingColumns = await this.getTableDefinition(entity.name); - if (existingColumns.length > 0) { - logger.debug( - `services.database.orm.cassandra - Existing columns for table ${entity.name}, generating altertable queries`, - ); - const alterQueryColumns = Object.keys(columns) - .filter(colName => existingColumns.indexOf(colName) < 0) - .map(colName => `${colName} ${cassandraType[columns[colName].type]}`); - query = `ALTER TABLE ${this.options.keyspace}.${entity.name} ADD (${alterQueryColumns.join( - ", ", - )})`; - - if (alterQueryColumns.length === 0) { - return true; - } - } - - // --- Write table --- // - try { - logger.debug(`service.database.orm.createTable - Creating table ${entity.name} : ${query}`); - await this.client.execute(query); - } catch (err) { - logger.warn( - { err }, - `service.database.orm.createTable - creation error for table ${entity.name} : ${err.message}`, - ); - result = false; - } - - // --- Create indexes --- // - if (entity.options.globalIndexes) { - for (const globalIndex of entity.options.globalIndexes) { - const indexName = globalIndex.join("_"); - const indexDbName = `index_${md5(indexName)}`; - - const query = `CREATE INDEX IF NOT EXISTS ${indexDbName} ON ${this.options.keyspace}."${ - entity.name - }" (${ - globalIndex.length === 1 ? globalIndex[0] : `(${globalIndex[0]}), ${globalIndex[1]}` - })`; - - try { - logger.debug( - `service.database.orm.createTable - Creating index ${indexName} (${indexDbName}) : ${query}`, - ); - await this.client.execute(query); - } catch (err) { - logger.warn( - { err }, - `service.database.orm.createTable - creation error for index ${indexName} (${indexDbName}) : ${err.message}`, - ); - result = false; - } - } - } - - return result; - } - - async upsert(entities: any[], _options: UpsertOptions = {}): Promise { - return new Promise(resolve => { - const promises: Promise[] = []; - - entities.forEach(entity => { - const { columnsDefinition, entityDefinition } = getEntityDefinition(entity); - const primaryKey = unwrapPrimarykey(entityDefinition); - - //Set updated content - const set = Object.keys(columnsDefinition) - .filter(key => primaryKey.indexOf(key) === -1) - .filter(key => entity[columnsDefinition[key].nodename] !== undefined) - .map(key => [ - `${key}`, - `${transformValueToDbString( - entity[columnsDefinition[key].nodename], - columnsDefinition[key].type, - { - columns: columnsDefinition[key].options, - secret: this.secret, - column: { key }, - }, - )}`, - ]); - //Set primary key - const where = primaryKey.map(key => [ - `${key}`, - `${transformValueToDbString( - entity[columnsDefinition[key].nodename], - columnsDefinition[key].type, - { - columns: columnsDefinition[key].options, - secret: this.secret, - disableSalts: true, - column: { key }, - }, - )}`, - ]); - - // Add time-to-live options - let ttlOptions = ""; - if (entityDefinition.options.ttl && entityDefinition.options.ttl > 0) { - ttlOptions = `USING TTL ${entityDefinition.options.ttl}`; - } - - // Insert and update are equivalent for most of Cassandra - // Update is prefered because the only solution for counters - let query = `UPDATE ${this.options.keyspace}.${ - entityDefinition.name - } ${ttlOptions} SET ${set.map(e => `${e[0]} = ${e[1]}`).join(", ")} WHERE ${where - .map(e => `${e[0]} = ${e[1]}`) - .join(" AND ")}`; - - // If no "set" part, we cannot do an update, so insert - if (set.length === 0) { - query = `INSERT INTO ${this.options.keyspace}.${ - entityDefinition.name - } ${ttlOptions} (${where.map(e => e[0]).join(", ")}) VALUES (${where - .map(e => e[1]) - .join(", ")})`; - } - - logger.debug(`service.database.orm.upsert - Query: "${query}"`); - - promises.push( - new Promise(async resolve => { - try { - await this.getClient().execute(query); - resolve(true); - } catch (err) { - logger.error( - { err }, - `services.database.orm.cassandra - Error with CQL query: ${query}`, - ); - resolve(false); - } - }), - ); - }); - - Promise.all(promises).then(resolve); - }); - } - - async remove(entities: any[]): Promise { - return new Promise(resolve => { - const promises: Promise[] = []; - - entities.forEach(entity => { - const { columnsDefinition, entityDefinition } = getEntityDefinition(entity); - const primaryKey = unwrapPrimarykey(entityDefinition); - - //Set primary key - const where = primaryKey.map( - key => - `${key} = ${transformValueToDbString( - entity[columnsDefinition[key].nodename], - columnsDefinition[key].type, - { - columns: columnsDefinition[key].options, - secret: this.secret, - disableSalts: true, - column: { key }, - }, - )}`, - ); - - const query = `DELETE FROM ${this.options.keyspace}.${ - entityDefinition.name - } WHERE ${where.join(" AND ")}`; - logger.debug(`services.database.orm.cassandra.remove - Query: ${query}`); - - promises.push( - new Promise(async resolve => { - try { - await this.getClient().execute(query); - resolve(true); - } catch (err) { - logger.error( - { err }, - `services.database.orm.cassandra.remove - Error with CQL query: ${query}`, - ); - resolve(false); - } - }), - ); - }); - - Promise.all(promises).then(resolve); - }); - } - - async find( - entityType: Table, - filters: any, - options: FindOptions = {}, - ): Promise> { - const instance = new (entityType as any)(); - const { columnsDefinition, entityDefinition } = getEntityDefinition(instance); - - const query = buildSelectQuery
( - entityType as unknown as ObjectType
, - filters, - options, - { - keyspace: this.options.keyspace, - secret: this.secret, - }, - ); - - logger.debug(`services.database.orm.cassandra.find - Query: ${query}`); - - const results = await this.getClient().execute(query, [], { - fetchSize: parseInt(options.pagination.limitStr) || 100, - pageState: options.pagination.page_token || undefined, - prepare: false, - }); - - const entities: Table[] = []; - results.rows.forEach(row => { - const entity = new (entityType as any)(); - Object.keys(row).forEach(key => { - if (columnsDefinition[key]) { - entity[columnsDefinition[key].nodename] = transformValueFromDbString( - row[key], - columnsDefinition[key].type, - { column: { key: key, ...columnsDefinition[key].options }, secret: this.secret }, - ); - } - }); - entities.push(entity); - }); - - const nextPage: Paginable = new Pagination( - results.pageState, - options.pagination.limitStr || "100", - ); - logger.debug( - `services.database.orm.cassandra.find - Query Result (items=${entities.length}): ${query}`, - ); - - return new ListResult
(entityDefinition.type, entities, nextPage); - } -} - -export function waitForTable( - client: cassandra.Client, - keyspace: string, - table: string, - retries: number = 10, - delay: number = 500, -): Promise { - const subject = new Subject(); - const obs$ = defer(() => checkForTable(client, keyspace, table)); - - obs$ - .pipe( - retryWhen(errors => - errors.pipe( - delayWhen((_, i) => timer(i * delay)), - //delay(1000), - tap(() => - logger.debug("services.database.orm.cassandra - Retrying to get table metadata..."), - ), - take(retries), - concat(throwError("Maximum number of retries reached")), - ), - ), - ) - .subscribe( - () => logger.debug(`services.database.orm.cassandra - Table ${table} has been found`), - err => { - logger.debug({ err }, `services.database.orm.cassandra - Table ${table} error`); - subject.error(new Error("Can not find table")); - }, - () => subject.complete(), - ); - - return subject.toPromise(); -} - -async function checkForTable( - client: cassandra.Client, - keyspace: string, - table: string, -): Promise { - try { - const result: cassandra.types.ResultSet = await client.execute( - `SELECT * FROM system_schema.tables WHERE keyspace_name='${keyspace}' AND table_name='${table}'`, - ); - - const tableMetadata = result.rows[0]; - - logger.debug( - "services.database.orm.cassandra.checkForTable - Table metadata %o", - tableMetadata, - ); - - if (!tableMetadata) { - throw new Error("Can not find table metadata"); - } - } catch (err) { - logger.error( - { err }, - "services.database.orm.cassandra.checkForTable - Error while getting table metadata", - ); - throw new Error("Error while getting table metadata"); - } -} diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/pagination.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/pagination.ts deleted file mode 100644 index 5bce1de9..00000000 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/pagination.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Paginable } from "../../../../../../framework/api/crud-service"; -import { Pagination } from "../../../../../../framework/api/crud-service"; - -export class CassandraPagination extends Pagination { - limit = 100; - - private constructor(readonly page_token: string, readonly limitStr = "100") { - super(page_token, limitStr); - this.limit = Number.parseInt(limitStr, 10); - } - - static from(pagination: Paginable): CassandraPagination { - return new CassandraPagination(pagination.page_token, pagination.limitStr); - } - - static next(current: Pagination, pageState: string): Paginable { - return new Pagination(pageState, current.limitStr); - } -} diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/query-builder.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/query-builder.ts deleted file mode 100644 index 5b8d28b3..00000000 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/query-builder.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { FindOptions } from "../../repository/repository"; -import { ObjectType } from "../../types"; -import { getEntityDefinition, secureOperators, filteringRequired } from "../../utils"; -import { transformValueToDbString } from "./typeTransforms"; - -export function buildSelectQuery( - entityType: ObjectType, - filters: Record, - findOptions: FindOptions, - options: { - secret?: string; - keyspace: string; - } = { - secret: "", - keyspace: "tdrive", - }, -): string { - const instance = new (entityType as any)(); - const { columnsDefinition, entityDefinition } = getEntityDefinition(instance); - let allowFiltering = false; - - const where = Object.keys(filters) - .map(key => { - let result: string; - const filter = filters[key]; - - if (filteringRequired(key)) { - allowFiltering = true; - } - - if (!filter) { - return; - } - - if (Array.isArray(filter)) { - if (!filter.length) { - return; - } - const inClause: string[] = filter.map( - value => - `${transformValueToDbString(value, columnsDefinition[key].type, { - columns: columnsDefinition[key].options, - secret: options.secret || "", - disableSalts: true, - })}`, - ); - - result = `${key} IN (${inClause.join(",")})`; - } else { - result = `${key} = ${transformValueToDbString(filter, columnsDefinition[key].type, { - columns: columnsDefinition[key].options, - secret: options.secret || "", - disableSalts: true, - })}`; - } - - return result; - }) - .filter(Boolean); - - secureOperators(transformValueToDbString, findOptions, entityType, options); - - const whereClause = `${[ - ...where, - ...(buildComparison(findOptions) || []), - ...(buildIn(findOptions) || []), - ...(buildLike(findOptions) || []), - ].join(" AND ")}`.trimEnd(); - - let orderByClause = ""; - if (findOptions.pagination?.reversed) { - orderByClause = `${entityDefinition.options.primaryKey - .slice(1) - .map( - (key: string) => - `${key} ${(columnsDefinition[key].options.order || "ASC") === "ASC" ? "DESC" : "ASC"}`, - ) - .join(", ")}`; - } - - const query = `SELECT * FROM ${options.keyspace}.${entityDefinition.name} ${ - whereClause.trim().length ? "WHERE " + whereClause : "" - } ${orderByClause.trim().length ? "ORDER BY " + orderByClause : ""}` - .trimEnd() - .concat(allowFiltering ? " ALLOW FILTERING;" : ";"); - - return query; -} - -export function buildComparison(options: FindOptions = {}): string[] { - let lessClause; - let lessEqualClause; - let greaterClause; - let greaterEqualClause; - - if (options.$lt) { - lessClause = options.$lt.map(element => `${element[0]} < ${element[1]}`); - } - - if (options.$lte) { - lessEqualClause = options.$lte.map(element => `${element[0]} <= ${element[1]}`); - } - - if (options.$gt) { - greaterClause = options.$gt.map(element => `${element[0]} > ${element[1]}`); - } - - if (options.$gte) { - greaterEqualClause = options.$gte.map(element => `${element[0]} >= ${element[1]}`); - } - - return [ - ...(lessClause || []), - ...(lessEqualClause || []), - ...(greaterClause || []), - ...(greaterEqualClause || []), - ]; -} - -export function buildIn(options: FindOptions = {}): string[] { - let inClauses: string[]; - if (options.$in) { - inClauses = options.$in.map(element => `${element[0]} IN (${element[1].join(",")})`); - } - - return inClauses || []; -} - -export function buildLike(options: FindOptions = {}): string[] { - let likeClauses: string[]; - if (options.$like) { - likeClauses = options.$like.map(element => `${element[0]} LIKE '%${element[1]}%`); - } - - return likeClauses || []; -} diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/typeTransforms.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/typeTransforms.ts deleted file mode 100644 index 2213ef8d..00000000 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/cassandra/typeTransforms.ts +++ /dev/null @@ -1,174 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -/* eslint-disable @typescript-eslint/no-unused-vars */ -/* eslint-disable @typescript-eslint/explicit-module-boundary-types */ -import { isBoolean, isInteger, isNull, isUndefined } from "lodash"; -import { ColumnOptions, ColumnType } from "../../types"; -import { decrypt, encrypt } from "../../../../../../../crypto"; -import { logger } from "../../../../../../../../core/platform/framework"; - -export const cassandraType = { - encoded_string: "TEXT", - encoded_json: "TEXT", - string: "TEXT", - json: "TEXT", - number: "BIGINT", - timeuuid: "TIMEUUID", - uuid: "UUID", - counter: "COUNTER", - blob: "BLOB", - boolean: "BOOLEAN", - - // backward compatibility - tdrive_boolean: "TINYINT", - tdrive_int: "INT", //Depreciated - tdrive_datetime: "TIMESTAMP", //Depreciated -}; - -type TransformOptions = { - secret?: any; - disableSalts?: boolean; - columns?: ColumnOptions; - column?: any; -}; - -export const transformValueToDbString = ( - v: any, - type: ColumnType, - options: TransformOptions = {}, -): string => { - if (type === "tdrive_datetime") { - if (isNaN(v) || isNull(v)) { - return "null"; - } - return `${v}`; - } - - if (type === "number" || type === "tdrive_int") { - if (isNull(v)) { - return "null"; - } - if (isNaN(v)) { - return "null"; - } - return `${parseInt(v)}`; - } - if (type === "uuid" || type === "timeuuid") { - if (isNull(v)) { - return "null"; - } - - v = (v || "").toString().replace(/[^a-zA-Z0-9-]/g, ""); - return `${v}`; - } - if (type === "boolean") { - //Security to avoid string with "false" in it - if (!isInteger(v) && !isBoolean(v) && !isNull(v) && !isUndefined(v)) { - throw new Error(`'${v}' is not a ${type}`); - } - return `${!!v}`; - } - if (type === "tdrive_boolean") { - if (!isBoolean(v)) { - throw new Error(`'${v}' is not a ${type}`); - } - return v ? "1" : "0"; - } - if (type === "encoded_string" || type === "encoded_json") { - if (type === "encoded_json") { - try { - v = JSON.stringify(v); - } catch (err) { - v = null; - } - } - const encrypted = encrypt(v, options.secret, { disableSalts: options.disableSalts }); - return `'${(encrypted.data || "").toString().replace(/'/gm, "''")}'`; - } - if (type === "blob") { - return "''"; //Not implemented yet - } - if (type === "string" || type === "json") { - if (type === "json" && v !== null) { - try { - v = JSON.stringify(v); - } catch (err) { - v = null; - } - } - return `'${(v || "").toString().replace(/'/gm, "''")}'`; - } - if (type === "counter") { - if (isNaN(v)) throw new Error("Counter value should be a number"); - return `${options.column.key} + ${v}`; - } - return `'${(v || "").toString().replace(/'/gm, "''")}'`; -}; - -export const transformValueFromDbString = ( - v: any, - type: string, - options: TransformOptions = {}, -): any => { - logger.trace(`Transform value %o of type ${type}`, v); - - if (type === "tdrive_datetime") { - return new Date(`${v}`).getTime(); - } - - if (v !== null && (type === "encoded_string" || type === "encoded_json")) { - let decryptedValue: any; - - if (typeof v === "string" && v.trim() === "") { - return v; - } - - try { - decryptedValue = decrypt(v, options.secret).data; - } catch (err) { - logger.debug(`Can not decrypt data (${err.message}) %o of type ${type}`, v); - - decryptedValue = v; - } - - if (type === "encoded_json") { - try { - decryptedValue = JSON.parse(decryptedValue); - } catch (err) { - logger.debug( - { err }, - `Can not parse JSON from decrypted data %o of type ${type}`, - decryptedValue, - ); - decryptedValue = null; - } - } - - return decryptedValue; - } - - if (type === "tdrive_boolean" || type === "boolean") { - return Boolean(v).valueOf(); - } - - if (type === "json") { - try { - return JSON.parse(v); - } catch (err) { - return null; - } - } - - if (type === "uuid" || type === "timeuuid") { - return v ? String(v).valueOf() : null; - } - - if (type === "number") { - return new Number(v).valueOf(); - } - - if (type === "counter") { - return new Number(v).valueOf(); - } - - return v; -}; diff --git a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts index 4be73d87..8b01704d 100644 --- a/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts +++ b/tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/index.ts @@ -1,6 +1,5 @@ import { Initializable } from "../../../../../framework"; import { DatabaseType } from "../.."; -import { CassandraConnectionOptions } from "./cassandra/cassandra"; import { MongoConnectionOptions } from "./mongodb/mongodb"; import { ColumnDefinition, EntityDefinition } from "../types"; import { FindOptions } from "../repository/repository"; @@ -8,7 +7,6 @@ import { ListResult } from "../../../../../framework/api/crud-service"; import { PostgresConnectionOptions } from "./postgres/postgres"; export * from "./mongodb/mongodb"; -export * from "./cassandra/cassandra"; export type UpsertOptions = { action?: "INSERT" | "UPDATE"; @@ -68,7 +66,4 @@ export interface Connector extends Initializable { ): Promise>; } -export declare type ConnectionOptions = - | MongoConnectionOptions - | CassandraConnectionOptions - | PostgresConnectionOptions; +export declare type ConnectionOptions = MongoConnectionOptions | PostgresConnectionOptions; diff --git a/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/cassandra/query-builder.test.ts b/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/cassandra/query-builder.test.ts deleted file mode 100644 index de2cda54..00000000 --- a/tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/cassandra/query-builder.test.ts +++ /dev/null @@ -1,166 +0,0 @@ -import "reflect-metadata"; -import { describe, expect, it } from "@jest/globals"; -import { - buildSelectQuery, - buildComparison, - buildIn, -} from "../../../../../../../../../src/core/platform/services/database/services/orm/connectors/cassandra/query-builder"; -import { DriveFile } from "../../../../../../../../../src/services/documents/entities/drive-file"; - -describe("The QueryBuilder module", () => { - describe("The buildSelectQuery function", () => { - it("should build a valid query from primary key parameters", () => { - const filters = { - company_id: "comp1", - parent_id: "parent1", - }; - const result = buildSelectQuery(DriveFile, filters, {}, { keyspace: "tdrive" }); - - expect(result).toEqual( - "SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1';", - ); - }); - - it("should build a valid query from primary key parameters and comparison", () => { - const filters = { - company_id: "comp1", - parent_id: "parent1", - }; - const result = buildSelectQuery( - DriveFile, - filters, - { - $lt: [["size", 1000]], - }, - { keyspace: "tdrive" }, - ); - - expect(result).toEqual( - "SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1' AND size < 1000;", - ); - }); - - it("should build IN query from array parameters", () => { - const filters = { - company_id: "comp1", - parent_id: "parent1", - creator: ["u1", "u2", "u3"], - }; - const result = buildSelectQuery(DriveFile, filters, {}, { keyspace: "tdrive" }); - - expect(result).toEqual( - "SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1' AND creator IN (u1,u2,u3);", - ); - }); - - it("should not build IN query from array parameters when array is empty", () => { - const filters = { - company_id: "comp1", - parent_id: "parent1", - user_id: [], - }; - const result = buildSelectQuery(DriveFile, filters, {}, { keyspace: "tdrive" }); - - expect(result).toEqual( - "SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1';", - ); - }); - }); - - describe("The buildComparison function", () => { - it("should create a valid < string", () => { - expect( - buildComparison({ - $lt: [["foo", 1]], - }), - ).toContain("foo < 1"); - - const result = buildComparison({ - $lt: [ - ["foo", 1], - ["bar", 2], - ], - }); - - expect(result).toContain("foo < 1"); - expect(result).toContain("bar < 2"); - }); - - it("should create a valid <= string", () => { - expect( - buildComparison({ - $lte: [["foo", 1]], - }), - ).toContain("foo <= 1"); - - const result = buildComparison({ - $lte: [ - ["foo", 1], - ["bar", 2], - ], - }); - - expect(result).toContain("foo <= 1"); - expect(result).toContain("bar <= 2"); - }); - - it("should create a valid > string", () => { - expect( - buildComparison({ - $gt: [["foo", 1]], - }), - ).toContain("foo > 1"); - - const result = buildComparison({ - $gt: [ - ["foo", 1], - ["bar", 2], - ], - }); - - expect(result).toContain("foo > 1"); - expect(result).toContain("bar > 2"); - }); - - it("should create a valid >= string", () => { - expect( - buildComparison({ - $gte: [["foo", 1]], - }), - ).toContain("foo >= 1"); - - const result = buildComparison({ - $gte: [ - ["foo", 1], - ["bar", 2], - ], - }); - - expect(result).toContain("foo >= 1"); - expect(result).toContain("bar >= 2"); - }); - - it("should combine conditions", () => { - const result = buildComparison({ - $gt: [["foo", 1]], - $gte: [["bar", 2]], - $lt: [["baz", 3]], - $lte: [["qix", 4]], - }); - expect(result).toContain("foo > 1"); - expect(result).toContain("bar >= 2"); - expect(result).toContain("baz < 3"); - expect(result).toContain("qix <= 4"); - }); - }); - - describe("The buildIn function", () => { - it("should create a id IN (ids) string", () => { - expect( - buildIn({ - $in: [["id", ["1", "2", "3"]]], - }), - ).toContain("id IN (1,2,3)"); - }); - }); -}); diff --git a/tdrive/default-configuration/backend-node/production.json b/tdrive/default-configuration/backend-node/production.json deleted file mode 100644 index d9de0a72..00000000 --- a/tdrive/default-configuration/backend-node/production.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "websocket": { - "path": "/socket/", - "adapters": { - "types": [] - }, - "auth": { - "jwt": { - "secret": "supersecret" - } - } - }, - "auth": { - "jwt": { - "secret": "supersecret" - } - }, - "database": { - "secret": "ab63bb3e90c0271c9a1c06651a7c0967eab8851a7a897766", - "type": "cassandra", - "cassandra": { - "contactPoints": ["scylladb:9042"], - "localDataCenter": "datacenter1", - "keyspace": "tdrive", - "wait": false, - "retries": 10, - "delay": 200 - } - }, - "pubsub": { - "type": "amqp", - "amqp": { - "urls": ["amqp://guest:guest@rabbitmq:5672"] - } - } -} diff --git a/tdrive/default-configuration/backend-node/production.json.template b/tdrive/default-configuration/backend-node/production.json.template deleted file mode 100644 index 29a26450..00000000 --- a/tdrive/default-configuration/backend-node/production.json.template +++ /dev/null @@ -1,37 +0,0 @@ -{ - "websocket": { - "path": "/socket/", - "adapters": { - "types": [] - }, - "auth": { - "jwt": { - "secret": "supersecret (use the same as in php)" - } - } - }, - "auth": { - "jwt": { - "secret": "supersecret (use the same as in php)" - } - }, - "database": { - "secret": "GET YOUR SECRET FROM PHP Parameters.php: db.secret", - "type": "cassandra", - "cassandra": { - "contactPoints": [ - "scylladb:9042" - ], - "localDataCenter": "datacenter1", - "keyspace": "tdrive", - "wait": false, - "retries": 10, - "delay": 200 - } - }, - "pubsub": { - "urls": [ - "amqp://admin:admin@rabbitmq:5672" - ] - } -} diff --git a/tdrive/default-configuration/backend/Parameters.php b/tdrive/default-configuration/backend/Parameters.php deleted file mode 100644 index 890ffef0..00000000 --- a/tdrive/default-configuration/backend/Parameters.php +++ /dev/null @@ -1,220 +0,0 @@ -configuration = [ - "env" => [ - "type" => "prod", - "timer" => false, - "admin_api_token" => "", - "licence_key" => "", - "standalone" => true, - //"frontend_server_name" => "http://localhost:8000/", // define only if needed - "server_name" => "http://localhost:8000/", - "internal_server_name" => "http://nginx/" - ], - "jwt" => [ - "secret" => "supersecret", - "expiration" => 60*60, //1 hour - "refresh_expiration" => 60*60*24*31 //1 month - ], - "node" => [ - "api" => "http://node:3000/private/", - "secret" => "api_supersecret" - ], - "db" => [ - "driver" => "pdo_cassandra", - "host" => "scylladb", - "port" => 9042, - "dbname" => "tdrive", - "user" => "root", - "password" => "root", - "encryption_key" => "ab63bb3e90c0271c9a1c06651a7c0967eab8851a7a897766", - "replication" => "{'class': 'SimpleStrategy', 'replication_factor': '1'}" - ], - "es" => [ - "host" => false - ], - "queues" => [ - "rabbitmq" => [ - "use" => true, - "host" => "rabbitmq", - "port" => 5672, - "username" => "guest", - "password" => "guest", - "vhost" => false - ] - ], - "storage" => [ - "drive_previews_tmp_folder" => "/tmp/", - "drive_tmp_folder" => "/tmp/", - "drive_salt" => "SecretPassword", - "providers" => [ - [ - "label" => "someOpenStack", - "type" => "openstack", - "use" => false, - "project_id" => "", - "auth_url" => "https//auth.cloud.ovh.net/v3", - "buckets_prefix" => "", - "disable_encryption" => false, - "buckets" => [ - "fr" => [ - "public" => "", - "private" => "", - "region" => "SBG5" - ] - ], - "user" => [ - "id" => "", - "password" => "", - "domain_name" => "default" - ] - ], - [ - "label" => "someS3", - "type" => "S3", - "base_url" => "http://127.0.0.1:9000", - "use" => false, - "version" => "latest", - "disable_encryption" => false, - "buckets_prefix" => "dev.", - "buckets" => [ - "fr" => "eu-west-3" - ], - "credentials" => [ - "key" => "", - "secret" => "" - ] - ], - [ - "label" => "someLocal", - "type" => "local", - "use" => true, - "disable_encryption" => false, - "location" => "../drive/", - "preview_location" => "../web/medias/", - "preview_public_path" => "/medias/" - ], - ], - ], - "mail" => [ - "sender" => [ - "host" => "", - "port" => "", - "username" => "", - "password" => "", - "auth_mode" => "plain" - ], - "template_dir" => "/src/Tdrive/Core/Resources/views/", - "tdrive_domain_url" => "https://tdriveapp.com/", - "from" => "noreply@tdriveapp.com", - "from_name" => "Tdrive", - "tdrive_address" => "Tdrive, 54000 Nancy, France", - "dkim" => [ - "private_key" => "", - "domain_name" => '', - "selector" => '' - ], - "mailjet"=> [ - "contact_list_subscribe"=> false, - "contact_list_newsletter"=> false - ] - ], - "push_notifications" => [ - "apns_certificate" => __DIR__ . "/certs/apns_prod.pem", - "firebase_api_key" => "KEY", - ], - - //Defaults values for all clients but editable in database - "defaults" => [ - "applications" => [ - "tdrive_drive" => [ "default" => true ], //False to not install - "tdrive_calendar" => [ "default" => true ], - "tdrive_tasks" => [ "default" => true ], - "connectors" => [ - "jitsi" => [ "default" => true ], - ] - ], - "connectors" => [ - ], - "branding" => [ - "name" => "Tdrive", - "enable_newsletter" => false, - /* - "header" => [ - "logo" => 'https://openpaas.linagora.com/images/white-logo.svg', - "apps" => [ - [ - "name"=> 'Accueil', - "url"=> 'https://openpaas.linagora.com/', - "icon"=> 'https://openpaas.linagora.com/images/application-menu/home-icon.svg', - ], - [ - "name"=> 'Inbox', - "url"=> 'https://openpaas.linagora.com/#/unifiedinbox/inbox', - "icon"=> 'https://openpaas.linagora.com/unifiedinbox/images/inbox-icon.svg', - ], - [ - "name"=> 'Calendrier', - "url"=> 'https://openpaas.linagora.com/#/calendar', - "icon"=> 'https://openpaas.linagora.com/calendar/images/calendar-icon.svg', - ], - [ - "name"=> 'Contacts', - "url"=> 'https://openpaas.linagora.com/#/contact/addressbooks/', - "icon"=> 'https://openpaas.linagora.com/contact/images/contacts-icon.svg', - ], - ], - ], - "style" => [ - "color" => '#2196F3', - "default_border_radius" => '2', - ], - "link" => "https://open-paas.org/", - "logo" => "https://open-paas.org/wp-content/uploads/2019/10/openpaas.png" - */ - ], - "auth" => [ - "internal" => [ - "use" => true, - "disable_account_creation" => false, - "disable_email_verification" => true - ], - "openid" => [ - "use" => false, - "provider_uri" => 'https://auth0.com', - "client_id" => '', - "client_secret" => '', - //"disable_logout_redirect" => false - "provider_config" => [ - //token_endpoint - //token_endpoint_auth_methods_supported - //userinfo_endpoint - //end_session_endpoint - //authorization_endpoint - ] - ], - "cas" => [ - "use" => false, - "base_url" => '', - "email_key" => '', - "lastname_key" => '', - "firstname_key" => '' - ], - "console" => [ - "use" => false - ] - ] - ], - ]; - } - -} diff --git a/tdrive/docker-compose.dev.yml b/tdrive/docker-compose.dev.yml deleted file mode 100644 index e8754a3d..00000000 --- a/tdrive/docker-compose.dev.yml +++ /dev/null @@ -1,70 +0,0 @@ -version: "3.4" - -services: - scylladb: - image: scylladb/scylla:4.1.0 - command: --smp 1 --memory 400M - ports: - - 9042:9042 - volumes: - - ./docker-data/scylladb:/var/lib/scylla - rabbitmq: - image: rabbitmq:3 - - node: - image: tdrive/tdrive-node - ports: - - 9229:9229 - environment: - - DEV=dev - - DB_DRIVER=cassandra - - DB_CASSANDRA_URI=scylladb:9042 - - DB_CASSANDRA_KEYSPACE=tdrive - - PUBSUB_URLS=amqp://guest:guest@rabbitmq:5672 - build: - context: . - dockerfile: docker/tdrive-node/Dockerfile - target: development - volumes: - - ./backend/node:/usr/src/app - - ./docker-data/documents/:/storage/ - depends_on: - - scylladb - - rabbitmq - - elasticsearch - links: - - scylladb - extra_hosts: - - "host.docker.internal:host-gateway" - - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.13.4 - environment: - - cluster.name=docker-cluster - - discovery.type=single-node - - bootstrap.memory_lock=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - ulimits: - memlock: - soft: -1 - hard: -1 - volumes: - - ./docker-data/es:/usr/share/elasticsearch/data - - nginx: - image: tdrive/tdrive-frontend - environment: - - DEV=dev - build: - context: . - dockerfile: docker/tdrive-frontend/Dockerfile - ports: - - 8000:80 - depends_on: - - node - volumes: - - ./docker-data/logs/nginx/:/var/log/nginx - - ./docker-data/letsencrypt/:/etc/letsencrypt/ - - ./frontend/:/tdrive-react/ - - ./docker-data/drive-preview/:/tdrive-core/web/medias/ - - ./docker-data/uploads/:/tdrive-core/web/upload/