♻️Remove Cassandra Support
This commit is contained in:
Anton SHEPILOV
committed by
Anton Shepilov
Anton Shepilov
parent
8214a5f6d4
commit
8f75c04fdd
@@ -68,12 +68,6 @@ jobs:
|
|||||||
docker-compose -f docker-compose.dev.tests.opensearch.yml logs
|
docker-compose -f docker-compose.dev.tests.opensearch.yml logs
|
||||||
docker-compose -f docker-compose.dev.tests.opensearch.yml run -e NODE_OPTIONS=--unhandled-rejections=warn -e SEARCH_DRIVER=opensearch -e DB_DRIVER=postgres -e PUBSUB_TYPE=local node npm run test:all
|
docker-compose -f docker-compose.dev.tests.opensearch.yml run -e NODE_OPTIONS=--unhandled-rejections=warn -e SEARCH_DRIVER=opensearch -e DB_DRIVER=postgres -e PUBSUB_TYPE=local node npm run test:all
|
||||||
docker-compose -f docker-compose.dev.tests.opensearch.yml down
|
docker-compose -f docker-compose.dev.tests.opensearch.yml down
|
||||||
- name: e2e-cassandra-test
|
|
||||||
run: |
|
|
||||||
cd tdrive
|
|
||||||
docker-compose -f docker-compose.tests.yml up -d scylladb elasticsearch rabbitmq
|
|
||||||
sleep 60
|
|
||||||
docker-compose -f docker-compose.tests.yml run -e NODE_OPTIONS=--unhandled-rejections=warn -e SEARCH_DRIVER=elasticsearch -e DB_DRIVER=cassandra node npm run test:all
|
|
||||||
- name: coverage
|
- name: coverage
|
||||||
uses: adRise/jest-cov-reporter@e13d7638888cad7788298f4022b203b31a10c7c8
|
uses: adRise/jest-cov-reporter@e13d7638888cad7788298f4022b203b31a10c7c8
|
||||||
with:
|
with:
|
||||||
|
|||||||
@@ -43,11 +43,6 @@
|
|||||||
"uri": "DB_MONGO_URI",
|
"uri": "DB_MONGO_URI",
|
||||||
"database": "DB_MONGO_DATABASE"
|
"database": "DB_MONGO_DATABASE"
|
||||||
},
|
},
|
||||||
"cassandra": {
|
|
||||||
"contactPoints": "DB_CASSANDRA_URI",
|
|
||||||
"localDataCenter": "DB_CASSANDRA_LOCALDATACENTER",
|
|
||||||
"keyspace": "DB_CASSANDRA_KEYSPACE"
|
|
||||||
},
|
|
||||||
"postgres": {
|
"postgres": {
|
||||||
"database": "DB_POSTGRES_DBNAME",
|
"database": "DB_POSTGRES_DBNAME",
|
||||||
"host": "DB_POSTGRES_HOST",
|
"host": "DB_POSTGRES_HOST",
|
||||||
|
|||||||
@@ -63,20 +63,12 @@
|
|||||||
},
|
},
|
||||||
"database": {
|
"database": {
|
||||||
"secret": "",
|
"secret": "",
|
||||||
"type": "cassandra",
|
"type": "mongodb",
|
||||||
"encryption": "aes-256-cbc",
|
"encryption": "aes-256-cbc",
|
||||||
"mongodb": {
|
"mongodb": {
|
||||||
"uri": "mongodb://mongo:27017",
|
"uri": "mongodb://mongo:27017",
|
||||||
"database": "tdrive"
|
"database": "tdrive"
|
||||||
},
|
},
|
||||||
"cassandra": {
|
|
||||||
"contactPoints": ["scylladb:9042"],
|
|
||||||
"localDataCenter": "datacenter1",
|
|
||||||
"keyspace": "tdrive",
|
|
||||||
"wait": false,
|
|
||||||
"retries": 10,
|
|
||||||
"delay": 200
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"message-queue": {
|
"message-queue": {
|
||||||
"// possible 'type' values are": "'amqp' or 'local'",
|
"// possible 'type' values are": "'amqp' or 'local'",
|
||||||
|
|||||||
@@ -8,15 +8,6 @@
|
|||||||
"secret": "",
|
"secret": "",
|
||||||
"mongodb": {
|
"mongodb": {
|
||||||
"uri": "mongodb://mongo:27017"
|
"uri": "mongodb://mongo:27017"
|
||||||
},
|
|
||||||
"cassandra": {
|
|
||||||
"contactPoints": ["scylladb:9042"],
|
|
||||||
"wait": true,
|
|
||||||
"retries": 50,
|
|
||||||
"delay": 500,
|
|
||||||
"queryOptions": {
|
|
||||||
"consistency": 1
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"message-queue": {
|
"message-queue": {
|
||||||
|
|||||||
@@ -129,7 +129,6 @@
|
|||||||
"archiver": "^5.3.1",
|
"archiver": "^5.3.1",
|
||||||
"axios": "^1.6.8",
|
"axios": "^1.6.8",
|
||||||
"bcrypt": "^5.0.1",
|
"bcrypt": "^5.0.1",
|
||||||
"cassandra-driver": "^4.6.0",
|
|
||||||
"class-transformer": "^0.3.1",
|
"class-transformer": "^0.3.1",
|
||||||
"cli-table": "^0.3.6",
|
"cli-table": "^0.3.6",
|
||||||
"config": "^3.3.2",
|
"config": "^3.3.2",
|
||||||
|
|||||||
@@ -29,14 +29,6 @@ export default function printConfigSummary(useIcons: boolean = !!process.env.HAV
|
|||||||
"user",
|
"user",
|
||||||
user,
|
user,
|
||||||
],
|
],
|
||||||
cassandra: ({ contactPoints, localDataCenter, keyspace }) => [
|
|
||||||
"host",
|
|
||||||
contactPoints,
|
|
||||||
"datacenter",
|
|
||||||
localDataCenter,
|
|
||||||
"keyspace",
|
|
||||||
keyspace,
|
|
||||||
],
|
|
||||||
S3: ({ endPoint, port, accessKey, bucket }) => [
|
S3: ({ endPoint, port, accessKey, bucket }) => [
|
||||||
"host",
|
"host",
|
||||||
[endPoint, port],
|
[endPoint, port],
|
||||||
@@ -62,7 +54,6 @@ export default function printConfigSummary(useIcons: boolean = !!process.env.HAV
|
|||||||
elasticsearch: "ES",
|
elasticsearch: "ES",
|
||||||
opensearch: "OS",
|
opensearch: "OS",
|
||||||
mongodb: "mongo",
|
mongodb: "mongo",
|
||||||
cassandra: "Cass.",
|
|
||||||
};
|
};
|
||||||
if (useIcons) {
|
if (useIcons) {
|
||||||
const makeLyingLengthIcon = str => ({ length: 1, toString: _ => str } as string);
|
const makeLyingLengthIcon = str => ({ length: 1, toString: _ => str } as string);
|
||||||
|
|||||||
@@ -1,2 +0,0 @@
|
|||||||
node_modules
|
|
||||||
yarn.lock
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
Edit the file to your needs
|
|
||||||
|
|
||||||
yarn install
|
|
||||||
node index.js
|
|
||||||
@@ -1,214 +0,0 @@
|
|||||||
/* eslint-disable */
|
|
||||||
var cassandra = require("cassandra-driver");
|
|
||||||
|
|
||||||
var fromAuthProvider = new cassandra.auth.PlainTextAuthProvider("", "");
|
|
||||||
var fromContactPoints = [""];
|
|
||||||
var fromKeyspace = "tdrive";
|
|
||||||
|
|
||||||
var toAuthProvider = new cassandra.auth.PlainTextAuthProvider("", "");
|
|
||||||
var toContactPoints = [""];
|
|
||||||
var toKeyspace = "tdrive";
|
|
||||||
|
|
||||||
var forceUpdateAll = false;
|
|
||||||
var ignoreTables = ["notification"];
|
|
||||||
var countersTables = ["statistics", "stats_counter", "channel_counters", "scheduled_queue_counter"];
|
|
||||||
|
|
||||||
var specialConversions = {
|
|
||||||
"tdrive.group_user.date_added": value => {
|
|
||||||
return new Date(value).getTime();
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
// -- start process
|
|
||||||
|
|
||||||
var fromClient = new cassandra.Client({
|
|
||||||
localDataCenter: "datacenter1",
|
|
||||||
contactPoints: fromContactPoints,
|
|
||||||
authProvider: fromAuthProvider,
|
|
||||||
keyspace: fromKeyspace,
|
|
||||||
});
|
|
||||||
|
|
||||||
var toClient = new cassandra.Client({
|
|
||||||
localDataCenter: "datacenter1",
|
|
||||||
contactPoints: toContactPoints,
|
|
||||||
authProvider: toAuthProvider,
|
|
||||||
keyspace: toKeyspace,
|
|
||||||
queryOptions: {
|
|
||||||
consistency: cassandra.types.consistencies.quorum,
|
|
||||||
},
|
|
||||||
});
|
|
||||||
|
|
||||||
// -- Get all tables and copy schema
|
|
||||||
|
|
||||||
async function client(origin, query, parameters, options) {
|
|
||||||
return await new Promise((resolve, reject) => {
|
|
||||||
origin.execute(query, [], {}, function (err, result) {
|
|
||||||
if (err) {
|
|
||||||
reject({ err, result });
|
|
||||||
} else {
|
|
||||||
resolve(result);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
(async () => {
|
|
||||||
const result = await client(
|
|
||||||
fromClient,
|
|
||||||
"SELECT table_name from system_schema.tables WHERE keyspace_name = '" + fromKeyspace + "'",
|
|
||||||
[],
|
|
||||||
{},
|
|
||||||
);
|
|
||||||
for (row of result.rows) {
|
|
||||||
try {
|
|
||||||
const fromTable = fromKeyspace + "." + row.table_name;
|
|
||||||
const toTable = toKeyspace + "." + row.table_name;
|
|
||||||
|
|
||||||
if (ignoreTables.includes(row.table_name)) {
|
|
||||||
console.log(fromTable.padEnd(50) + " | " + "ignored".padEnd(20) + " | ⏺");
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
let fromCount = 0;
|
|
||||||
|
|
||||||
const destColumns = (
|
|
||||||
await client(
|
|
||||||
toClient,
|
|
||||||
"SELECT column_name from system_schema.columns where keyspace_name = '" +
|
|
||||||
toKeyspace +
|
|
||||||
"' and table_name = '" +
|
|
||||||
row.table_name +
|
|
||||||
"'",
|
|
||||||
[],
|
|
||||||
{},
|
|
||||||
)
|
|
||||||
).rows.map(r => r.column_name);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const fromResult = await client(
|
|
||||||
fromClient,
|
|
||||||
"SELECT count(*) from " + fromTable + "",
|
|
||||||
[],
|
|
||||||
{},
|
|
||||||
);
|
|
||||||
fromCount = parseInt(fromResult.rows[0].count);
|
|
||||||
} catch (err) {
|
|
||||||
fromCount = NaN;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (destColumns.length === 0) {
|
|
||||||
if (fromCount > 0)
|
|
||||||
console.log(
|
|
||||||
fromTable.padEnd(50) + " | " + ("not_in_dest" + "/" + fromCount).padEnd(20) + " | ⏺",
|
|
||||||
);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
const toResult = await client(toClient, "SELECT count(*) from " + toTable + "", [], {});
|
|
||||||
const toCount = parseInt(toResult.rows[0].count);
|
|
||||||
|
|
||||||
console.log(
|
|
||||||
fromTable.padEnd(50) +
|
|
||||||
" | " +
|
|
||||||
(toCount + "/" + fromCount).padEnd(20) +
|
|
||||||
" | " +
|
|
||||||
(toCount >= fromCount ? "✅" : "❌"),
|
|
||||||
);
|
|
||||||
|
|
||||||
if (row.table_name.indexOf("counter") >= 0 || countersTables.includes(row.table_name)) {
|
|
||||||
console.log(
|
|
||||||
fromTable.padEnd(50) + " | " + ("counter_table" + "/" + fromCount).padEnd(20) + " | 🧮",
|
|
||||||
);
|
|
||||||
if (fromCount > toCount || !fromCount || forceUpdateAll) {
|
|
||||||
await new Promise(r => {
|
|
||||||
fromClient.eachRow(
|
|
||||||
"SELECT JSON * from " + fromTable,
|
|
||||||
[],
|
|
||||||
{ prepare: true, fetchSize: 1000 },
|
|
||||||
async function (n, row) {
|
|
||||||
//
|
|
||||||
//TODO handle counters (it is special !)
|
|
||||||
},
|
|
||||||
async function (err, result) {
|
|
||||||
if (result && result.nextPage) {
|
|
||||||
await new Promise(r => setTimeout(r, 1000));
|
|
||||||
result.nextPage();
|
|
||||||
} else {
|
|
||||||
r();
|
|
||||||
}
|
|
||||||
},
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (fromCount > toCount || !fromCount || forceUpdateAll) {
|
|
||||||
await new Promise(r => {
|
|
||||||
fromClient.eachRow(
|
|
||||||
"SELECT JSON * from " + fromTable,
|
|
||||||
[],
|
|
||||||
{ prepare: true, fetchSize: 1000 },
|
|
||||||
async function (n, row) {
|
|
||||||
try {
|
|
||||||
const json = JSON.parse(row["[json]"]);
|
|
||||||
|
|
||||||
//The from table can have additional depreciated fields, we need to remove them
|
|
||||||
const filteredJson = {};
|
|
||||||
for (const col of destColumns) {
|
|
||||||
if (
|
|
||||||
typeof json[col] == "string" &&
|
|
||||||
(json[col] || "").match(
|
|
||||||
/[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]+/,
|
|
||||||
)
|
|
||||||
) {
|
|
||||||
json[col] = json[col].split(".")[0];
|
|
||||||
}
|
|
||||||
|
|
||||||
if (specialConversions[fromTable + "." + col]) {
|
|
||||||
json[col] = specialConversions[fromTable + "." + col](json[col]);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (json[col] !== undefined) filteredJson[col] = json[col];
|
|
||||||
}
|
|
||||||
|
|
||||||
await client(
|
|
||||||
toClient,
|
|
||||||
"INSERT INTO " +
|
|
||||||
toTable +
|
|
||||||
" JSON '" +
|
|
||||||
JSON.stringify(filteredJson).replace(/'/g, "'$&") +
|
|
||||||
"'",
|
|
||||||
[],
|
|
||||||
{},
|
|
||||||
);
|
|
||||||
} catch (err) {
|
|
||||||
console.log(err);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
async function (err, result) {
|
|
||||||
if (result && result.nextPage) {
|
|
||||||
await new Promise(r => setTimeout(r, 1000));
|
|
||||||
result.nextPage();
|
|
||||||
} else {
|
|
||||||
r();
|
|
||||||
}
|
|
||||||
},
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.log(
|
|
||||||
fromTable.padEnd(50) + " | " + ("error" + "/" + fromCount).padEnd(20) + " | ❌",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.log(err);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
//TODO copy content
|
|
||||||
}
|
|
||||||
})();
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "copy-cluster",
|
|
||||||
"version": "1.0.0",
|
|
||||||
"main": "index.js",
|
|
||||||
"license": "MIT",
|
|
||||||
"dependencies": {
|
|
||||||
"cassandra-driver": "^4.6.3"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
Edit the file to your needs
|
|
||||||
|
|
||||||
yarn install
|
|
||||||
node index.js
|
|
||||||
@@ -1,88 +0,0 @@
|
|||||||
/* eslint-disable */
|
|
||||||
var cassandra = require("cassandra-driver");
|
|
||||||
|
|
||||||
var authProvider = new cassandra.auth.PlainTextAuthProvider("", "");
|
|
||||||
var contactPoints = [""];
|
|
||||||
var from = "table_a";
|
|
||||||
var to = "table_b";
|
|
||||||
|
|
||||||
var client = new cassandra.Client({
|
|
||||||
localDataCenter: "datacenter1",
|
|
||||||
contactPoints: contactPoints,
|
|
||||||
authProvider: authProvider,
|
|
||||||
keyspace: "tdrive",
|
|
||||||
});
|
|
||||||
|
|
||||||
//Copy object to other table
|
|
||||||
(async () => {
|
|
||||||
const query = "SELECT * FROM " + from;
|
|
||||||
const parameters = [];
|
|
||||||
|
|
||||||
let count = 0;
|
|
||||||
let max = 0;
|
|
||||||
let min = new Date().getTime();
|
|
||||||
|
|
||||||
let rows = [];
|
|
||||||
|
|
||||||
const options = { prepare: true, fetchSize: 1000 };
|
|
||||||
|
|
||||||
await new Promise(r => {
|
|
||||||
client.eachRow(
|
|
||||||
query,
|
|
||||||
parameters,
|
|
||||||
options,
|
|
||||||
function (n, row) {
|
|
||||||
min = Math.min(parseInt(row.created_at.toString()), min);
|
|
||||||
max = Math.max(parseInt(row.created_at.toString()), max);
|
|
||||||
count++;
|
|
||||||
if (count % 100 == 0) {
|
|
||||||
console.log(count);
|
|
||||||
}
|
|
||||||
rows.push(row);
|
|
||||||
},
|
|
||||||
function (err, result) {
|
|
||||||
if (result && result.nextPage) {
|
|
||||||
result.nextPage();
|
|
||||||
} else {
|
|
||||||
r();
|
|
||||||
}
|
|
||||||
},
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
console.log("Downloaded ", count, " rows of ", from, ". Starting copy...");
|
|
||||||
|
|
||||||
let copyCount = 0;
|
|
||||||
|
|
||||||
for (let i = 0; i < count; i++) {
|
|
||||||
copyCount++;
|
|
||||||
if (copyCount % 100 == 0) {
|
|
||||||
console.log(copyCount, "of", count);
|
|
||||||
}
|
|
||||||
|
|
||||||
const row = rows[i];
|
|
||||||
|
|
||||||
await new Promise(r => {
|
|
||||||
let query =
|
|
||||||
"UPDATE " +
|
|
||||||
to +
|
|
||||||
" SET answers=?, created_at=?, created_by=?, last_activity=?, participants=?, updated_at=? WHERE id=?";
|
|
||||||
client.execute(
|
|
||||||
query,
|
|
||||||
[
|
|
||||||
row.answers,
|
|
||||||
row.created_at,
|
|
||||||
row.created_by,
|
|
||||||
row.last_activity,
|
|
||||||
row.participants,
|
|
||||||
row.updated_at,
|
|
||||||
row.id,
|
|
||||||
],
|
|
||||||
() => {
|
|
||||||
r();
|
|
||||||
},
|
|
||||||
);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
console.log("Ended with ", count, "threads");
|
|
||||||
})();
|
|
||||||
@@ -1,6 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "copy-table",
|
|
||||||
"version": "1.0.0",
|
|
||||||
"main": "index.js",
|
|
||||||
"license": "MIT"
|
|
||||||
}
|
|
||||||
@@ -1,17 +1,11 @@
|
|||||||
import { DatabaseType } from ".";
|
import { DatabaseType } from ".";
|
||||||
import { ConnectionOptions, Connector } from "./orm/connectors";
|
import { ConnectionOptions, Connector } from "./orm/connectors";
|
||||||
import {
|
|
||||||
CassandraConnectionOptions,
|
|
||||||
CassandraConnector,
|
|
||||||
} from "./orm/connectors/cassandra/cassandra";
|
|
||||||
import { MongoConnectionOptions, MongoConnector } from "./orm/connectors/mongodb/mongodb";
|
import { MongoConnectionOptions, MongoConnector } from "./orm/connectors/mongodb/mongodb";
|
||||||
import { PostgresConnectionOptions, PostgresConnector } from "./orm/connectors/postgres/postgres";
|
import { PostgresConnectionOptions, PostgresConnector } from "./orm/connectors/postgres/postgres";
|
||||||
|
|
||||||
export class ConnectorFactory {
|
export class ConnectorFactory {
|
||||||
public create(type: DatabaseType, options: ConnectionOptions, secret: string): Connector {
|
public create(type: DatabaseType, options: ConnectionOptions, secret: string): Connector {
|
||||||
switch (type) {
|
switch (type) {
|
||||||
case "cassandra":
|
|
||||||
return new CassandraConnector(type, options as CassandraConnectionOptions, secret);
|
|
||||||
case "mongodb":
|
case "mongodb":
|
||||||
return new MongoConnector(type, options as MongoConnectionOptions, secret);
|
return new MongoConnector(type, options as MongoConnectionOptions, secret);
|
||||||
case "postgres":
|
case "postgres":
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ import { ConnectorFactory } from "./connector-factory";
|
|||||||
import { Connector } from "./orm/connectors";
|
import { Connector } from "./orm/connectors";
|
||||||
import Manager from "./orm/manager";
|
import Manager from "./orm/manager";
|
||||||
import Repository from "./orm/repository/repository";
|
import Repository from "./orm/repository/repository";
|
||||||
import { CassandraConnectionOptions } from "./orm/connectors/cassandra/cassandra";
|
|
||||||
import { MongoConnectionOptions } from "./orm/connectors/mongodb/mongodb";
|
import { MongoConnectionOptions } from "./orm/connectors/mongodb/mongodb";
|
||||||
import { EntityTarget } from "./orm/types";
|
import { EntityTarget } from "./orm/types";
|
||||||
import { RepositoryManager } from "./orm/repository/manager";
|
import { RepositoryManager } from "./orm/repository/manager";
|
||||||
@@ -48,9 +47,6 @@ export default class DatabaseService implements DatabaseServiceAPI {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export declare type ConnectionOptions =
|
export declare type ConnectionOptions = MongoConnectionOptions | PostgresConnectionOptions;
|
||||||
| MongoConnectionOptions
|
|
||||||
| CassandraConnectionOptions
|
|
||||||
| PostgresConnectionOptions;
|
|
||||||
|
|
||||||
export declare type DatabaseType = "mongodb" | "cassandra" | "postgres";
|
export declare type DatabaseType = "mongodb" | "postgres";
|
||||||
|
|||||||
-565
@@ -1,565 +0,0 @@
|
|||||||
/* eslint-disable @typescript-eslint/no-explicit-any, @typescript-eslint/explicit-module-boundary-types */
|
|
||||||
import cassandra, { types } from "cassandra-driver";
|
|
||||||
import { md5 } from "../../../../../../../crypto";
|
|
||||||
import { defer, Subject, throwError, timer } from "rxjs";
|
|
||||||
import { concat, delayWhen, retryWhen, take, tap } from "rxjs/operators";
|
|
||||||
import { UpsertOptions } from "..";
|
|
||||||
import { logger } from "../../../../../../framework";
|
|
||||||
import { getEntityDefinition, unwrapPrimarykey } from "../../utils";
|
|
||||||
import { EntityDefinition, ColumnDefinition, ObjectType } from "../../types";
|
|
||||||
import { AbstractConnector } from "../abstract-connector";
|
|
||||||
import {
|
|
||||||
transformValueToDbString,
|
|
||||||
cassandraType,
|
|
||||||
transformValueFromDbString,
|
|
||||||
} from "./typeTransforms";
|
|
||||||
import { FindOptions } from "../../repository/repository";
|
|
||||||
import { ListResult, Pagination } from "../../../../../../framework/api/crud-service";
|
|
||||||
import { Paginable } from "../../../../../../framework/api/crud-service";
|
|
||||||
import { buildSelectQuery } from "./query-builder";
|
|
||||||
|
|
||||||
export { CassandraPagination } from "./pagination";
|
|
||||||
|
|
||||||
export interface CassandraConnectionOptions {
|
|
||||||
contactPoints: string[];
|
|
||||||
localDataCenter: string;
|
|
||||||
username: string;
|
|
||||||
password: string;
|
|
||||||
keyspace: string;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Consistency level
|
|
||||||
*/
|
|
||||||
queryOptions: { consistency: number };
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Wait for keyspace and tables to be created at init
|
|
||||||
*/
|
|
||||||
wait: boolean;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* When wait = true, retry to get the resources N times where N = retries
|
|
||||||
*/
|
|
||||||
retries?: number;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delay in ms between the retries. The delay is growing each time a retry fails like delay = retryCount * delay
|
|
||||||
*/
|
|
||||||
delay?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export class CassandraConnector extends AbstractConnector<CassandraConnectionOptions> {
|
|
||||||
private client: cassandra.Client;
|
|
||||||
private keyspaceExists = false;
|
|
||||||
|
|
||||||
getClient(): cassandra.Client {
|
|
||||||
return this.client;
|
|
||||||
}
|
|
||||||
|
|
||||||
async init(): Promise<this> {
|
|
||||||
if (!this.client) {
|
|
||||||
await this.connect();
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await this.createKeyspace();
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn("services.database.orm.cassandra - Keyspace can not be created", err);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.options.wait) {
|
|
||||||
await this.waitForKeyspace(this.options.delay, this.options.retries);
|
|
||||||
}
|
|
||||||
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
createKeyspace(): Promise<cassandra.types.ResultSet> {
|
|
||||||
const query = `CREATE KEYSPACE IF NOT EXISTS ${this.options.keyspace} WITH replication = {'class': 'NetworkTopologyStrategy', 'datacenter1': '2'} AND durable_writes = true;`;
|
|
||||||
logger.info(query);
|
|
||||||
return this.client.execute(query);
|
|
||||||
}
|
|
||||||
|
|
||||||
async isKeyspaceCreated(): Promise<boolean> {
|
|
||||||
let result;
|
|
||||||
|
|
||||||
if (this.keyspaceExists) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
result = await this.client.execute(
|
|
||||||
`SELECT * FROM system_schema.keyspaces where keyspace_name = '${this.options.keyspace}'`,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!result) {
|
|
||||||
throw new Error("No result for keyspace query");
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
throw new Error("Keyspace query error");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (result) {
|
|
||||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
||||||
this.keyspaceExists = true;
|
|
||||||
logger.info(`Keyspace '${this.options.keyspace}' found.`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return result ? Promise.resolve(true) : Promise.reject(new Error("Keyspace not found"));
|
|
||||||
}
|
|
||||||
|
|
||||||
waitForKeyspace(delay: number = 500, retries: number = 10): Promise<boolean> {
|
|
||||||
const subject = new Subject<boolean>();
|
|
||||||
const obs$ = defer(() => this.isKeyspaceCreated());
|
|
||||||
|
|
||||||
obs$
|
|
||||||
.pipe(
|
|
||||||
retryWhen(errors =>
|
|
||||||
errors.pipe(
|
|
||||||
delayWhen((_, i) => timer(i * delay)),
|
|
||||||
//delay(1000), if we want fixed delay
|
|
||||||
tap(() => logger.debug("services.database.orm.cassandra - Retrying...")),
|
|
||||||
take(retries),
|
|
||||||
concat(throwError("Maximum number of retries reached")),
|
|
||||||
),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.subscribe(
|
|
||||||
() => logger.debug("services.database.orm.cassandra - Keyspace has been found"),
|
|
||||||
err => {
|
|
||||||
logger.error(
|
|
||||||
{ err },
|
|
||||||
"services.database.orm.cassandra - Error while getting keyspace information",
|
|
||||||
);
|
|
||||||
subject.error(new Error("Can not find keyspace information"));
|
|
||||||
},
|
|
||||||
() => subject.complete(),
|
|
||||||
);
|
|
||||||
|
|
||||||
return subject.toPromise();
|
|
||||||
}
|
|
||||||
|
|
||||||
async drop(): Promise<this> {
|
|
||||||
logger.info("Drop database is not implemented for security reasons.");
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
async connect(): Promise<this> {
|
|
||||||
if (this.client) {
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Environment variable format is comma separated string
|
|
||||||
const contactPoints =
|
|
||||||
typeof this.options.contactPoints === "string"
|
|
||||||
? (this.options.contactPoints as string).split(",")
|
|
||||||
: this.options.contactPoints;
|
|
||||||
|
|
||||||
const cassandraOptions: cassandra.DseClientOptions = {
|
|
||||||
contactPoints: contactPoints,
|
|
||||||
localDataCenter: this.options.localDataCenter,
|
|
||||||
queryOptions: {},
|
|
||||||
};
|
|
||||||
|
|
||||||
if (this.options.username && this.options.password) {
|
|
||||||
cassandraOptions.authProvider = new cassandra.auth.PlainTextAuthProvider(
|
|
||||||
this.options.username,
|
|
||||||
this.options.password,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
//Set default consistency level to quorum
|
|
||||||
cassandraOptions.queryOptions.consistency =
|
|
||||||
this.options?.queryOptions?.consistency || types.consistencies.quorum;
|
|
||||||
|
|
||||||
this.client = new cassandra.Client(cassandraOptions);
|
|
||||||
await this.client.connect();
|
|
||||||
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
async disconnect(): Promise<this> {
|
|
||||||
if (this.client) await this.client.shutdown();
|
|
||||||
this.client = null;
|
|
||||||
return this;
|
|
||||||
}
|
|
||||||
|
|
||||||
async getTableDefinition(name: string): Promise<string[]> {
|
|
||||||
let result: string[];
|
|
||||||
|
|
||||||
try {
|
|
||||||
const query = `SELECT column_name from system_schema.columns WHERE keyspace_name = '${this.options.keyspace}' AND table_name='${name}';`;
|
|
||||||
const dbResult = await this.client.execute(query);
|
|
||||||
result = dbResult.rows.map(row => row.values()[0]);
|
|
||||||
} catch (err) {
|
|
||||||
throw "Table query error";
|
|
||||||
}
|
|
||||||
|
|
||||||
return result ? Promise.resolve(result || []) : Promise.resolve([]);
|
|
||||||
}
|
|
||||||
|
|
||||||
async createTable(
|
|
||||||
entity: EntityDefinition,
|
|
||||||
columns: { [name: string]: ColumnDefinition },
|
|
||||||
): Promise<boolean> {
|
|
||||||
await this.waitForKeyspace(this.options.delay, this.options.retries);
|
|
||||||
|
|
||||||
let result = true;
|
|
||||||
|
|
||||||
// --- Generate column and key definition --- //
|
|
||||||
const primaryKey = entity.options.primaryKey || [];
|
|
||||||
|
|
||||||
if (primaryKey.length === 0) {
|
|
||||||
logger.error(
|
|
||||||
`services.database.orm.cassandra - Primary key was not defined for table ${entity.name}`,
|
|
||||||
);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const partitionKeyPart = primaryKey.shift();
|
|
||||||
const partitionKey: string[] =
|
|
||||||
typeof partitionKeyPart === "string" ? [partitionKeyPart] : partitionKeyPart;
|
|
||||||
const clusteringKeys: string[] = primaryKey as string[];
|
|
||||||
if ([...partitionKey, ...clusteringKeys].some(key => columns[key] === undefined)) {
|
|
||||||
logger.error(
|
|
||||||
`services.database.orm.cassandra - One primary key item doesn't exists in entity columns for table ${entity.name}`,
|
|
||||||
);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const clusteringOrderBy = clusteringKeys.map(key => {
|
|
||||||
const direction: "ASC" | "DESC" = columns[key].options.order || "ASC";
|
|
||||||
return `${key} ${direction}`;
|
|
||||||
});
|
|
||||||
const clusteringOrderByString =
|
|
||||||
clusteringOrderBy.length > 0
|
|
||||||
? `WITH CLUSTERING ORDER BY (${clusteringOrderBy.join(", ")})`
|
|
||||||
: "";
|
|
||||||
|
|
||||||
const allKeys = [`(${partitionKey.join(", ")})`, ...clusteringKeys];
|
|
||||||
const primaryKeyString = `(${allKeys.join(", ")})`;
|
|
||||||
|
|
||||||
const columnsString = Object.keys(columns)
|
|
||||||
.map(colName => {
|
|
||||||
const definition = columns[colName];
|
|
||||||
return `${colName} ${cassandraType[definition.type]}`;
|
|
||||||
})
|
|
||||||
.join(",\n");
|
|
||||||
|
|
||||||
// --- Generate final create table query --- //
|
|
||||||
let query = `
|
|
||||||
CREATE TABLE IF NOT EXISTS ${this.options.keyspace}.${entity.name}
|
|
||||||
(
|
|
||||||
${columnsString},
|
|
||||||
PRIMARY KEY ${primaryKeyString}
|
|
||||||
) ${clusteringOrderByString};`;
|
|
||||||
|
|
||||||
// --- Alter table if not up to date --- //
|
|
||||||
const existingColumns = await this.getTableDefinition(entity.name);
|
|
||||||
if (existingColumns.length > 0) {
|
|
||||||
logger.debug(
|
|
||||||
`services.database.orm.cassandra - Existing columns for table ${entity.name}, generating altertable queries`,
|
|
||||||
);
|
|
||||||
const alterQueryColumns = Object.keys(columns)
|
|
||||||
.filter(colName => existingColumns.indexOf(colName) < 0)
|
|
||||||
.map(colName => `${colName} ${cassandraType[columns[colName].type]}`);
|
|
||||||
query = `ALTER TABLE ${this.options.keyspace}.${entity.name} ADD (${alterQueryColumns.join(
|
|
||||||
", ",
|
|
||||||
)})`;
|
|
||||||
|
|
||||||
if (alterQueryColumns.length === 0) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Write table --- //
|
|
||||||
try {
|
|
||||||
logger.debug(`service.database.orm.createTable - Creating table ${entity.name} : ${query}`);
|
|
||||||
await this.client.execute(query);
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn(
|
|
||||||
{ err },
|
|
||||||
`service.database.orm.createTable - creation error for table ${entity.name} : ${err.message}`,
|
|
||||||
);
|
|
||||||
result = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Create indexes --- //
|
|
||||||
if (entity.options.globalIndexes) {
|
|
||||||
for (const globalIndex of entity.options.globalIndexes) {
|
|
||||||
const indexName = globalIndex.join("_");
|
|
||||||
const indexDbName = `index_${md5(indexName)}`;
|
|
||||||
|
|
||||||
const query = `CREATE INDEX IF NOT EXISTS ${indexDbName} ON ${this.options.keyspace}."${
|
|
||||||
entity.name
|
|
||||||
}" (${
|
|
||||||
globalIndex.length === 1 ? globalIndex[0] : `(${globalIndex[0]}), ${globalIndex[1]}`
|
|
||||||
})`;
|
|
||||||
|
|
||||||
try {
|
|
||||||
logger.debug(
|
|
||||||
`service.database.orm.createTable - Creating index ${indexName} (${indexDbName}) : ${query}`,
|
|
||||||
);
|
|
||||||
await this.client.execute(query);
|
|
||||||
} catch (err) {
|
|
||||||
logger.warn(
|
|
||||||
{ err },
|
|
||||||
`service.database.orm.createTable - creation error for index ${indexName} (${indexDbName}) : ${err.message}`,
|
|
||||||
);
|
|
||||||
result = false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
async upsert(entities: any[], _options: UpsertOptions = {}): Promise<boolean[]> {
|
|
||||||
return new Promise(resolve => {
|
|
||||||
const promises: Promise<boolean>[] = [];
|
|
||||||
|
|
||||||
entities.forEach(entity => {
|
|
||||||
const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
|
|
||||||
const primaryKey = unwrapPrimarykey(entityDefinition);
|
|
||||||
|
|
||||||
//Set updated content
|
|
||||||
const set = Object.keys(columnsDefinition)
|
|
||||||
.filter(key => primaryKey.indexOf(key) === -1)
|
|
||||||
.filter(key => entity[columnsDefinition[key].nodename] !== undefined)
|
|
||||||
.map(key => [
|
|
||||||
`${key}`,
|
|
||||||
`${transformValueToDbString(
|
|
||||||
entity[columnsDefinition[key].nodename],
|
|
||||||
columnsDefinition[key].type,
|
|
||||||
{
|
|
||||||
columns: columnsDefinition[key].options,
|
|
||||||
secret: this.secret,
|
|
||||||
column: { key },
|
|
||||||
},
|
|
||||||
)}`,
|
|
||||||
]);
|
|
||||||
//Set primary key
|
|
||||||
const where = primaryKey.map(key => [
|
|
||||||
`${key}`,
|
|
||||||
`${transformValueToDbString(
|
|
||||||
entity[columnsDefinition[key].nodename],
|
|
||||||
columnsDefinition[key].type,
|
|
||||||
{
|
|
||||||
columns: columnsDefinition[key].options,
|
|
||||||
secret: this.secret,
|
|
||||||
disableSalts: true,
|
|
||||||
column: { key },
|
|
||||||
},
|
|
||||||
)}`,
|
|
||||||
]);
|
|
||||||
|
|
||||||
// Add time-to-live options
|
|
||||||
let ttlOptions = "";
|
|
||||||
if (entityDefinition.options.ttl && entityDefinition.options.ttl > 0) {
|
|
||||||
ttlOptions = `USING TTL ${entityDefinition.options.ttl}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Insert and update are equivalent for most of Cassandra
|
|
||||||
// Update is prefered because the only solution for counters
|
|
||||||
let query = `UPDATE ${this.options.keyspace}.${
|
|
||||||
entityDefinition.name
|
|
||||||
} ${ttlOptions} SET ${set.map(e => `${e[0]} = ${e[1]}`).join(", ")} WHERE ${where
|
|
||||||
.map(e => `${e[0]} = ${e[1]}`)
|
|
||||||
.join(" AND ")}`;
|
|
||||||
|
|
||||||
// If no "set" part, we cannot do an update, so insert
|
|
||||||
if (set.length === 0) {
|
|
||||||
query = `INSERT INTO ${this.options.keyspace}.${
|
|
||||||
entityDefinition.name
|
|
||||||
} ${ttlOptions} (${where.map(e => e[0]).join(", ")}) VALUES (${where
|
|
||||||
.map(e => e[1])
|
|
||||||
.join(", ")})`;
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.debug(`service.database.orm.upsert - Query: "${query}"`);
|
|
||||||
|
|
||||||
promises.push(
|
|
||||||
new Promise(async resolve => {
|
|
||||||
try {
|
|
||||||
await this.getClient().execute(query);
|
|
||||||
resolve(true);
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
{ err },
|
|
||||||
`services.database.orm.cassandra - Error with CQL query: ${query}`,
|
|
||||||
);
|
|
||||||
resolve(false);
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
Promise.all(promises).then(resolve);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async remove(entities: any[]): Promise<boolean[]> {
|
|
||||||
return new Promise(resolve => {
|
|
||||||
const promises: Promise<boolean>[] = [];
|
|
||||||
|
|
||||||
entities.forEach(entity => {
|
|
||||||
const { columnsDefinition, entityDefinition } = getEntityDefinition(entity);
|
|
||||||
const primaryKey = unwrapPrimarykey(entityDefinition);
|
|
||||||
|
|
||||||
//Set primary key
|
|
||||||
const where = primaryKey.map(
|
|
||||||
key =>
|
|
||||||
`${key} = ${transformValueToDbString(
|
|
||||||
entity[columnsDefinition[key].nodename],
|
|
||||||
columnsDefinition[key].type,
|
|
||||||
{
|
|
||||||
columns: columnsDefinition[key].options,
|
|
||||||
secret: this.secret,
|
|
||||||
disableSalts: true,
|
|
||||||
column: { key },
|
|
||||||
},
|
|
||||||
)}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
const query = `DELETE FROM ${this.options.keyspace}.${
|
|
||||||
entityDefinition.name
|
|
||||||
} WHERE ${where.join(" AND ")}`;
|
|
||||||
logger.debug(`services.database.orm.cassandra.remove - Query: ${query}`);
|
|
||||||
|
|
||||||
promises.push(
|
|
||||||
new Promise(async resolve => {
|
|
||||||
try {
|
|
||||||
await this.getClient().execute(query);
|
|
||||||
resolve(true);
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
{ err },
|
|
||||||
`services.database.orm.cassandra.remove - Error with CQL query: ${query}`,
|
|
||||||
);
|
|
||||||
resolve(false);
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
Promise.all(promises).then(resolve);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async find<Table>(
|
|
||||||
entityType: Table,
|
|
||||||
filters: any,
|
|
||||||
options: FindOptions = {},
|
|
||||||
): Promise<ListResult<Table>> {
|
|
||||||
const instance = new (entityType as any)();
|
|
||||||
const { columnsDefinition, entityDefinition } = getEntityDefinition(instance);
|
|
||||||
|
|
||||||
const query = buildSelectQuery<Table>(
|
|
||||||
entityType as unknown as ObjectType<Table>,
|
|
||||||
filters,
|
|
||||||
options,
|
|
||||||
{
|
|
||||||
keyspace: this.options.keyspace,
|
|
||||||
secret: this.secret,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
logger.debug(`services.database.orm.cassandra.find - Query: ${query}`);
|
|
||||||
|
|
||||||
const results = await this.getClient().execute(query, [], {
|
|
||||||
fetchSize: parseInt(options.pagination.limitStr) || 100,
|
|
||||||
pageState: options.pagination.page_token || undefined,
|
|
||||||
prepare: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
const entities: Table[] = [];
|
|
||||||
results.rows.forEach(row => {
|
|
||||||
const entity = new (entityType as any)();
|
|
||||||
Object.keys(row).forEach(key => {
|
|
||||||
if (columnsDefinition[key]) {
|
|
||||||
entity[columnsDefinition[key].nodename] = transformValueFromDbString(
|
|
||||||
row[key],
|
|
||||||
columnsDefinition[key].type,
|
|
||||||
{ column: { key: key, ...columnsDefinition[key].options }, secret: this.secret },
|
|
||||||
);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
entities.push(entity);
|
|
||||||
});
|
|
||||||
|
|
||||||
const nextPage: Paginable = new Pagination(
|
|
||||||
results.pageState,
|
|
||||||
options.pagination.limitStr || "100",
|
|
||||||
);
|
|
||||||
logger.debug(
|
|
||||||
`services.database.orm.cassandra.find - Query Result (items=${entities.length}): ${query}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
return new ListResult<Table>(entityDefinition.type, entities, nextPage);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export function waitForTable(
|
|
||||||
client: cassandra.Client,
|
|
||||||
keyspace: string,
|
|
||||||
table: string,
|
|
||||||
retries: number = 10,
|
|
||||||
delay: number = 500,
|
|
||||||
): Promise<boolean> {
|
|
||||||
const subject = new Subject<boolean>();
|
|
||||||
const obs$ = defer(() => checkForTable(client, keyspace, table));
|
|
||||||
|
|
||||||
obs$
|
|
||||||
.pipe(
|
|
||||||
retryWhen(errors =>
|
|
||||||
errors.pipe(
|
|
||||||
delayWhen((_, i) => timer(i * delay)),
|
|
||||||
//delay(1000),
|
|
||||||
tap(() =>
|
|
||||||
logger.debug("services.database.orm.cassandra - Retrying to get table metadata..."),
|
|
||||||
),
|
|
||||||
take(retries),
|
|
||||||
concat(throwError("Maximum number of retries reached")),
|
|
||||||
),
|
|
||||||
),
|
|
||||||
)
|
|
||||||
.subscribe(
|
|
||||||
() => logger.debug(`services.database.orm.cassandra - Table ${table} has been found`),
|
|
||||||
err => {
|
|
||||||
logger.debug({ err }, `services.database.orm.cassandra - Table ${table} error`);
|
|
||||||
subject.error(new Error("Can not find table"));
|
|
||||||
},
|
|
||||||
() => subject.complete(),
|
|
||||||
);
|
|
||||||
|
|
||||||
return subject.toPromise();
|
|
||||||
}
|
|
||||||
|
|
||||||
async function checkForTable(
|
|
||||||
client: cassandra.Client,
|
|
||||||
keyspace: string,
|
|
||||||
table: string,
|
|
||||||
): Promise<void> {
|
|
||||||
try {
|
|
||||||
const result: cassandra.types.ResultSet = await client.execute(
|
|
||||||
`SELECT * FROM system_schema.tables WHERE keyspace_name='${keyspace}' AND table_name='${table}'`,
|
|
||||||
);
|
|
||||||
|
|
||||||
const tableMetadata = result.rows[0];
|
|
||||||
|
|
||||||
logger.debug(
|
|
||||||
"services.database.orm.cassandra.checkForTable - Table metadata %o",
|
|
||||||
tableMetadata,
|
|
||||||
);
|
|
||||||
|
|
||||||
if (!tableMetadata) {
|
|
||||||
throw new Error("Can not find table metadata");
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
logger.error(
|
|
||||||
{ err },
|
|
||||||
"services.database.orm.cassandra.checkForTable - Error while getting table metadata",
|
|
||||||
);
|
|
||||||
throw new Error("Error while getting table metadata");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-19
@@ -1,19 +0,0 @@
|
|||||||
import { Paginable } from "../../../../../../framework/api/crud-service";
|
|
||||||
import { Pagination } from "../../../../../../framework/api/crud-service";
|
|
||||||
|
|
||||||
export class CassandraPagination extends Pagination {
|
|
||||||
limit = 100;
|
|
||||||
|
|
||||||
private constructor(readonly page_token: string, readonly limitStr = "100") {
|
|
||||||
super(page_token, limitStr);
|
|
||||||
this.limit = Number.parseInt(limitStr, 10);
|
|
||||||
}
|
|
||||||
|
|
||||||
static from(pagination: Paginable): CassandraPagination {
|
|
||||||
return new CassandraPagination(pagination.page_token, pagination.limitStr);
|
|
||||||
}
|
|
||||||
|
|
||||||
static next(current: Pagination, pageState: string): Paginable {
|
|
||||||
return new Pagination(pageState, current.limitStr);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
-136
@@ -1,136 +0,0 @@
|
|||||||
import { FindOptions } from "../../repository/repository";
|
|
||||||
import { ObjectType } from "../../types";
|
|
||||||
import { getEntityDefinition, secureOperators, filteringRequired } from "../../utils";
|
|
||||||
import { transformValueToDbString } from "./typeTransforms";
|
|
||||||
|
|
||||||
export function buildSelectQuery<Entity>(
|
|
||||||
entityType: ObjectType<Entity>,
|
|
||||||
filters: Record<string, unknown>,
|
|
||||||
findOptions: FindOptions,
|
|
||||||
options: {
|
|
||||||
secret?: string;
|
|
||||||
keyspace: string;
|
|
||||||
} = {
|
|
||||||
secret: "",
|
|
||||||
keyspace: "tdrive",
|
|
||||||
},
|
|
||||||
): string {
|
|
||||||
const instance = new (entityType as any)();
|
|
||||||
const { columnsDefinition, entityDefinition } = getEntityDefinition(instance);
|
|
||||||
let allowFiltering = false;
|
|
||||||
|
|
||||||
const where = Object.keys(filters)
|
|
||||||
.map(key => {
|
|
||||||
let result: string;
|
|
||||||
const filter = filters[key];
|
|
||||||
|
|
||||||
if (filteringRequired(key)) {
|
|
||||||
allowFiltering = true;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!filter) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Array.isArray(filter)) {
|
|
||||||
if (!filter.length) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const inClause: string[] = filter.map(
|
|
||||||
value =>
|
|
||||||
`${transformValueToDbString(value, columnsDefinition[key].type, {
|
|
||||||
columns: columnsDefinition[key].options,
|
|
||||||
secret: options.secret || "",
|
|
||||||
disableSalts: true,
|
|
||||||
})}`,
|
|
||||||
);
|
|
||||||
|
|
||||||
result = `${key} IN (${inClause.join(",")})`;
|
|
||||||
} else {
|
|
||||||
result = `${key} = ${transformValueToDbString(filter, columnsDefinition[key].type, {
|
|
||||||
columns: columnsDefinition[key].options,
|
|
||||||
secret: options.secret || "",
|
|
||||||
disableSalts: true,
|
|
||||||
})}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
return result;
|
|
||||||
})
|
|
||||||
.filter(Boolean);
|
|
||||||
|
|
||||||
secureOperators(transformValueToDbString, findOptions, entityType, options);
|
|
||||||
|
|
||||||
const whereClause = `${[
|
|
||||||
...where,
|
|
||||||
...(buildComparison(findOptions) || []),
|
|
||||||
...(buildIn(findOptions) || []),
|
|
||||||
...(buildLike(findOptions) || []),
|
|
||||||
].join(" AND ")}`.trimEnd();
|
|
||||||
|
|
||||||
let orderByClause = "";
|
|
||||||
if (findOptions.pagination?.reversed) {
|
|
||||||
orderByClause = `${entityDefinition.options.primaryKey
|
|
||||||
.slice(1)
|
|
||||||
.map(
|
|
||||||
(key: string) =>
|
|
||||||
`${key} ${(columnsDefinition[key].options.order || "ASC") === "ASC" ? "DESC" : "ASC"}`,
|
|
||||||
)
|
|
||||||
.join(", ")}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
const query = `SELECT * FROM ${options.keyspace}.${entityDefinition.name} ${
|
|
||||||
whereClause.trim().length ? "WHERE " + whereClause : ""
|
|
||||||
} ${orderByClause.trim().length ? "ORDER BY " + orderByClause : ""}`
|
|
||||||
.trimEnd()
|
|
||||||
.concat(allowFiltering ? " ALLOW FILTERING;" : ";");
|
|
||||||
|
|
||||||
return query;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildComparison(options: FindOptions = {}): string[] {
|
|
||||||
let lessClause;
|
|
||||||
let lessEqualClause;
|
|
||||||
let greaterClause;
|
|
||||||
let greaterEqualClause;
|
|
||||||
|
|
||||||
if (options.$lt) {
|
|
||||||
lessClause = options.$lt.map(element => `${element[0]} < ${element[1]}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.$lte) {
|
|
||||||
lessEqualClause = options.$lte.map(element => `${element[0]} <= ${element[1]}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.$gt) {
|
|
||||||
greaterClause = options.$gt.map(element => `${element[0]} > ${element[1]}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (options.$gte) {
|
|
||||||
greaterEqualClause = options.$gte.map(element => `${element[0]} >= ${element[1]}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return [
|
|
||||||
...(lessClause || []),
|
|
||||||
...(lessEqualClause || []),
|
|
||||||
...(greaterClause || []),
|
|
||||||
...(greaterEqualClause || []),
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildIn(options: FindOptions = {}): string[] {
|
|
||||||
let inClauses: string[];
|
|
||||||
if (options.$in) {
|
|
||||||
inClauses = options.$in.map(element => `${element[0]} IN (${element[1].join(",")})`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return inClauses || [];
|
|
||||||
}
|
|
||||||
|
|
||||||
export function buildLike(options: FindOptions = {}): string[] {
|
|
||||||
let likeClauses: string[];
|
|
||||||
if (options.$like) {
|
|
||||||
likeClauses = options.$like.map(element => `${element[0]} LIKE '%${element[1]}%`);
|
|
||||||
}
|
|
||||||
|
|
||||||
return likeClauses || [];
|
|
||||||
}
|
|
||||||
-174
@@ -1,174 +0,0 @@
|
|||||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
|
||||||
/* eslint-disable @typescript-eslint/no-unused-vars */
|
|
||||||
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
|
|
||||||
import { isBoolean, isInteger, isNull, isUndefined } from "lodash";
|
|
||||||
import { ColumnOptions, ColumnType } from "../../types";
|
|
||||||
import { decrypt, encrypt } from "../../../../../../../crypto";
|
|
||||||
import { logger } from "../../../../../../../../core/platform/framework";
|
|
||||||
|
|
||||||
export const cassandraType = {
|
|
||||||
encoded_string: "TEXT",
|
|
||||||
encoded_json: "TEXT",
|
|
||||||
string: "TEXT",
|
|
||||||
json: "TEXT",
|
|
||||||
number: "BIGINT",
|
|
||||||
timeuuid: "TIMEUUID",
|
|
||||||
uuid: "UUID",
|
|
||||||
counter: "COUNTER",
|
|
||||||
blob: "BLOB",
|
|
||||||
boolean: "BOOLEAN",
|
|
||||||
|
|
||||||
// backward compatibility
|
|
||||||
tdrive_boolean: "TINYINT",
|
|
||||||
tdrive_int: "INT", //Depreciated
|
|
||||||
tdrive_datetime: "TIMESTAMP", //Depreciated
|
|
||||||
};
|
|
||||||
|
|
||||||
type TransformOptions = {
|
|
||||||
secret?: any;
|
|
||||||
disableSalts?: boolean;
|
|
||||||
columns?: ColumnOptions;
|
|
||||||
column?: any;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const transformValueToDbString = (
|
|
||||||
v: any,
|
|
||||||
type: ColumnType,
|
|
||||||
options: TransformOptions = {},
|
|
||||||
): string => {
|
|
||||||
if (type === "tdrive_datetime") {
|
|
||||||
if (isNaN(v) || isNull(v)) {
|
|
||||||
return "null";
|
|
||||||
}
|
|
||||||
return `${v}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (type === "number" || type === "tdrive_int") {
|
|
||||||
if (isNull(v)) {
|
|
||||||
return "null";
|
|
||||||
}
|
|
||||||
if (isNaN(v)) {
|
|
||||||
return "null";
|
|
||||||
}
|
|
||||||
return `${parseInt(v)}`;
|
|
||||||
}
|
|
||||||
if (type === "uuid" || type === "timeuuid") {
|
|
||||||
if (isNull(v)) {
|
|
||||||
return "null";
|
|
||||||
}
|
|
||||||
|
|
||||||
v = (v || "").toString().replace(/[^a-zA-Z0-9-]/g, "");
|
|
||||||
return `${v}`;
|
|
||||||
}
|
|
||||||
if (type === "boolean") {
|
|
||||||
//Security to avoid string with "false" in it
|
|
||||||
if (!isInteger(v) && !isBoolean(v) && !isNull(v) && !isUndefined(v)) {
|
|
||||||
throw new Error(`'${v}' is not a ${type}`);
|
|
||||||
}
|
|
||||||
return `${!!v}`;
|
|
||||||
}
|
|
||||||
if (type === "tdrive_boolean") {
|
|
||||||
if (!isBoolean(v)) {
|
|
||||||
throw new Error(`'${v}' is not a ${type}`);
|
|
||||||
}
|
|
||||||
return v ? "1" : "0";
|
|
||||||
}
|
|
||||||
if (type === "encoded_string" || type === "encoded_json") {
|
|
||||||
if (type === "encoded_json") {
|
|
||||||
try {
|
|
||||||
v = JSON.stringify(v);
|
|
||||||
} catch (err) {
|
|
||||||
v = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const encrypted = encrypt(v, options.secret, { disableSalts: options.disableSalts });
|
|
||||||
return `'${(encrypted.data || "").toString().replace(/'/gm, "''")}'`;
|
|
||||||
}
|
|
||||||
if (type === "blob") {
|
|
||||||
return "''"; //Not implemented yet
|
|
||||||
}
|
|
||||||
if (type === "string" || type === "json") {
|
|
||||||
if (type === "json" && v !== null) {
|
|
||||||
try {
|
|
||||||
v = JSON.stringify(v);
|
|
||||||
} catch (err) {
|
|
||||||
v = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return `'${(v || "").toString().replace(/'/gm, "''")}'`;
|
|
||||||
}
|
|
||||||
if (type === "counter") {
|
|
||||||
if (isNaN(v)) throw new Error("Counter value should be a number");
|
|
||||||
return `${options.column.key} + ${v}`;
|
|
||||||
}
|
|
||||||
return `'${(v || "").toString().replace(/'/gm, "''")}'`;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const transformValueFromDbString = (
|
|
||||||
v: any,
|
|
||||||
type: string,
|
|
||||||
options: TransformOptions = {},
|
|
||||||
): any => {
|
|
||||||
logger.trace(`Transform value %o of type ${type}`, v);
|
|
||||||
|
|
||||||
if (type === "tdrive_datetime") {
|
|
||||||
return new Date(`${v}`).getTime();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (v !== null && (type === "encoded_string" || type === "encoded_json")) {
|
|
||||||
let decryptedValue: any;
|
|
||||||
|
|
||||||
if (typeof v === "string" && v.trim() === "") {
|
|
||||||
return v;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
decryptedValue = decrypt(v, options.secret).data;
|
|
||||||
} catch (err) {
|
|
||||||
logger.debug(`Can not decrypt data (${err.message}) %o of type ${type}`, v);
|
|
||||||
|
|
||||||
decryptedValue = v;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (type === "encoded_json") {
|
|
||||||
try {
|
|
||||||
decryptedValue = JSON.parse(decryptedValue);
|
|
||||||
} catch (err) {
|
|
||||||
logger.debug(
|
|
||||||
{ err },
|
|
||||||
`Can not parse JSON from decrypted data %o of type ${type}`,
|
|
||||||
decryptedValue,
|
|
||||||
);
|
|
||||||
decryptedValue = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return decryptedValue;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (type === "tdrive_boolean" || type === "boolean") {
|
|
||||||
return Boolean(v).valueOf();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (type === "json") {
|
|
||||||
try {
|
|
||||||
return JSON.parse(v);
|
|
||||||
} catch (err) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (type === "uuid" || type === "timeuuid") {
|
|
||||||
return v ? String(v).valueOf() : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (type === "number") {
|
|
||||||
return new Number(v).valueOf();
|
|
||||||
}
|
|
||||||
|
|
||||||
if (type === "counter") {
|
|
||||||
return new Number(v).valueOf();
|
|
||||||
}
|
|
||||||
|
|
||||||
return v;
|
|
||||||
};
|
|
||||||
+1
-6
@@ -1,6 +1,5 @@
|
|||||||
import { Initializable } from "../../../../../framework";
|
import { Initializable } from "../../../../../framework";
|
||||||
import { DatabaseType } from "../..";
|
import { DatabaseType } from "../..";
|
||||||
import { CassandraConnectionOptions } from "./cassandra/cassandra";
|
|
||||||
import { MongoConnectionOptions } from "./mongodb/mongodb";
|
import { MongoConnectionOptions } from "./mongodb/mongodb";
|
||||||
import { ColumnDefinition, EntityDefinition } from "../types";
|
import { ColumnDefinition, EntityDefinition } from "../types";
|
||||||
import { FindOptions } from "../repository/repository";
|
import { FindOptions } from "../repository/repository";
|
||||||
@@ -8,7 +7,6 @@ import { ListResult } from "../../../../../framework/api/crud-service";
|
|||||||
import { PostgresConnectionOptions } from "./postgres/postgres";
|
import { PostgresConnectionOptions } from "./postgres/postgres";
|
||||||
|
|
||||||
export * from "./mongodb/mongodb";
|
export * from "./mongodb/mongodb";
|
||||||
export * from "./cassandra/cassandra";
|
|
||||||
|
|
||||||
export type UpsertOptions = {
|
export type UpsertOptions = {
|
||||||
action?: "INSERT" | "UPDATE";
|
action?: "INSERT" | "UPDATE";
|
||||||
@@ -68,7 +66,4 @@ export interface Connector extends Initializable {
|
|||||||
): Promise<ListResult<EntityType>>;
|
): Promise<ListResult<EntityType>>;
|
||||||
}
|
}
|
||||||
|
|
||||||
export declare type ConnectionOptions =
|
export declare type ConnectionOptions = MongoConnectionOptions | PostgresConnectionOptions;
|
||||||
| MongoConnectionOptions
|
|
||||||
| CassandraConnectionOptions
|
|
||||||
| PostgresConnectionOptions;
|
|
||||||
|
|||||||
-166
@@ -1,166 +0,0 @@
|
|||||||
import "reflect-metadata";
|
|
||||||
import { describe, expect, it } from "@jest/globals";
|
|
||||||
import {
|
|
||||||
buildSelectQuery,
|
|
||||||
buildComparison,
|
|
||||||
buildIn,
|
|
||||||
} from "../../../../../../../../../src/core/platform/services/database/services/orm/connectors/cassandra/query-builder";
|
|
||||||
import { DriveFile } from "../../../../../../../../../src/services/documents/entities/drive-file";
|
|
||||||
|
|
||||||
describe("The QueryBuilder module", () => {
|
|
||||||
describe("The buildSelectQuery function", () => {
|
|
||||||
it("should build a valid query from primary key parameters", () => {
|
|
||||||
const filters = {
|
|
||||||
company_id: "comp1",
|
|
||||||
parent_id: "parent1",
|
|
||||||
};
|
|
||||||
const result = buildSelectQuery<DriveFile>(DriveFile, filters, {}, { keyspace: "tdrive" });
|
|
||||||
|
|
||||||
expect(result).toEqual(
|
|
||||||
"SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1';",
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should build a valid query from primary key parameters and comparison", () => {
|
|
||||||
const filters = {
|
|
||||||
company_id: "comp1",
|
|
||||||
parent_id: "parent1",
|
|
||||||
};
|
|
||||||
const result = buildSelectQuery<DriveFile>(
|
|
||||||
DriveFile,
|
|
||||||
filters,
|
|
||||||
{
|
|
||||||
$lt: [["size", 1000]],
|
|
||||||
},
|
|
||||||
{ keyspace: "tdrive" },
|
|
||||||
);
|
|
||||||
|
|
||||||
expect(result).toEqual(
|
|
||||||
"SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1' AND size < 1000;",
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should build IN query from array parameters", () => {
|
|
||||||
const filters = {
|
|
||||||
company_id: "comp1",
|
|
||||||
parent_id: "parent1",
|
|
||||||
creator: ["u1", "u2", "u3"],
|
|
||||||
};
|
|
||||||
const result = buildSelectQuery<DriveFile>(DriveFile, filters, {}, { keyspace: "tdrive" });
|
|
||||||
|
|
||||||
expect(result).toEqual(
|
|
||||||
"SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1' AND creator IN (u1,u2,u3);",
|
|
||||||
);
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should not build IN query from array parameters when array is empty", () => {
|
|
||||||
const filters = {
|
|
||||||
company_id: "comp1",
|
|
||||||
parent_id: "parent1",
|
|
||||||
user_id: [],
|
|
||||||
};
|
|
||||||
const result = buildSelectQuery<DriveFile>(DriveFile, filters, {}, { keyspace: "tdrive" });
|
|
||||||
|
|
||||||
expect(result).toEqual(
|
|
||||||
"SELECT * FROM tdrive.drive_files WHERE company_id = comp1 AND parent_id = 'parent1';",
|
|
||||||
);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("The buildComparison function", () => {
|
|
||||||
it("should create a valid < string", () => {
|
|
||||||
expect(
|
|
||||||
buildComparison({
|
|
||||||
$lt: [["foo", 1]],
|
|
||||||
}),
|
|
||||||
).toContain("foo < 1");
|
|
||||||
|
|
||||||
const result = buildComparison({
|
|
||||||
$lt: [
|
|
||||||
["foo", 1],
|
|
||||||
["bar", 2],
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result).toContain("foo < 1");
|
|
||||||
expect(result).toContain("bar < 2");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should create a valid <= string", () => {
|
|
||||||
expect(
|
|
||||||
buildComparison({
|
|
||||||
$lte: [["foo", 1]],
|
|
||||||
}),
|
|
||||||
).toContain("foo <= 1");
|
|
||||||
|
|
||||||
const result = buildComparison({
|
|
||||||
$lte: [
|
|
||||||
["foo", 1],
|
|
||||||
["bar", 2],
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result).toContain("foo <= 1");
|
|
||||||
expect(result).toContain("bar <= 2");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should create a valid > string", () => {
|
|
||||||
expect(
|
|
||||||
buildComparison({
|
|
||||||
$gt: [["foo", 1]],
|
|
||||||
}),
|
|
||||||
).toContain("foo > 1");
|
|
||||||
|
|
||||||
const result = buildComparison({
|
|
||||||
$gt: [
|
|
||||||
["foo", 1],
|
|
||||||
["bar", 2],
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result).toContain("foo > 1");
|
|
||||||
expect(result).toContain("bar > 2");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should create a valid >= string", () => {
|
|
||||||
expect(
|
|
||||||
buildComparison({
|
|
||||||
$gte: [["foo", 1]],
|
|
||||||
}),
|
|
||||||
).toContain("foo >= 1");
|
|
||||||
|
|
||||||
const result = buildComparison({
|
|
||||||
$gte: [
|
|
||||||
["foo", 1],
|
|
||||||
["bar", 2],
|
|
||||||
],
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(result).toContain("foo >= 1");
|
|
||||||
expect(result).toContain("bar >= 2");
|
|
||||||
});
|
|
||||||
|
|
||||||
it("should combine conditions", () => {
|
|
||||||
const result = buildComparison({
|
|
||||||
$gt: [["foo", 1]],
|
|
||||||
$gte: [["bar", 2]],
|
|
||||||
$lt: [["baz", 3]],
|
|
||||||
$lte: [["qix", 4]],
|
|
||||||
});
|
|
||||||
expect(result).toContain("foo > 1");
|
|
||||||
expect(result).toContain("bar >= 2");
|
|
||||||
expect(result).toContain("baz < 3");
|
|
||||||
expect(result).toContain("qix <= 4");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
|
|
||||||
describe("The buildIn function", () => {
|
|
||||||
it("should create a id IN (ids) string", () => {
|
|
||||||
expect(
|
|
||||||
buildIn({
|
|
||||||
$in: [["id", ["1", "2", "3"]]],
|
|
||||||
}),
|
|
||||||
).toContain("id IN (1,2,3)");
|
|
||||||
});
|
|
||||||
});
|
|
||||||
});
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
{
|
|
||||||
"websocket": {
|
|
||||||
"path": "/socket/",
|
|
||||||
"adapters": {
|
|
||||||
"types": []
|
|
||||||
},
|
|
||||||
"auth": {
|
|
||||||
"jwt": {
|
|
||||||
"secret": "supersecret"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"auth": {
|
|
||||||
"jwt": {
|
|
||||||
"secret": "supersecret"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"database": {
|
|
||||||
"secret": "ab63bb3e90c0271c9a1c06651a7c0967eab8851a7a897766",
|
|
||||||
"type": "cassandra",
|
|
||||||
"cassandra": {
|
|
||||||
"contactPoints": ["scylladb:9042"],
|
|
||||||
"localDataCenter": "datacenter1",
|
|
||||||
"keyspace": "tdrive",
|
|
||||||
"wait": false,
|
|
||||||
"retries": 10,
|
|
||||||
"delay": 200
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"pubsub": {
|
|
||||||
"type": "amqp",
|
|
||||||
"amqp": {
|
|
||||||
"urls": ["amqp://guest:guest@rabbitmq:5672"]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,37 +0,0 @@
|
|||||||
{
|
|
||||||
"websocket": {
|
|
||||||
"path": "/socket/",
|
|
||||||
"adapters": {
|
|
||||||
"types": []
|
|
||||||
},
|
|
||||||
"auth": {
|
|
||||||
"jwt": {
|
|
||||||
"secret": "supersecret (use the same as in php)"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"auth": {
|
|
||||||
"jwt": {
|
|
||||||
"secret": "supersecret (use the same as in php)"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"database": {
|
|
||||||
"secret": "GET YOUR SECRET FROM PHP Parameters.php: db.secret",
|
|
||||||
"type": "cassandra",
|
|
||||||
"cassandra": {
|
|
||||||
"contactPoints": [
|
|
||||||
"scylladb:9042"
|
|
||||||
],
|
|
||||||
"localDataCenter": "datacenter1",
|
|
||||||
"keyspace": "tdrive",
|
|
||||||
"wait": false,
|
|
||||||
"retries": 10,
|
|
||||||
"delay": 200
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"pubsub": {
|
|
||||||
"urls": [
|
|
||||||
"amqp://admin:admin@rabbitmq:5672"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,220 +0,0 @@
|
|||||||
<?php
|
|
||||||
|
|
||||||
namespace Configuration;
|
|
||||||
|
|
||||||
class Parameters extends \Common\Configuration
|
|
||||||
{
|
|
||||||
|
|
||||||
public $configuration = [];
|
|
||||||
|
|
||||||
public function __construct()
|
|
||||||
{
|
|
||||||
$this->configuration = [
|
|
||||||
"env" => [
|
|
||||||
"type" => "prod",
|
|
||||||
"timer" => false,
|
|
||||||
"admin_api_token" => "",
|
|
||||||
"licence_key" => "",
|
|
||||||
"standalone" => true,
|
|
||||||
//"frontend_server_name" => "http://localhost:8000/", // define only if needed
|
|
||||||
"server_name" => "http://localhost:8000/",
|
|
||||||
"internal_server_name" => "http://nginx/"
|
|
||||||
],
|
|
||||||
"jwt" => [
|
|
||||||
"secret" => "supersecret",
|
|
||||||
"expiration" => 60*60, //1 hour
|
|
||||||
"refresh_expiration" => 60*60*24*31 //1 month
|
|
||||||
],
|
|
||||||
"node" => [
|
|
||||||
"api" => "http://node:3000/private/",
|
|
||||||
"secret" => "api_supersecret"
|
|
||||||
],
|
|
||||||
"db" => [
|
|
||||||
"driver" => "pdo_cassandra",
|
|
||||||
"host" => "scylladb",
|
|
||||||
"port" => 9042,
|
|
||||||
"dbname" => "tdrive",
|
|
||||||
"user" => "root",
|
|
||||||
"password" => "root",
|
|
||||||
"encryption_key" => "ab63bb3e90c0271c9a1c06651a7c0967eab8851a7a897766",
|
|
||||||
"replication" => "{'class': 'SimpleStrategy', 'replication_factor': '1'}"
|
|
||||||
],
|
|
||||||
"es" => [
|
|
||||||
"host" => false
|
|
||||||
],
|
|
||||||
"queues" => [
|
|
||||||
"rabbitmq" => [
|
|
||||||
"use" => true,
|
|
||||||
"host" => "rabbitmq",
|
|
||||||
"port" => 5672,
|
|
||||||
"username" => "guest",
|
|
||||||
"password" => "guest",
|
|
||||||
"vhost" => false
|
|
||||||
]
|
|
||||||
],
|
|
||||||
"storage" => [
|
|
||||||
"drive_previews_tmp_folder" => "/tmp/",
|
|
||||||
"drive_tmp_folder" => "/tmp/",
|
|
||||||
"drive_salt" => "SecretPassword",
|
|
||||||
"providers" => [
|
|
||||||
[
|
|
||||||
"label" => "someOpenStack",
|
|
||||||
"type" => "openstack",
|
|
||||||
"use" => false,
|
|
||||||
"project_id" => "",
|
|
||||||
"auth_url" => "https//auth.cloud.ovh.net/v3",
|
|
||||||
"buckets_prefix" => "",
|
|
||||||
"disable_encryption" => false,
|
|
||||||
"buckets" => [
|
|
||||||
"fr" => [
|
|
||||||
"public" => "",
|
|
||||||
"private" => "",
|
|
||||||
"region" => "SBG5"
|
|
||||||
]
|
|
||||||
],
|
|
||||||
"user" => [
|
|
||||||
"id" => "",
|
|
||||||
"password" => "",
|
|
||||||
"domain_name" => "default"
|
|
||||||
]
|
|
||||||
],
|
|
||||||
[
|
|
||||||
"label" => "someS3",
|
|
||||||
"type" => "S3",
|
|
||||||
"base_url" => "http://127.0.0.1:9000",
|
|
||||||
"use" => false,
|
|
||||||
"version" => "latest",
|
|
||||||
"disable_encryption" => false,
|
|
||||||
"buckets_prefix" => "dev.",
|
|
||||||
"buckets" => [
|
|
||||||
"fr" => "eu-west-3"
|
|
||||||
],
|
|
||||||
"credentials" => [
|
|
||||||
"key" => "",
|
|
||||||
"secret" => ""
|
|
||||||
]
|
|
||||||
],
|
|
||||||
[
|
|
||||||
"label" => "someLocal",
|
|
||||||
"type" => "local",
|
|
||||||
"use" => true,
|
|
||||||
"disable_encryption" => false,
|
|
||||||
"location" => "../drive/",
|
|
||||||
"preview_location" => "../web/medias/",
|
|
||||||
"preview_public_path" => "/medias/"
|
|
||||||
],
|
|
||||||
],
|
|
||||||
],
|
|
||||||
"mail" => [
|
|
||||||
"sender" => [
|
|
||||||
"host" => "",
|
|
||||||
"port" => "",
|
|
||||||
"username" => "",
|
|
||||||
"password" => "",
|
|
||||||
"auth_mode" => "plain"
|
|
||||||
],
|
|
||||||
"template_dir" => "/src/Tdrive/Core/Resources/views/",
|
|
||||||
"tdrive_domain_url" => "https://tdriveapp.com/",
|
|
||||||
"from" => "noreply@tdriveapp.com",
|
|
||||||
"from_name" => "Tdrive",
|
|
||||||
"tdrive_address" => "Tdrive, 54000 Nancy, France",
|
|
||||||
"dkim" => [
|
|
||||||
"private_key" => "",
|
|
||||||
"domain_name" => '',
|
|
||||||
"selector" => ''
|
|
||||||
],
|
|
||||||
"mailjet"=> [
|
|
||||||
"contact_list_subscribe"=> false,
|
|
||||||
"contact_list_newsletter"=> false
|
|
||||||
]
|
|
||||||
],
|
|
||||||
"push_notifications" => [
|
|
||||||
"apns_certificate" => __DIR__ . "/certs/apns_prod.pem",
|
|
||||||
"firebase_api_key" => "KEY",
|
|
||||||
],
|
|
||||||
|
|
||||||
//Defaults values for all clients but editable in database
|
|
||||||
"defaults" => [
|
|
||||||
"applications" => [
|
|
||||||
"tdrive_drive" => [ "default" => true ], //False to not install
|
|
||||||
"tdrive_calendar" => [ "default" => true ],
|
|
||||||
"tdrive_tasks" => [ "default" => true ],
|
|
||||||
"connectors" => [
|
|
||||||
"jitsi" => [ "default" => true ],
|
|
||||||
]
|
|
||||||
],
|
|
||||||
"connectors" => [
|
|
||||||
],
|
|
||||||
"branding" => [
|
|
||||||
"name" => "Tdrive",
|
|
||||||
"enable_newsletter" => false,
|
|
||||||
/*
|
|
||||||
"header" => [
|
|
||||||
"logo" => 'https://openpaas.linagora.com/images/white-logo.svg',
|
|
||||||
"apps" => [
|
|
||||||
[
|
|
||||||
"name"=> 'Accueil',
|
|
||||||
"url"=> 'https://openpaas.linagora.com/',
|
|
||||||
"icon"=> 'https://openpaas.linagora.com/images/application-menu/home-icon.svg',
|
|
||||||
],
|
|
||||||
[
|
|
||||||
"name"=> 'Inbox',
|
|
||||||
"url"=> 'https://openpaas.linagora.com/#/unifiedinbox/inbox',
|
|
||||||
"icon"=> 'https://openpaas.linagora.com/unifiedinbox/images/inbox-icon.svg',
|
|
||||||
],
|
|
||||||
[
|
|
||||||
"name"=> 'Calendrier',
|
|
||||||
"url"=> 'https://openpaas.linagora.com/#/calendar',
|
|
||||||
"icon"=> 'https://openpaas.linagora.com/calendar/images/calendar-icon.svg',
|
|
||||||
],
|
|
||||||
[
|
|
||||||
"name"=> 'Contacts',
|
|
||||||
"url"=> 'https://openpaas.linagora.com/#/contact/addressbooks/',
|
|
||||||
"icon"=> 'https://openpaas.linagora.com/contact/images/contacts-icon.svg',
|
|
||||||
],
|
|
||||||
],
|
|
||||||
],
|
|
||||||
"style" => [
|
|
||||||
"color" => '#2196F3',
|
|
||||||
"default_border_radius" => '2',
|
|
||||||
],
|
|
||||||
"link" => "https://open-paas.org/",
|
|
||||||
"logo" => "https://open-paas.org/wp-content/uploads/2019/10/openpaas.png"
|
|
||||||
*/
|
|
||||||
],
|
|
||||||
"auth" => [
|
|
||||||
"internal" => [
|
|
||||||
"use" => true,
|
|
||||||
"disable_account_creation" => false,
|
|
||||||
"disable_email_verification" => true
|
|
||||||
],
|
|
||||||
"openid" => [
|
|
||||||
"use" => false,
|
|
||||||
"provider_uri" => 'https://auth0.com',
|
|
||||||
"client_id" => '',
|
|
||||||
"client_secret" => '',
|
|
||||||
//"disable_logout_redirect" => false
|
|
||||||
"provider_config" => [
|
|
||||||
//token_endpoint
|
|
||||||
//token_endpoint_auth_methods_supported
|
|
||||||
//userinfo_endpoint
|
|
||||||
//end_session_endpoint
|
|
||||||
//authorization_endpoint
|
|
||||||
]
|
|
||||||
],
|
|
||||||
"cas" => [
|
|
||||||
"use" => false,
|
|
||||||
"base_url" => '',
|
|
||||||
"email_key" => '',
|
|
||||||
"lastname_key" => '',
|
|
||||||
"firstname_key" => ''
|
|
||||||
],
|
|
||||||
"console" => [
|
|
||||||
"use" => false
|
|
||||||
]
|
|
||||||
]
|
|
||||||
],
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@@ -1,70 +0,0 @@
|
|||||||
version: "3.4"
|
|
||||||
|
|
||||||
services:
|
|
||||||
scylladb:
|
|
||||||
image: scylladb/scylla:4.1.0
|
|
||||||
command: --smp 1 --memory 400M
|
|
||||||
ports:
|
|
||||||
- 9042:9042
|
|
||||||
volumes:
|
|
||||||
- ./docker-data/scylladb:/var/lib/scylla
|
|
||||||
rabbitmq:
|
|
||||||
image: rabbitmq:3
|
|
||||||
|
|
||||||
node:
|
|
||||||
image: tdrive/tdrive-node
|
|
||||||
ports:
|
|
||||||
- 9229:9229
|
|
||||||
environment:
|
|
||||||
- DEV=dev
|
|
||||||
- DB_DRIVER=cassandra
|
|
||||||
- DB_CASSANDRA_URI=scylladb:9042
|
|
||||||
- DB_CASSANDRA_KEYSPACE=tdrive
|
|
||||||
- PUBSUB_URLS=amqp://guest:guest@rabbitmq:5672
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: docker/tdrive-node/Dockerfile
|
|
||||||
target: development
|
|
||||||
volumes:
|
|
||||||
- ./backend/node:/usr/src/app
|
|
||||||
- ./docker-data/documents/:/storage/
|
|
||||||
depends_on:
|
|
||||||
- scylladb
|
|
||||||
- rabbitmq
|
|
||||||
- elasticsearch
|
|
||||||
links:
|
|
||||||
- scylladb
|
|
||||||
extra_hosts:
|
|
||||||
- "host.docker.internal:host-gateway"
|
|
||||||
|
|
||||||
elasticsearch:
|
|
||||||
image: docker.elastic.co/elasticsearch/elasticsearch:7.13.4
|
|
||||||
environment:
|
|
||||||
- cluster.name=docker-cluster
|
|
||||||
- discovery.type=single-node
|
|
||||||
- bootstrap.memory_lock=true
|
|
||||||
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
|
|
||||||
ulimits:
|
|
||||||
memlock:
|
|
||||||
soft: -1
|
|
||||||
hard: -1
|
|
||||||
volumes:
|
|
||||||
- ./docker-data/es:/usr/share/elasticsearch/data
|
|
||||||
|
|
||||||
nginx:
|
|
||||||
image: tdrive/tdrive-frontend
|
|
||||||
environment:
|
|
||||||
- DEV=dev
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: docker/tdrive-frontend/Dockerfile
|
|
||||||
ports:
|
|
||||||
- 8000:80
|
|
||||||
depends_on:
|
|
||||||
- node
|
|
||||||
volumes:
|
|
||||||
- ./docker-data/logs/nginx/:/var/log/nginx
|
|
||||||
- ./docker-data/letsencrypt/:/etc/letsencrypt/
|
|
||||||
- ./frontend/:/tdrive-react/
|
|
||||||
- ./docker-data/drive-preview/:/tdrive-core/web/medias/
|
|
||||||
- ./docker-data/uploads/:/tdrive-core/web/upload/
|
|
||||||
Reference in New Issue
Block a user