🌟 Data migration tool from nextcloud (#299)
- created server configuration - implemented downloading files from nextcloud to a local directory - build of the docker image added to the GutHub workflow - Twake client added - Creating user in Twake Drive from ldap - Ldap client with ldapsearch(TODO make it work with ldapjs) - Jest debug tests
This commit is contained in:
Anton Shepilov
@@ -63,6 +63,13 @@
|
||||
"useAuth": "SEARCH_ES_USE_AUTH",
|
||||
"username": "SEARCH_ES_USERNAME",
|
||||
"password": "SEARCH_ES_PASSWORD"
|
||||
},
|
||||
"opensearch": {
|
||||
"endpoint": "SEARCH_OS_ENDPOINT",
|
||||
"flushInterval": "SEARCH_OS_FLUSHINTERVAL",
|
||||
"useAuth": "SEARCH_OS_USE_AUTH",
|
||||
"username": "SEARCH_OS_USERNAME",
|
||||
"password": "SEARCH_OS_PASSWORD"
|
||||
}
|
||||
},
|
||||
"storage": {
|
||||
|
||||
Generated
+24
@@ -1349,6 +1349,25 @@
|
||||
"fastq": "^1.6.0"
|
||||
}
|
||||
},
|
||||
"@opensearch-project/opensearch": {
|
||||
"version": "2.4.0",
|
||||
"resolved": "https://registry.npmjs.org/@opensearch-project/opensearch/-/opensearch-2.4.0.tgz",
|
||||
"integrity": "sha512-r0ZNIlDxAua1ZecOBJ8qOXshf2ZQhNKmfly7o0aNuACf0pDa6Et/8mWMZuaFOu7xlNEeRNB7IjDQUYFy2SPElw==",
|
||||
"requires": {
|
||||
"aws4": "^1.11.0",
|
||||
"debug": "^4.3.1",
|
||||
"hpagent": "^1.2.0",
|
||||
"ms": "^2.1.3",
|
||||
"secure-json-parse": "^2.4.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"hpagent": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz",
|
||||
"integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"@segment/loosely-validate-event": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@segment/loosely-validate-event/-/loosely-validate-event-2.0.0.tgz",
|
||||
@@ -2456,6 +2475,11 @@
|
||||
"queue-microtask": "^1.1.2"
|
||||
}
|
||||
},
|
||||
"aws4": {
|
||||
"version": "1.12.0",
|
||||
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz",
|
||||
"integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg=="
|
||||
},
|
||||
"axios": {
|
||||
"version": "0.21.4",
|
||||
"resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
|
||||
|
||||
@@ -107,6 +107,7 @@
|
||||
"@fastify/formbody": "^6.0.0",
|
||||
"@fastify/static": "^5.0.1",
|
||||
"@ffprobe-installer/ffprobe": "^1.4.1",
|
||||
"@opensearch-project/opensearch": "^2.4.0",
|
||||
"@sentry/node": "^6.19.7",
|
||||
"@sentry/tracing": "^6.19.7",
|
||||
"@socket.io/redis-adapter": "^7.2.0",
|
||||
|
||||
+328
@@ -0,0 +1,328 @@
|
||||
import { logger } from "../../../../framework";
|
||||
import _ from "lodash";
|
||||
import {
|
||||
ColumnDefinition,
|
||||
EntityDefinition,
|
||||
EntityTarget,
|
||||
FindFilter,
|
||||
FindOptions,
|
||||
IndexedEntity,
|
||||
SearchAdapterInterface,
|
||||
SearchConfiguration,
|
||||
} from "../../api";
|
||||
import { SearchAdapter } from "../abstract";
|
||||
import { DatabaseServiceAPI } from "../../../database/api";
|
||||
import { getEntityDefinition, unwrapPrimarykey } from "../../api";
|
||||
import { ListResult, Paginable, Pagination } from "../../../../framework/api/crud-service";
|
||||
import { asciiFold, parsePrimaryKey, stringifyPrimaryKey } from "../utils";
|
||||
import { buildSearchQuery } from "../elasticsearch/search";
|
||||
import { Client } from "@opensearch-project/opensearch";
|
||||
|
||||
type Operation = {
|
||||
index?: { _index: string; _id: string };
|
||||
delete?: { _index: string; _id: string };
|
||||
[key: string]: any;
|
||||
};
|
||||
|
||||
export default class OpenSearch extends SearchAdapter implements SearchAdapterInterface {
|
||||
private client: Client;
|
||||
private bulkReaders = 0;
|
||||
private buffer: Operation[] = [];
|
||||
private name = "ElasticSearch";
|
||||
|
||||
constructor(
|
||||
readonly database: DatabaseServiceAPI,
|
||||
readonly configuration: SearchConfiguration["elasticsearch"],
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
public async connect() {
|
||||
try {
|
||||
const clientOptions: any = {
|
||||
node: this.configuration.endpoint,
|
||||
ssl: {
|
||||
rejectUnauthorized: false,
|
||||
},
|
||||
};
|
||||
|
||||
if (this.configuration.useAuth) {
|
||||
logger.info("Using auth for ES client");
|
||||
clientOptions.auth = {
|
||||
username: this.configuration.username,
|
||||
password: this.configuration.password,
|
||||
};
|
||||
}
|
||||
|
||||
this.client = new Client(clientOptions);
|
||||
} catch (e) {
|
||||
logger.error(
|
||||
`Unable to connect to ElasticSearch for options: ${JSON.stringify({
|
||||
node: this.configuration.endpoint,
|
||||
auth: {
|
||||
useAuth: this.configuration.useAuth,
|
||||
username: this.configuration.username,
|
||||
password: this.configuration.password,
|
||||
},
|
||||
ssl: {
|
||||
rejectUnauthorized: false,
|
||||
},
|
||||
})} at: ${this.configuration.endpoint}`,
|
||||
);
|
||||
}
|
||||
this.startBulkReader();
|
||||
}
|
||||
|
||||
private async createIndex(
|
||||
entity: EntityDefinition,
|
||||
_columns: { [name: string]: ColumnDefinition },
|
||||
) {
|
||||
if (!entity.options?.search) {
|
||||
return;
|
||||
}
|
||||
|
||||
const name = entity.options?.search?.index || entity.name;
|
||||
const mapping = entity.options?.search?.esMapping;
|
||||
|
||||
try {
|
||||
await this.client.indices.get({
|
||||
index: name,
|
||||
});
|
||||
logger.info(`Index "${name}" already created`);
|
||||
} catch (e) {
|
||||
logger.info(`Create index ${name} with mapping %o`, mapping);
|
||||
|
||||
const indice = {
|
||||
index: name,
|
||||
body: {
|
||||
settings: {
|
||||
analysis: {
|
||||
analyzer: {
|
||||
folding: {
|
||||
tokenizer: "standard",
|
||||
filter: ["lowercase", "asciifolding"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
mappings: { ...mapping, _source: { enabled: false } },
|
||||
},
|
||||
};
|
||||
|
||||
const rep = await this.client.indices.create(indice, { ignore: [400] });
|
||||
|
||||
if (rep.statusCode !== 200) {
|
||||
logger.error(`${this.name} - ${JSON.stringify(rep.body)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async upsert(entities: any[]) {
|
||||
for (const entity of entities) {
|
||||
const { entityDefinition, columnsDefinition } = getEntityDefinition(entity);
|
||||
const pkColumns = unwrapPrimarykey(entityDefinition);
|
||||
|
||||
await this.ensureIndex(entityDefinition, columnsDefinition, this.createIndex.bind(this));
|
||||
|
||||
if (!entityDefinition.options?.search) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (
|
||||
entityDefinition.options.search.shouldUpdate &&
|
||||
!entityDefinition.options.search.shouldUpdate(entity)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!entityDefinition.options?.search?.source) {
|
||||
logger.info(`Unable to do operation upsert to elasticsearch for doc ${entity}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const body = {
|
||||
..._.pick(entity, ...pkColumns),
|
||||
...entityDefinition.options.search.source(entity),
|
||||
};
|
||||
|
||||
Object.keys(entityDefinition.options?.search.esMapping?.properties || []).forEach(
|
||||
(key: string) => {
|
||||
const mapping: any = entityDefinition.options?.search?.esMapping?.properties[key];
|
||||
if (mapping.type === "text") {
|
||||
body[key] = asciiFold(body[key]).toLocaleLowerCase();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
const index = entityDefinition.options?.search?.index || entityDefinition.name;
|
||||
|
||||
const record: Operation = {
|
||||
index: {
|
||||
_index: index,
|
||||
_id: stringifyPrimaryKey(entity),
|
||||
},
|
||||
...body,
|
||||
};
|
||||
|
||||
logger.info(`Add operation upsert to elasticsearch for doc ${record.id}`);
|
||||
|
||||
this.buffer.push(record);
|
||||
}
|
||||
|
||||
this.startBulkReader();
|
||||
}
|
||||
|
||||
public async remove(entities: any[]) {
|
||||
for (const entity of entities) {
|
||||
const { entityDefinition, columnsDefinition } = getEntityDefinition(entity);
|
||||
|
||||
await this.ensureIndex(entityDefinition, columnsDefinition, this.createIndex.bind(this));
|
||||
|
||||
if (!entityDefinition.options?.search) {
|
||||
return;
|
||||
}
|
||||
|
||||
const index = entityDefinition.options?.search?.index || entityDefinition.name;
|
||||
|
||||
const record: Operation = {
|
||||
delete: {
|
||||
_index: index,
|
||||
_id: stringifyPrimaryKey(entity),
|
||||
},
|
||||
};
|
||||
|
||||
logger.info(`Add operation remove from elasticsearch for doc ${record.id}`);
|
||||
|
||||
this.buffer.push(record);
|
||||
}
|
||||
|
||||
this.startBulkReader();
|
||||
}
|
||||
|
||||
private async startBulkReader() {
|
||||
if (this.bulkReaders > 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info("Start new Elasticsearch bulk reader.");
|
||||
this.bulkReaders += 1;
|
||||
|
||||
let buffer;
|
||||
do {
|
||||
await new Promise(r =>
|
||||
setTimeout(r, parseInt(`${this.configuration.flushInterval}`) || 3000),
|
||||
);
|
||||
buffer = this.buffer;
|
||||
} while (buffer.length === 0);
|
||||
this.buffer = [];
|
||||
|
||||
try {
|
||||
await this.client.helpers.bulk({
|
||||
flushInterval: 1,
|
||||
datasource: buffer,
|
||||
onDocument: (doc: Operation) => {
|
||||
if (doc.delete) {
|
||||
logger.info(
|
||||
`Operation ${"DELETE"} pushed to elasticsearch index ${doc.delete._index} (doc.id: ${
|
||||
doc.delete._id
|
||||
})`,
|
||||
);
|
||||
return {
|
||||
delete: doc.delete,
|
||||
};
|
||||
}
|
||||
if (doc.index) {
|
||||
logger.info(
|
||||
`Operation ${"INDEX"} pushed to elasticsearch index ${doc.index._index} (doc.id: ${
|
||||
doc.index._id
|
||||
})`,
|
||||
);
|
||||
return {
|
||||
index: doc.index,
|
||||
...doc.index,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
},
|
||||
onDrop: res => {
|
||||
const doc = res.document;
|
||||
logger.error(
|
||||
`Operation ${
|
||||
doc.action
|
||||
} was droped while pushing to elasticsearch index ${JSON.stringify(
|
||||
doc.index,
|
||||
)} (doc.id: ${doc.id})`,
|
||||
);
|
||||
logger.error(res.error);
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error(`${this.name} - An error occured with the bulk reader`);
|
||||
logger.error(err);
|
||||
}
|
||||
|
||||
logger.info("Elasticsearch bulk flushed.");
|
||||
this.bulkReaders += -1;
|
||||
|
||||
this.startBulkReader();
|
||||
}
|
||||
|
||||
public async search<EntityType>(
|
||||
_table: string,
|
||||
entityType: EntityTarget<EntityType>,
|
||||
filters: FindFilter,
|
||||
options: FindOptions = {},
|
||||
) {
|
||||
const instance = new (entityType as any)();
|
||||
const { entityDefinition } = getEntityDefinition(instance);
|
||||
|
||||
const { esParams, esOptions } = buildSearchQuery<EntityType>(entityType, filters, options);
|
||||
const esParamsWithScroll = {
|
||||
...esParams,
|
||||
size: parseInt(options.pagination.limitStr || "100"),
|
||||
scroll: "1m",
|
||||
};
|
||||
|
||||
let esResponse: any;
|
||||
|
||||
if (options.pagination.page_token) {
|
||||
esResponse = await this.client.scroll(
|
||||
{
|
||||
scroll_id: options.pagination.page_token,
|
||||
},
|
||||
esOptions,
|
||||
);
|
||||
} else {
|
||||
esResponse = await this.client.search(esParamsWithScroll, esOptions);
|
||||
}
|
||||
|
||||
if (esResponse.statusCode !== 200) {
|
||||
logger.error(`${this.name} - ${JSON.stringify(esResponse.body)}`);
|
||||
}
|
||||
|
||||
const nextToken = esResponse.body?._scroll_id || "";
|
||||
const hits = esResponse.body?.hits?.hits || [];
|
||||
|
||||
logger.debug(`${this.name} got response: ${JSON.stringify(esResponse)}`);
|
||||
|
||||
const entities: IndexedEntity[] = [];
|
||||
for await (const hit of hits) {
|
||||
try {
|
||||
entities.push({
|
||||
primaryKey: parsePrimaryKey(entityDefinition, hit._id),
|
||||
score: hit._score,
|
||||
});
|
||||
} catch (err) {
|
||||
logger.error(
|
||||
`${this.name} failed to get entity from search result: ${JSON.stringify(
|
||||
hit._id,
|
||||
)}, ${JSON.stringify(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const nextPage: Paginable = new Pagination(nextToken, options.pagination.limitStr || "100");
|
||||
|
||||
return new ListResult(entityDefinition.type, entities, nextPage);
|
||||
}
|
||||
}
|
||||
@@ -1,8 +1,3 @@
|
||||
# The task is:
|
||||
# to find all the files that I havee access to, it shoud be a doc files,
|
||||
# that was modified during the last week and share with my by Diana Potoking
|
||||
|
||||
|
||||
### Drop index
|
||||
DELETE https://localhost:9200/tdrive_files_extended
|
||||
|
||||
@@ -11,6 +6,7 @@ DELETE https://localhost:9200/drive_files
|
||||
|
||||
### Create index
|
||||
PUT https://localhost:9200/tdrive_files_extended
|
||||
Authorization: Basic admin admin
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
### Drop index
|
||||
DELETE https://localhost:9200/tdrive_files_extended
|
||||
Authorization: Basic admin admin
|
||||
|
||||
### Delete Index
|
||||
DELETE https://localhost:9200/drive_files
|
||||
Authorization: Basic admin admin
|
||||
|
||||
### Create index
|
||||
PUT https://localhost:9200/tdrive_files_extended?pretty
|
||||
Authorization: Basic admin admin
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"aliases": {},
|
||||
"mappings": {
|
||||
"_source": {
|
||||
"enabled": true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
### Check that it is created
|
||||
GET https://localhost:9200/_cat/indices
|
||||
Authorization: Basic admin admin
|
||||
|
||||
### GET index info
|
||||
GET https://localhost:9200/drive_files
|
||||
Authorization: Basic admin admin
|
||||
|
||||
### GET data to the index
|
||||
GET https://localhost:9200/drive_files/_doc/8c03c5a1-0146-11ee-82d9-f503f8a58e9d
|
||||
Authorization: Basic admin admin
|
||||
Content-Type: application/json
|
||||
|
||||
### Put data to the index
|
||||
PUT https://localhost:9200/tdrive_files_extended/_doc/0001
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"file_name": "file0001.txt",
|
||||
"access_info": [
|
||||
{
|
||||
"type": "user",
|
||||
"id": "user01",
|
||||
"level": "read",
|
||||
"grantor": "user02"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
###
|
||||
PUT https://localhost:9200/tdrive_files_extended/_doc/8c03c5a1-0146-11ee-82d9-f503f8a58e9d
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"file_name": "file0002.png",
|
||||
"access_info": [
|
||||
{
|
||||
"type": "user",
|
||||
"id": "user01",
|
||||
"level": "read",
|
||||
"grantor": "user03"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
### Search all in the index by last_modified
|
||||
GET https://localhost:9200/drive_files/_search
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"query": {
|
||||
|
||||
"bool": {
|
||||
"boost": 1,
|
||||
"must": [
|
||||
{
|
||||
"range":
|
||||
{
|
||||
"last_modified": {
|
||||
"gte": "0"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"range":
|
||||
{
|
||||
"last_modified": {
|
||||
"lte": "1685958426348"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"match": {
|
||||
"access_entities": {
|
||||
"query": "edfb30e0-0385-11ee-80e8-41892804174d",
|
||||
"operator": "AND"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"access_entities": {
|
||||
"query": "edd447f1-0385-11ee-80e8-41892804174d",
|
||||
"operator": "AND"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"minimum_should_match": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"match": {
|
||||
"company_id": {
|
||||
"query": "edd447f1-0385-11ee-80e8-41892804174d",
|
||||
"operator": "AND"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"minimum_should_match": 1
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
### Search all in the index and order by name
|
||||
GET https://localhost:9200/drive_files/_search
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"boost": 1,
|
||||
"must": [
|
||||
{
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"match": {
|
||||
"access_entities": {
|
||||
"query": "12fb6a40-03b3-11ee-af31-a569970b7f74",
|
||||
"operator": "AND"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"access_entities": {
|
||||
"query": "12f290a1-03b3-11ee-af31-a569970b7f74",
|
||||
"operator": "AND"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"minimum_should_match": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"match": {
|
||||
"company_id": {
|
||||
"query": "12f290a1-03b3-11ee-af31-a569970b7f74",
|
||||
"operator": "AND"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"minimum_should_match": 1
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"sort": [
|
||||
{
|
||||
"name": "asc"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
### Search all in the index and order by name
|
||||
GET https://localhost:9200/drive_files/_search
|
||||
Content-Type: application/json
|
||||
|
||||
{
|
||||
"query": {
|
||||
"bool": {
|
||||
"boost": 1,
|
||||
"must": [
|
||||
{
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"match": {
|
||||
"access_entities": {
|
||||
"query": "4090dce0-03ba-11ee-97d7-fb8fa15d86a1",
|
||||
"operator": "AND"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"access_entities": {
|
||||
"query": "40808931-03ba-11ee-97d7-fb8fa15d86a1",
|
||||
"operator": "AND"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"minimum_should_match": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"bool": {
|
||||
"should": [
|
||||
{
|
||||
"match": {
|
||||
"company_id": {
|
||||
"query": "40808931-03ba-11ee-97d7-fb8fa15d86a1",
|
||||
"operator": "AND"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"minimum_should_match": 1
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user