diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts index 1f789ed3..5474270a 100644 --- a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all.ts @@ -1,311 +1,39 @@ import yargs from "yargs"; -import ora from "ora"; -import { TdrivePlatform } from "../../../core/platform/platform"; -import { DatabaseServiceAPI } from "../../../core/platform/services/database/api"; - -import User, { TYPE as UserTYPE } from "../../../services/user/entities/user"; -import { DriveFile, TYPE as DriveFileTYPE } from "../../../services/documents/entities/drive-file"; - -import Repository from "../../../core/platform/services/database/services/orm/repository/repository"; -import { - EntityTarget, - FindOptions, - SearchServiceAPI, -} from "../../../core/platform/services/search/api"; -import CompanyUser, { TYPE as CompanyUserTYPE } from "../../../services/user/entities/company_user"; +import type { TdrivePlatform } from "../../../core/platform/platform"; import runWithPlatform from "../../lib/run_with_platform"; -import SearchRepository from "src/core/platform/services/search/repository"; import parseYargsCommaSeparatedStringArray from "../../utils/yargs-comma-array"; -import waitTimeoutMS from "../../utils/wait-timeout"; -import iterateOverRepoPages from "../../utils/iterate-over-repository-pages"; -import globalResolver from "../../../services/global-resolver"; -import { couldGetKeywordsOfFile, getKeywordsOfFile } from "../../../services/documents/utils"; -type Options = { - spinner: ora.Ora; - repairEntities: boolean; - filterDocumentsByUserEMail: string[]; -}; - -const repairEntitiesArgumentDetails = []; - -/** This is an abstract base class for re-index cli commands; it stores runtime options - * into fields; runs repair first if request and re-indexes generically from the database - * to search services. */ -abstract class ReindexerCLICommand { - protected readonly database: DatabaseServiceAPI; - protected readonly search: SearchServiceAPI; - constructor( - protected readonly platform: TdrivePlatform, - protected readonly options: Options, - private readonly table: string, - private readonly entity: EntityTarget, - ) { - this.database = this.platform.getProvider("database"); - this.search = this.platform.getProvider("search"); - } - private _dbRepo: Repository; - protected async dbRepository() { - return (this._dbRepo = - this._dbRepo || (await this.database.getRepository(this.table, this.entity))); - } - private _searchRepo: SearchRepository; - protected async searchRepository() { - return (this._searchRepo = - this._searchRepo || this.search.getRepository(this.table, this.entity)); - } - - protected statusStart(info: string) { - this.options.spinner.start(`${this.table} > ${info}`); - } - protected statusSucceed(info: string) { - this.options.spinner.succeed(`${this.table} > ${info}`); - } - protected statusWarn(info: string) { - this.options.spinner.warn(`${this.table} > ${info}`); - } - protected statusFail(info: string) { - this.options.spinner.fail(`${this.table} > ${info}`); - } - protected statusInfo(info: string) { - this.options.spinner.info(`${this.table} > ${info}`); - } - - /** Override in sub-classes to translate a page from database to search entities. - * This is called by `reindexFromDBToSearch` so if that is overriden; this won't be called. - */ - protected async mapEntitiesToReIndexFromDBToSearch(entities: Entity[]): Promise { - return entities; - } - - /** Override in sub-classes that have filters (eg. from arguments) for the entities to re-index/repair */ - protected async prepareFindOptionsForItemsToReIndex(): Promise { - return undefined; - } - - /** Override in sub-classes that handle the --repairEntities argument. - * It is executed before reindexFromDBToSearch. What repair means depends - * on the entity type. - */ - protected async repairEntities(_findOptions: FindOptions): Promise { - this.statusWarn(`repairEntities > No repair action for ${this.table}`); - } - - /** Override in a sub-class to completely replace the re-indexing logic. - * - Iterates over pages of the entity from the database repository - * - calls `mapEntitiesToReIndexFromDBToSearch` to convert each page to a search entity - * - and upserts the result into the search repository - */ - protected async reindexFromDBToSearch(findOptions: FindOptions): Promise { - const repository = await this.dbRepository(); - this.statusStart("Start indexing..."); - let count = 0; - await iterateOverRepoPages( - repository, - async entities => { - entities = await this.mapEntitiesToReIndexFromDBToSearch(entities); - await this.search.upsert(entities); - count += entities.length; - this.statusStart(`Indexed ${count} ${this.table}...`); - }, - findOptions, - ); - if (count === 0) this.statusWarn(`Index ${this.table} finished; but 0 items included`); - else this.statusSucceed(`${count} ${this.table} indexed`); - const giveFlushAChanceDurationMS = 10000; - this.statusStart(`Emptying flush (${giveFlushAChanceDurationMS / 1000}s)...`); - await waitTimeoutMS(giveFlushAChanceDurationMS); - this.statusSucceed("Done!"); - } - - /** Run both operations: repair if requested and re-index */ - public async run(): Promise { - const findOptions = await this.prepareFindOptionsForItemsToReIndex(); - if (this.options.repairEntities) await this.repairEntities(findOptions); - await this.reindexFromDBToSearch(findOptions); - } -} +import type BaseReindexer from "./index-all/base-reindexer"; +import type ReindexerOptions from "./index-all/reindexer-options"; /** Serves as an index of classes for the repos to reindex; to specialise bits of behaviour and what not */ const RepositoryNameToCTOR = new Map< string, - (platform: TdrivePlatform, options: Options) => ReindexerCLICommand + { + ctor: (platform: TdrivePlatform, options: ReindexerOptions) => BaseReindexer; + repairDescription: string; + } >(); -class UserReindexerCLICommand extends ReindexerCLICommand { - constructor(platform: TdrivePlatform, options: Options) { - super(platform, options, UserTYPE, User); - } +import UsersReindexer from "./index-all/users-reindexer"; +RepositoryNameToCTOR.set("users", { + ctor: (platform, options) => new UsersReindexer(platform, options), + repairDescription: UsersReindexer.repairActionDescription, +}); - protected override async repairEntities(_findOptions: FindOptions): Promise { - this.statusStart("repairEntities > Adding companies to cache of user"); - const companiesUsersRepository = await this.database.getRepository( - CompanyUserTYPE, - CompanyUser, - ); - let count = 0; - const repository = await this.dbRepository(); - await iterateOverRepoPages(repository, async entities => { - for (const user of entities) { - const companies = await companiesUsersRepository.find({ user_id: user.id }, {}, undefined); - const prevCache = JSON.stringify(user.cache); - user.cache ||= { companies: [] }; - user.cache.companies = companies.getEntities().map(company => company.group_id); - const newCache = JSON.stringify(user.cache); - if (prevCache != newCache) await repository.save(user, undefined); - } - count += entities.length; - this.statusStart(`repairEntities > Adding companies to cache of ${count} users...`); - }); - this.statusSucceed(`repairEntities > Added companies to cache of ${count} users`); - } -} -RepositoryNameToCTOR.set( - "users", - (platform, options) => new UserReindexerCLICommand(platform, options), -); -repairEntitiesArgumentDetails.push( - "users: Rebuild cache.companies and save to database if changed", -); - -class DocumentsReindexerCLICommand extends ReindexerCLICommand { - constructor(platform: TdrivePlatform, options: Options) { - super(platform, options, DriveFileTYPE, DriveFile); - } - protected override async prepareFindOptionsForItemsToReIndex(): Promise { - if (this.options.filterDocumentsByUserEMail.length === 0) - throw new Error( - "No creator e-mail specified. This would affect all documents. To do this on purpose, add '--filterDocumentsByUserEMail all'", - ); - if (this.options.filterDocumentsByUserEMail.join("-") === "all") { - this.statusWarn("All users included in document re-index"); - return {}; - } - if (this.options.filterDocumentsByUserEMail.indexOf("all") > -1) - throw new Error( - "If specifying 'all' to include all users, don't include another email argument.", - ); - const userRepo = await this.database.getRepository(UserTYPE, User); - const emailsToIds = new Map(); - const rawUserIds = []; - this.options.filterDocumentsByUserEMail.forEach(email => { - if (email.match(/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i)) rawUserIds.push(email); - else emailsToIds.set(email, false); - }); - await iterateOverRepoPages( - userRepo, - async entities => { - entities.forEach(({ id, email_canonical }) => emailsToIds.set(email_canonical, id)); - }, - { $in: [["email_canonical", this.options.filterDocumentsByUserEMail]] }, - ); - const failedToFindEmails = []; - for (const [email, id] of emailsToIds) if (id === false) failedToFindEmails.push(email); - if (failedToFindEmails.length > 0) - throw new Error("No user id found for email(s): " + JSON.stringify(failedToFindEmails)); - this.statusInfo("Documents of user(s):"); - for (const [email, id] of emailsToIds) this.statusInfo(`\t- ${email}: ${id}`); - for (const id of rawUserIds) this.statusInfo(`\t- ${id}`); - return { $in: [["creator", [...rawUserIds, ...emailsToIds.values()]]] }; - } - protected override async repairEntities(findOptions: FindOptions): Promise { - /* - //Todo: - - [ ] Batch saves - - [ ] Refactor repair into mapping - - [ ] Check save does upsert - - [ ] Remove logs - - [ ] Document this: - For each item in DB - Download - - [ ] Decide if download fails ? Flag file ? Delete entity ? - If keywords changed - Save to DB - - [ ] Delete file - */ - - const repository = await this.dbRepository(); - await iterateOverRepoPages( - repository, - async entities => { - for (const entity of entities) { - if (entity.is_directory) - // || entity.is_in_trash) // why not index trash too after all - continue; - this.statusInfo( - `-> Downloading ${entity.name} (${entity.size}b id: ${entity.id} creator: ${entity.creator}`, - ); - try { - if ( - entity.size === 0 || - !couldGetKeywordsOfFile( - entity.last_version_cache.file_metadata.mime, - entity.last_version_cache.file_metadata.name || entity.name, - ) - ) { - this.statusInfo("\tSkipping, 0 size or couldGetKeywordsOfFile returned nope"); - continue; - } - const storedFile = await globalResolver.services.files.download( - entity.last_version_cache.file_metadata.external_id, - { company: { id: entity.company_id }, user: null }, - ); - const content_keywords = await getKeywordsOfFile( - storedFile.mime, - storedFile.name, - storedFile.file, - ); - if (storedFile.size != entity.size) - this.statusWarn( - `Warning, file ${entity.id} (${entity.name}) has size ${entity.size} in DB but ${storedFile.size} when downloaded`, - ); - if (content_keywords !== entity.content_keywords) { - entity.content_keywords = content_keywords; - repository.save(entity); - } - } catch (err) { - this.statusFail(err.stack); - } - } - }, - findOptions, - ); - } -} -RepositoryNameToCTOR.set( - "documents", - (platform, options) => new DocumentsReindexerCLICommand(platform, options), -); -repairEntitiesArgumentDetails.push( - "documents: Download and re-extract keywords before re-indexing", -); -class _BufferedAction { - private buffer: Entity[]; - constructor( - private readonly batchSize: number, - private readonly action: (entities: Entity[]) => Promise, - ) {} - public async flush() { - const buffer = this.buffer; - if (!buffer.length) return; - this.buffer = new Array(this.batchSize); - return this.action(buffer); - } - public async save(entity: Entity) { - this.buffer.push(entity); - if (this.buffer.length >= this.batchSize) await this.flush(); - } -} -import { logger } from "../../../core/platform/framework/logger"; +import DocumentsReindexer from "./index-all/documents-reindexer"; +RepositoryNameToCTOR.set("documents", { + ctor: (platform, options) => new DocumentsReindexer(platform, options), + repairDescription: DocumentsReindexer.repairActionDescription, +}); const reindexingArgumentGroupTitle = "Re-indexing options"; -const repositoryArgumentName = "repository"; const command: yargs.CommandModule = { command: "index", describe: "command to reindex search middleware from db entities", builder: { - [repositoryArgumentName]: { + repository: { type: "string", description: "Repository to re-index.", choices: [...RepositoryNameToCTOR.keys()], @@ -315,9 +43,12 @@ const command: yargs.CommandModule = { repairEntities: { default: false, type: "boolean", - description: ["Repair entities too when possible", ...repairEntitiesArgumentDetails].join( - "\n- ", - ), + description: [ + "Repair entities too when possible", + ...[...RepositoryNameToCTOR.keys()] + .sort() + .map(name => `${name}: ${RepositoryNameToCTOR.get(name).repairDescription}`), + ].join("\n- "), group: reindexingArgumentGroupTitle, }, filterDocumentsByUserEMail: { @@ -330,20 +61,21 @@ const command: yargs.CommandModule = { }, handler: async argv => { const repositories = parseYargsCommaSeparatedStringArray( - argv[repositoryArgumentName] as string /* ignore typechecker */, + argv.repository as string /* ignore typechecker */, ); const filterDocumentsByUserEMail = parseYargsCommaSeparatedStringArray( argv.filterDocumentsByUserEMail as string /* ignore typechecker */, ); await runWithPlatform("Re-index", async ({ spinner, platform }) => { try { - logger.level = "debug"; for (const repositoryName of repositories) - await RepositoryNameToCTOR.get(repositoryName)(platform, { - spinner, - repairEntities: !!argv.repairEntities, - filterDocumentsByUserEMail, - }).run(); + await RepositoryNameToCTOR.get(repositoryName) + .ctor(platform, { + spinner, + repairEntities: !!argv.repairEntities, + filterDocumentsByUserEMail, + }) + .run(); } catch (err) { spinner.fail(err.stack || err); return 1; diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/base-reindexer.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/base-reindexer.ts new file mode 100644 index 00000000..0ac1aff4 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/base-reindexer.ts @@ -0,0 +1,110 @@ +import type { DatabaseServiceAPI } from "../../../../core/platform/services/database/api"; +import type { TdrivePlatform } from "../../../../core/platform/platform"; +import type Repository from "../../../../core/platform/services/database/services/orm/repository/repository"; +import type SearchRepository from "../../../../core/platform/services/search/repository"; +import type { + EntityTarget, + FindOptions, + SearchServiceAPI, +} from "../../../../core/platform/services/search/api"; + +import waitTimeoutMS from "../../../utils/wait-timeout"; +import iterateOverRepoPages from "../../../utils/iterate-over-repository-pages"; +import type ReindexerOptions from "./reindexer-options"; + +/** This is an abstract base class for re-index cli commands; it stores runtime options + * into fields; runs repair first if requested and re-indexes generically from the database + * to search services. */ +export default abstract class BaseReindexer { + protected readonly database: DatabaseServiceAPI; + protected readonly search: SearchServiceAPI; + constructor( + protected readonly platform: TdrivePlatform, + protected readonly options: ReindexerOptions, + private readonly table: string, + private readonly entity: EntityTarget, + ) { + this.database = this.platform.getProvider("database"); + this.search = this.platform.getProvider("search"); + } + private _dbRepo: Repository; + protected async dbRepository() { + return (this._dbRepo = + this._dbRepo || (await this.database.getRepository(this.table, this.entity))); + } + private _searchRepo: SearchRepository; + protected async searchRepository() { + return (this._searchRepo = + this._searchRepo || this.search.getRepository(this.table, this.entity)); + } + + protected statusStart(info: string) { + this.options.spinner.start(`${this.table} > ${info}`); + } + protected statusSucceed(info: string) { + this.options.spinner.succeed(`${this.table} > ${info}`); + } + protected statusWarn(info: string) { + this.options.spinner.warn(`${this.table} > ${info}`); + } + protected statusFail(info: string) { + this.options.spinner.fail(`${this.table} > ${info}`); + } + protected statusInfo(info: string) { + this.options.spinner.info(`${this.table} > ${info}`); + } + + /** Override in sub-classes to translate a page from database to search entities. + * This is called by `reindexFromDBToSearch` so if that is overriden; this won't be called. + */ + protected async mapEntitiesToReIndexFromDBToSearch(entities: Entity[]): Promise { + return entities; + } + + /** Override in sub-classes that have filters (eg. from arguments) for the entities to re-index/repair */ + protected async prepareFindOptionsForItemsToReIndex(): Promise { + return undefined; + } + + /** Override in sub-classes that handle the --repairEntities argument. + * It is executed before reindexFromDBToSearch. What repair means depends + * on the entity type. + */ + protected async repairEntities(_findOptions: FindOptions): Promise { + this.statusWarn(`repairEntities > No repair action for ${this.table}`); + } + + /** Override in a sub-class to completely replace the re-indexing logic. + * - Iterates over pages of the entity from the database repository + * - calls `mapEntitiesToReIndexFromDBToSearch` to convert each page to a search entity + * - and upserts the result into the search repository + */ + protected async reindexFromDBToSearch(findOptions: FindOptions): Promise { + const repository = await this.dbRepository(); + this.statusStart("Start indexing..."); + let count = 0; + await iterateOverRepoPages( + repository, + async entities => { + entities = await this.mapEntitiesToReIndexFromDBToSearch(entities); + await this.search.upsert(entities); + count += entities.length; + this.statusStart(`Indexed ${count} ${this.table}...`); + }, + findOptions, + ); + if (count === 0) this.statusWarn(`Index ${this.table} finished; but 0 items included`); + else this.statusSucceed(`${count} ${this.table} indexed`); + const giveFlushAChanceDurationMS = 10000; + this.statusStart(`Emptying flush (${giveFlushAChanceDurationMS / 1000}s)...`); + await waitTimeoutMS(giveFlushAChanceDurationMS); + this.statusSucceed("Done!"); + } + + /** Run both operations: repair if requested and re-index */ + public async run(): Promise { + const findOptions = await this.prepareFindOptionsForItemsToReIndex(); + if (this.options.repairEntities) await this.repairEntities(findOptions); + await this.reindexFromDBToSearch(findOptions); + } +} diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/documents-reindexer.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/documents-reindexer.ts new file mode 100644 index 00000000..d071fe0f --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/documents-reindexer.ts @@ -0,0 +1,109 @@ +import type { TdrivePlatform } from "../../../../core/platform/platform"; +import type { FindOptions } from "../../../../core/platform/services/search/api"; +import globalResolver from "../../../../services/global-resolver"; + +import { + DriveFile, + TYPE as DriveFileTYPE, +} from "../../../../services/documents/entities/drive-file"; +import User, { TYPE as UserTYPE } from "../../../../services/user/entities/user"; + +import { couldGetKeywordsOfFile, getKeywordsOfFile } from "../../../../services/documents/utils"; +import iterateOverRepoPages from "../../../utils/iterate-over-repository-pages"; + +import BaseReindexer from "./base-reindexer"; +import type ReindexerOptions from "./reindexer-options"; + +export default class DocumentsReindexer extends BaseReindexer { + constructor(platform: TdrivePlatform, options: ReindexerOptions) { + super(platform, options, DriveFileTYPE, DriveFile); + } + + protected override async prepareFindOptionsForItemsToReIndex(): Promise { + if (this.options.filterDocumentsByUserEMail.length === 0) + throw new Error( + "No creator e-mail specified. This would affect all documents. To do this on purpose, add '--filterDocumentsByUserEMail all'", + ); + if (this.options.filterDocumentsByUserEMail.join("-") === "all") { + this.statusWarn("All users included in document re-index"); + return {}; + } + if (this.options.filterDocumentsByUserEMail.indexOf("all") > -1) + throw new Error( + "If specifying 'all' to include all users, don't include another email argument.", + ); + const userRepo = await this.database.getRepository(UserTYPE, User); + const emailsToIds = new Map(); + const rawUserIds = []; + this.options.filterDocumentsByUserEMail.forEach(email => { + if (email.match(/^[a-f0-9]{8}(?:-[a-f0-9]{4}){3}-[a-f0-9]{12}$/i)) rawUserIds.push(email); + else emailsToIds.set(email, false); + }); + await iterateOverRepoPages( + userRepo, + async entities => { + entities.forEach(({ id, email_canonical }) => emailsToIds.set(email_canonical, id)); + }, + { $in: [["email_canonical", this.options.filterDocumentsByUserEMail]] }, + ); + const failedToFindEmails = []; + for (const [email, id] of emailsToIds) if (id === false) failedToFindEmails.push(email); + if (failedToFindEmails.length > 0) + throw new Error("No user id found for email(s): " + JSON.stringify(failedToFindEmails)); + this.statusInfo("Documents of user(s):"); + for (const [email, id] of emailsToIds) this.statusInfo(`\t- ${email}: ${id}`); + for (const id of rawUserIds) this.statusInfo(`\t- ${id}`); + return { $in: [["creator", [...rawUserIds, ...emailsToIds.values()]]] }; + } + + public static readonly repairActionDescription = + "Download and re-extract keywords before re-indexing"; + protected override async repairEntities(findOptions: FindOptions): Promise { + const repository = await this.dbRepository(); + await iterateOverRepoPages( + repository, + async entities => { + for (const entity of entities) { + if (entity.is_directory) + // || entity.is_in_trash) // why not index trash too after all + continue; + this.statusInfo( + `-> Downloading ${entity.name} (${entity.size}b id: ${entity.id} creator: ${entity.creator}`, + ); + try { + if ( + entity.size === 0 || + !couldGetKeywordsOfFile( + entity.last_version_cache.file_metadata.mime, + entity.last_version_cache.file_metadata.name || entity.name, + ) + ) { + this.statusInfo("\tSkipping, 0 size or couldGetKeywordsOfFile returned nope"); + continue; + } + const storedFile = await globalResolver.services.files.download( + entity.last_version_cache.file_metadata.external_id, + { company: { id: entity.company_id }, user: null }, + ); + const content_keywords = await getKeywordsOfFile( + storedFile.mime, + storedFile.name, + storedFile.file, + ); + if (storedFile.size != entity.size) + this.statusWarn( + `Warning, file ${entity.id} (${entity.name}) has size ${entity.size} in DB but ${storedFile.size} when downloaded`, + ); + if (content_keywords !== entity.content_keywords) { + entity.content_keywords = content_keywords; + repository.save(entity); + } + } catch (err) { + this.statusFail(err.stack); + } + } + }, + findOptions, + ); + } +} diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/reindexer-options.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/reindexer-options.ts new file mode 100644 index 00000000..d847f574 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/reindexer-options.ts @@ -0,0 +1,7 @@ +import type ora from "ora"; + +export default interface ReindexerOptions { + spinner: ora.Ora; + repairEntities: boolean; + filterDocumentsByUserEMail: string[]; +} diff --git a/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/users-reindexer.ts b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/users-reindexer.ts new file mode 100644 index 00000000..c6b07aa9 --- /dev/null +++ b/tdrive/backend/node/src/cli/cmds/search_cmds/index-all/users-reindexer.ts @@ -0,0 +1,43 @@ +import type { TdrivePlatform } from "../../../../core/platform/platform"; +import type { FindOptions } from "../../../../core/platform/services/search/api"; + +import User, { TYPE as UserTYPE } from "../../../../services/user/entities/user"; +import CompanyUser, { + TYPE as CompanyUserTYPE, +} from "../../../../services/user/entities/company_user"; + +import iterateOverRepoPages from "../../../utils/iterate-over-repository-pages"; + +import BaseReindexer from "./base-reindexer"; +import type ReindexerOptions from "./reindexer-options"; + +export default class UsersReindexer extends BaseReindexer { + constructor(platform: TdrivePlatform, options: ReindexerOptions) { + super(platform, options, UserTYPE, User); + } + + public static readonly repairActionDescription = + "Rebuild cache.companies and save to database if changed"; + protected override async repairEntities(_findOptions: FindOptions): Promise { + this.statusStart("repairEntities > Adding companies to cache of user"); + const companiesUsersRepository = await this.database.getRepository( + CompanyUserTYPE, + CompanyUser, + ); + let count = 0; + const repository = await this.dbRepository(); + await iterateOverRepoPages(repository, async entities => { + for (const user of entities) { + const companies = await companiesUsersRepository.find({ user_id: user.id }, {}, undefined); + const prevCache = JSON.stringify(user.cache); + user.cache ||= { companies: [] }; + user.cache.companies = companies.getEntities().map(company => company.group_id); + const newCache = JSON.stringify(user.cache); + if (prevCache != newCache) await repository.save(user, undefined); + } + count += entities.length; + this.statusStart(`repairEntities > Adding companies to cache of ${count} users...`); + }); + this.statusSucceed(`repairEntities > Added companies to cache of ${count} users`); + } +}