✨ back cli: add user dump and list commands
This commit is contained in:
Eric Doughty-Papassideris
committed by
Anton Shepilov
Anton Shepilov
parent
eec23b230b
commit
8efcb2546c
@@ -0,0 +1,116 @@
|
||||
import yargs from "yargs";
|
||||
|
||||
import runWithPlatform from "../../lib/run-with-platform";
|
||||
import gr from "../../../services/global-resolver";
|
||||
import { messageQueueLogger, platformLogger } from "../../../core/platform/framework";
|
||||
import {
|
||||
buildUserDeletionRepositories,
|
||||
descendDriveItemsDepthFirstRandomOrder,
|
||||
loadRawVersionsOfItemForDeletion,
|
||||
type TUserDeletionRepos,
|
||||
} from "../../../core/platform/services/admin/utils";
|
||||
import type User from "../../../services/user/entities/user";
|
||||
import { EntityByKind } from "../../utils/print-entities";
|
||||
|
||||
let globalArgv: {
|
||||
verbose: boolean;
|
||||
|
||||
user_id: string;
|
||||
json_output: boolean;
|
||||
with_storage_paths: boolean;
|
||||
|
||||
user?: User; // Not an argument per say, hydrated in handler
|
||||
};
|
||||
|
||||
const lengthOfLongestEntityKind = Object.keys(EntityByKind).reduce(
|
||||
(acc, kind) => Math.max(acc, kind.length),
|
||||
0,
|
||||
);
|
||||
|
||||
const entityToHeaderString = <K extends keyof typeof EntityByKind>(
|
||||
kind: K & string,
|
||||
entity: Parameters<(typeof EntityByKind)[K]["headerOf"]>[0],
|
||||
depth: number = 0,
|
||||
): string => {
|
||||
if (globalArgv.json_output) return kind + " " + JSON.stringify(entity); // Can't really print out json directly, valid json seems to get prettified by pino
|
||||
// ts doesn't seem to infer entityHeaderPrinter[kind] correctly even with help
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return `${kind.padEnd(lengthOfLongestEntityKind)} ${entity.id} ${new Array(depth + 1).join(
|
||||
" ",
|
||||
)}${EntityByKind[kind].headerOf(entity as any)}`;
|
||||
};
|
||||
|
||||
async function dumpAccount(repos: TUserDeletionRepos): Promise<number> {
|
||||
console.log(entityToHeaderString("user", globalArgv.user));
|
||||
await descendDriveItemsDepthFirstRandomOrder(
|
||||
repos,
|
||||
"user_" + globalArgv.user.id,
|
||||
async (item, _children, parents) => {
|
||||
const depth = parents.length + 1;
|
||||
if (!item.is_directory) {
|
||||
const versionsAssets = await loadRawVersionsOfItemForDeletion(
|
||||
repos,
|
||||
item,
|
||||
globalArgv.with_storage_paths,
|
||||
);
|
||||
for (const { version, file, paths } of versionsAssets) {
|
||||
const outputPath = (path: string) =>
|
||||
console.log(entityToHeaderString("s3", { id: file.id, path }, depth + 2));
|
||||
if (globalArgv.json_output) {
|
||||
for (const path of paths || []) outputPath(path);
|
||||
} else if (paths) {
|
||||
paths.sort((a, b) => a.localeCompare(b));
|
||||
if (paths.length > 0) outputPath(paths[0]);
|
||||
if (paths.length == 3) outputPath(paths[1]);
|
||||
else if (paths.length > 3) outputPath(`... skipping ${paths.length - 2} paths ...`);
|
||||
if (paths.length > 1) outputPath(paths[paths.length - 1]);
|
||||
}
|
||||
file && console.log(entityToHeaderString("file", file, depth + 2));
|
||||
version && console.log(entityToHeaderString("version", version, depth + 1));
|
||||
}
|
||||
}
|
||||
console.log(entityToHeaderString("item", item, depth));
|
||||
},
|
||||
);
|
||||
return 0;
|
||||
}
|
||||
|
||||
export default {
|
||||
command: "dump <user_id>",
|
||||
describe: "Output a list of entities related to a user",
|
||||
builder: {
|
||||
user_id: {
|
||||
demandOption: true,
|
||||
type: "string",
|
||||
describe:
|
||||
"id of the user, or a search string for the email.\n(if the id doesn't match, will print the matching users and exit)",
|
||||
},
|
||||
json_output: {
|
||||
type: "boolean",
|
||||
alias: "j",
|
||||
describe: "Output in lines of '[kind] [json_object]'.",
|
||||
},
|
||||
with_storage_paths: {
|
||||
type: "boolean",
|
||||
alias: "p",
|
||||
describe: "If set, also query the storage to include all relevant paths in the output",
|
||||
},
|
||||
},
|
||||
handler: async (argv: typeof globalArgv) => {
|
||||
globalArgv = argv;
|
||||
if (!argv.verbose) {
|
||||
platformLogger.level = "warn";
|
||||
messageQueueLogger.level = "warn";
|
||||
}
|
||||
await runWithPlatform("dump_account", async ({ spinner: _spinner, platform: _platform }) => {
|
||||
const repos = await buildUserDeletionRepositories(gr.database, gr.platformServices.search);
|
||||
argv.user = await repos.user.findOne({ id: argv.user_id });
|
||||
if (!argv.user) {
|
||||
console.error(`Error, unknown user ${JSON.stringify(argv.user_id)}`);
|
||||
return 1;
|
||||
}
|
||||
await dumpAccount(repos);
|
||||
return 0;
|
||||
});
|
||||
},
|
||||
} as yargs.CommandModule<object, typeof globalArgv>;
|
||||
@@ -0,0 +1,69 @@
|
||||
import yargs from "yargs";
|
||||
|
||||
import runWithPlatform from "../../lib/run-with-platform";
|
||||
import { messageQueueLogger, platformLogger } from "../../../core/platform/framework";
|
||||
import { buildUserDeletionRepositories } from "../../../core/platform/services/admin/utils";
|
||||
import type User from "../../../services/user/entities/user";
|
||||
import { EntityByKind } from "../../utils/print-entities";
|
||||
|
||||
let globalArgv: {
|
||||
verbose: boolean;
|
||||
|
||||
email_like: string;
|
||||
json_output: boolean;
|
||||
|
||||
user?: User; // Not an argument per say, hydrated in handler
|
||||
};
|
||||
|
||||
const entityToHeaderString = <K extends keyof typeof EntityByKind>(
|
||||
kind: K & string,
|
||||
entity: Parameters<(typeof EntityByKind)[K]["headerOf"]>[0],
|
||||
): string => {
|
||||
if (globalArgv.json_output) return kind + " " + JSON.stringify(entity); // Can't really print out json directly, valid json seems to get prettified by pino
|
||||
// ts doesn't seem to infer entityHeaderPrinter[kind] correctly even with help
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return `${entity.id} ${EntityByKind[kind].headerOf(entity as any)}`;
|
||||
};
|
||||
|
||||
export default {
|
||||
command: "list",
|
||||
describe: "list users",
|
||||
builder: {
|
||||
email_like: {
|
||||
alias: "e",
|
||||
type: "string",
|
||||
describe:
|
||||
"id of the user, or a search string for the email.\n(if the id doesn't match, will print the matching users and exit)",
|
||||
},
|
||||
json_output: {
|
||||
type: "boolean",
|
||||
alias: "j",
|
||||
describe: "Output in lines of '[kind] [json_object]'.",
|
||||
},
|
||||
},
|
||||
handler: async (argv: typeof globalArgv) => {
|
||||
globalArgv = argv;
|
||||
if (!argv.verbose) {
|
||||
platformLogger.level = "warn";
|
||||
messageQueueLogger.level = "warn";
|
||||
}
|
||||
await runWithPlatform("dump_account", async ({ spinner: _spinner, platform: _platform }) => {
|
||||
const repos = await buildUserDeletionRepositories();
|
||||
console.log(
|
||||
(
|
||||
await repos.user.find(
|
||||
{},
|
||||
{
|
||||
...(argv.email_like ? { $like: [["email_canonical", argv.email_like]] } : {}),
|
||||
sort: { email_canonical: "asc" },
|
||||
},
|
||||
)
|
||||
)
|
||||
.getEntities()
|
||||
.map(user => entityToHeaderString("user", user))
|
||||
.join("\n"),
|
||||
);
|
||||
return 0;
|
||||
});
|
||||
},
|
||||
} as yargs.CommandModule<object, typeof globalArgv>;
|
||||
@@ -35,6 +35,8 @@ export default async function runWithPlatform(
|
||||
spinner.fail(err.stack || err);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
// Spinner seems to interrupt still buffered output otherwise
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
spinner.start("Platform: shutting down...");
|
||||
await platform.stop();
|
||||
spinner.succeed("Platform: shutdown");
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
import type { EntityTarget } from "../../core/platform/services/search/api";
|
||||
import gr from "../../services/global-resolver";
|
||||
|
||||
import User, { TYPE as UserTYPE } from "../../services/user/entities/user";
|
||||
import { DriveFile, TYPE as DriveFileTYPE } from "../../services/documents/entities/drive-file";
|
||||
import { File } from "../../services/files/entities/file";
|
||||
import {
|
||||
FileVersion,
|
||||
TYPE as FileVersionTYPE,
|
||||
} from "../../services/documents/entities/file-version";
|
||||
import CompanyUser, { TYPE as CompanyUserTYPE } from "../../services/user/entities/company_user";
|
||||
import {
|
||||
MissedDriveFile,
|
||||
TYPE as MissedDriveFileTYPE,
|
||||
} from "../../services/documents/entities/missed-drive-file";
|
||||
import Company, { TYPE as CompanyTYPE } from "../../services/user/entities/company";
|
||||
|
||||
const FileTYPE = "files";
|
||||
|
||||
/** Fake entity to represent S3 path entries as similar resources */
|
||||
interface S3Object {
|
||||
/** id of the File */
|
||||
id: string;
|
||||
/** path in bucket of the file */
|
||||
path: string;
|
||||
}
|
||||
|
||||
const epochToISO = epoch => (epoch ? new Date(epoch).toISOString() : "<null date>");
|
||||
export const EntityKindToString = {
|
||||
user: (entity: User): string => `🤺 ${entity.email_canonical}`,
|
||||
// extUser: (entity: ExternalUser): string => `🤺 ${entity.user_id} <-> 🌍 ${entity.external_id}`,
|
||||
compUser: (entity: CompanyUser): string => `🤺 ${entity.user_id} <-> 🏦 ${entity.group_id}`,
|
||||
// session: (entity: Session): string => `🤺 ${entity.sub} <-> 🏦 ${entity.company_id}`,
|
||||
company: (entity: Company): string => `🏦 ${entity.name || entity.displayName}`,
|
||||
item: (entity: DriveFile): string =>
|
||||
`${entity.is_in_trash ? "🗑️ " : ""}${entity.is_directory ? "📁" : "📄"}${
|
||||
entity.scope != "personal" ? "🏢 " : ""
|
||||
} ${entity.name}`,
|
||||
version: (entity: FileVersion): string =>
|
||||
`🕰️ ${epochToISO(entity.date_added)} - ${entity.file_size}`,
|
||||
file: (entity: File): string => `💿 ${epochToISO(entity.updated_at)}`,
|
||||
s3: (entity: S3Object): string => `☁️ ${entity.path}`,
|
||||
missed: (entity: MissedDriveFile): string =>
|
||||
`😭 ${entity.is_in_trash ? "🗑️ " : ""} ${entity.name}`,
|
||||
};
|
||||
|
||||
const makeEntityKind = <T>(
|
||||
kind: keyof typeof EntityKindToString,
|
||||
db?: { entity: EntityTarget<T>; type: string },
|
||||
) => {
|
||||
const headerToString = EntityKindToString[kind];
|
||||
return {
|
||||
kind,
|
||||
headerOf: headerToString,
|
||||
db: db
|
||||
? {
|
||||
getRepository: () => gr.database.getRepository<T>(db.type, db.entity),
|
||||
...db,
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
};
|
||||
|
||||
export type TEntityNames = keyof typeof Entities;
|
||||
export const Entities = {
|
||||
User: makeEntityKind<User>("user", { entity: User, type: UserTYPE }),
|
||||
DriveFile: makeEntityKind<DriveFile>("item", { entity: DriveFile, type: DriveFileTYPE }),
|
||||
File: makeEntityKind<File>("file", { entity: File, type: FileTYPE }),
|
||||
FileVersion: makeEntityKind<FileVersion>("version", {
|
||||
entity: FileVersion,
|
||||
type: FileVersionTYPE,
|
||||
}),
|
||||
// ExternalUser: makeEntityKind<ExternalUser>("extUser", {
|
||||
// entity: ExternalUser,
|
||||
// type: ExternalUserTYPE,
|
||||
// }),
|
||||
CompanyUser: makeEntityKind<CompanyUser>("compUser", {
|
||||
entity: CompanyUser,
|
||||
type: CompanyUserTYPE,
|
||||
}),
|
||||
MissedDriveFile: makeEntityKind<MissedDriveFile>("missed", {
|
||||
entity: MissedDriveFile,
|
||||
type: MissedDriveFileTYPE,
|
||||
}),
|
||||
Company: makeEntityKind<Company>("company", { entity: Company, type: CompanyTYPE }),
|
||||
// Session: makeEntityKind<Session>("session", { entity: Session, type: SessionTYPE }),
|
||||
$S3: makeEntityKind<S3Object>("s3"),
|
||||
};
|
||||
|
||||
export const EntityByKind = Object.fromEntries(
|
||||
Object.entries(Entities).map(([_, def]) => [def.kind, def]),
|
||||
);
|
||||
@@ -0,0 +1,150 @@
|
||||
import { getLogger } from "../../framework";
|
||||
|
||||
import type { DatabaseServiceAPI } from "../database/api";
|
||||
import type { SearchServiceAPI } from "../search/api";
|
||||
|
||||
import gr from "../../../../services/global-resolver";
|
||||
import type Repository from "../database/services/orm/repository/repository";
|
||||
|
||||
import User, { TYPE as UserTYPE } from "../../../../services/user/entities/user";
|
||||
import {
|
||||
DriveFile,
|
||||
TYPE as DriveFileTYPE,
|
||||
} from "../../../../services/documents/entities/drive-file";
|
||||
import { File } from "../../../../services/files/entities/file";
|
||||
import {
|
||||
FileVersion,
|
||||
TYPE as FileVersionTYPE,
|
||||
} from "../../../../services/documents/entities/file-version";
|
||||
import ExternalUser, {
|
||||
TYPE as ExternalUserTYPE,
|
||||
} from "../../../../services/user/entities/external_user";
|
||||
import CompanyUser, {
|
||||
TYPE as CompanyUserType,
|
||||
} from "../../../../services/user/entities/company_user";
|
||||
import {
|
||||
MissedDriveFile,
|
||||
TYPE as MissedDriveFileTYPE,
|
||||
} from "../../../../services/documents/entities/missed-drive-file";
|
||||
import Session from "../../../../services/console/entities/session";
|
||||
import { fisherYattesShuffleInPlace } from "../../../../utils/arrays";
|
||||
import { getFilePath } from "../../../../services/files/services";
|
||||
import Company, { TYPE as CompanyType } from "../../../../services/user/entities/company";
|
||||
|
||||
const FileTYPE = "files";
|
||||
|
||||
export const adminLogger = getLogger("Admin");
|
||||
|
||||
/** Run the `map` function on sets of `batchSize` in `list`, sequentially between batches */
|
||||
async function runInBatches<T, R>(
|
||||
batchSize: number,
|
||||
list: readonly T[],
|
||||
map: (items: T[]) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const listCopy = [...list];
|
||||
let batch;
|
||||
const result = [] as R[];
|
||||
while ((batch = listCopy.splice(0, batchSize)).length) {
|
||||
console.error("batch", batch);
|
||||
result.push(await map(batch));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Call {@see runInBatches} expecting a boolean result from `map`, return `true` only if empty or all batches returned true */
|
||||
export async function runInBatchesAreAllTrue<T>(
|
||||
batchSize: number,
|
||||
list: T[],
|
||||
map: (items: T[]) => Promise<boolean[]>,
|
||||
): Promise<boolean> {
|
||||
return (
|
||||
await runInBatches(batchSize, list, async results => (await map(results)).every(x => !!x))
|
||||
).every(x => !!x);
|
||||
}
|
||||
export type TUserDeletionRepos = Awaited<ReturnType<typeof buildUserDeletionRepositories>>;
|
||||
|
||||
/**
|
||||
* Create all repositories required for deleting a user
|
||||
* @deprecated Do not use this outside of this file, it is exported exclusively for e2e tests
|
||||
*/
|
||||
export async function buildUserDeletionRepositories(
|
||||
db: DatabaseServiceAPI = gr.database,
|
||||
search: SearchServiceAPI = gr.platformServices.search,
|
||||
) {
|
||||
return {
|
||||
driveFile: await db.getRepository<DriveFile>(DriveFileTYPE, DriveFile),
|
||||
file: await db.getRepository<File>(FileTYPE, File),
|
||||
fileVersion: await db.getRepository<FileVersion>(FileVersionTYPE, FileVersion),
|
||||
missedDriveFile: await db.getRepository<MissedDriveFile>(MissedDriveFileTYPE, MissedDriveFile),
|
||||
user: await db.getRepository<User>(UserTYPE, User),
|
||||
companyUser: await db.getRepository<CompanyUser>(CompanyUserType, CompanyUser),
|
||||
externalUser: await db.getRepository<ExternalUser>(ExternalUserTYPE, ExternalUser),
|
||||
company: await db.getRepository<Company>(CompanyType, Company),
|
||||
// This one is not typical because it's only valid for remote account type:
|
||||
session: gr.services.console.getSessionRepo() as Repository<Session> | null,
|
||||
//TODO: checkout what to do with session and user_online (seen in prod)
|
||||
|
||||
search: {
|
||||
driveFile: await search.getRepository<DriveFile>(DriveFileTYPE, DriveFile),
|
||||
user: await search.getRepository<User>(UserTYPE, User),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Return versions, sorted by oldest first, and related DB and S3 entries for a given DriveFile */
|
||||
export async function loadRawVersionsOfItemForDeletion(
|
||||
repos: TUserDeletionRepos,
|
||||
item: DriveFile,
|
||||
loadStoragePaths = false,
|
||||
): Promise<
|
||||
{
|
||||
version: FileVersion;
|
||||
file?: File;
|
||||
paths?: string[];
|
||||
}[]
|
||||
> {
|
||||
return Promise.all(
|
||||
(await repos.fileVersion.find({ drive_item_id: item.id }, { sort: { date_added: "asc" } }))
|
||||
.getEntities()
|
||||
.map(async version => {
|
||||
const file = await repos.file.findOne({ id: version.file_metadata.external_id });
|
||||
if (!loadStoragePaths) return { version, file };
|
||||
const paths = file
|
||||
? await gr.platformServices.storage.enumeratePathsForFile(getFilePath(file))
|
||||
: [];
|
||||
return { version, file, paths };
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Recursively descends through drive items in a depth-first manner with a random order at each level.
|
||||
*
|
||||
* @warn No security or filtering is done (e.g., no rights are checked, items in trash are included, etc.).
|
||||
*
|
||||
* @param {string} parent_id - The ID of the parent drive item (or virtual root) to start the descent from.
|
||||
* @param {(item: DriveFile, children: undefined | T[], parents: DriveFile[]) => Promise<T>} map - The mapping function to process each drive item.
|
||||
* - `item`: The current drive item being processed.
|
||||
* - `children`: The results of processing the children of the current drive item, or `undefined` if the item is not a directory.
|
||||
* - `parents`: An array of parent drive items leading to the current item.
|
||||
* @returns {Promise<T[]>} A promise that resolves to an array of results produced by the `map` function.
|
||||
*/
|
||||
export async function descendDriveItemsDepthFirstRandomOrder<T>(
|
||||
repos: TUserDeletionRepos,
|
||||
parent_id: string,
|
||||
map: (item: DriveFile, children: undefined | T[], parents: readonly DriveFile[]) => Promise<T>,
|
||||
) {
|
||||
async function descend(parent_id: string, parents: readonly DriveFile[]): Promise<T[]> {
|
||||
const items = (await repos.driveFile.find({ parent_id })).getEntities();
|
||||
fisherYattesShuffleInPlace(items);
|
||||
const result = new Array(items.length);
|
||||
for (const [index, item] of items.entries()) {
|
||||
const children = item.is_directory
|
||||
? await descend(item.id, parents.concat([item]))
|
||||
: undefined;
|
||||
result[index] = await map(item, children, parents);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
return descend(parent_id, []);
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
/** Shuffle an array in place, returns its parameter for convenience */
|
||||
export function fisherYattesShuffleInPlace<T>(list: T[]): T[] {
|
||||
let index = list.length;
|
||||
while (index) {
|
||||
const randomIndex = Math.floor(Math.random() * index);
|
||||
index--;
|
||||
[list[index], list[randomIndex]] = [list[randomIndex], list[index]];
|
||||
}
|
||||
return list;
|
||||
}
|
||||
Reference in New Issue
Block a user