eduvia/workavia-drive
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✨ back: delete user data administrative endpoint (#799)

Browse Source
This commit is contained in:
Eric Doughty-Papassideris
2025-03-09 23:36:58 +01:00
committed by Anton Shepilov
parent 9d35fbe0d7
commit cb61d4d8d1
21 changed files with 485 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tdrive/backend/node/src/core/platform/services/admin/controller/delete-user-controller.ts
+16 -12
View File
@@ -12,31 +12,35 @@ export class AdminDeleteUserController {
return this._repos;
}
/** Begin or forward the deletion process of a user */
async deleteUser(userId: string): Promise<"failed" | "deleting" | "done"> {
/** Begin or forward the deletion process of a user, if `deleteData` is false, only anonymises the user entry */
async deleteUser(userId: string, deleteData: boolean): Promise<"failed" | "deleting" | "done"> {
try {
await gr.services.users.anonymizeAndDelete({ id: userId }, {
user: { server_request: true },
company: { id: "// TODO: REPLACE WITH COMPANY ID" },
} as unknown as ExecutionContext);
if (
await gr.services.users.anonymizeAndDelete(
{ id: userId },
{
user: { server_request: true },
} as unknown as ExecutionContext,
deleteData,
)
)
return "done";
const existingUser = await (await this.getRepos()).user.findOne({ id: userId });
if (existingUser?.deleted) {
if (existingUser.delete_process_started_epoch > 0) return "deleting";
}
} catch (err) {
adminLogger.error({ err, userId }, "User deletion error");
console.log(err); //TODO: NONONO
return "failed";
}
return "done";
}
/** Get an array of user IDs that are incompletely deleted */
/** Get an array of 2 item arrays with `[ user IDs, delete_process_started_epoch ]` that are incompletely deleted */
async listUsersPendingDeletion() {
return (
const users = (
await (await this.getRepos()).user.find({}, { $gt: [["delete_process_started_epoch", 0]] })
)
.getEntities()
.map(({ id }) => id);
).getEntities();
return users.map(({ id, delete_process_started_epoch }) => [id, delete_process_started_epoch]);
}
}
tdrive/backend/node/src/core/platform/services/admin/service.ts
+217 -96
View File
@@ -6,9 +6,16 @@ import {
adminLogger,
buildUserDeletionRepositories,
descendDriveItemsDepthFirstRandomOrder,
findFileOfVersion,
loadRawVersionsOfItemForDeletion,
runInBatchesAreAllTrue,
} from "./utils";
import type Repository from "../database/services/orm/repository/repository";
import type { Logger } from "pino";
import type { DriveFile } from "../../../../services/documents/entities/drive-file";
import { getFilePath, getUserPath } from "../../../../services/files/services";
import type { File } from "../../../../services/files/entities/file";
import type { FileVersion } from "../../../../services/documents/entities/file-version";
export default class AdminServiceImpl implements AdminServiceAPI {
version: "1";
@@ -28,15 +35,19 @@ export default class AdminServiceImpl implements AdminServiceAPI {
private async deleteS3Paths(
logger: pino.Logger,
paths: string[],
batchSize = 5,
batchSize = 10,
): Promise<boolean> {
return await runInBatchesAreAllTrue(batchSize, paths, async paths =>
Promise.all(
paths.map(path => {
paths.map(async path => {
try {
return gr.platformServices.storage.remove("x" + path);
return await gr.platformServices.storage.remove(
path,
undefined,
undefined,
"admin:user_account_deletion",
);
} catch (err) {
console.log(err);
logger.error({ err, path }, "Error deleting storage item");
return false;
}
@@ -45,109 +56,219 @@ export default class AdminServiceImpl implements AdminServiceAPI {
);
}
/** Remove a DB entity, returns false if it threw an error, just logs if no entries were deleted and returns true */
private async safeDBDelete<T>(
deleteUserLogger: Logger,
repo: Repository<T>,
item: T,
): Promise<boolean> {
try {
const result = await repo.remove(item);
if (!result)
// No error but nothing deleted, just move on
deleteUserLogger.warn({ item, result }, `Failed to delete ${repo.table}, 0 result count`);
return true;
} catch (err) {
deleteUserLogger.error({ err, item }, `Error deleting ${repo.table}`);
}
return false;
}
/** Delete all the storage for a File, and if succesful, the File entry from the database */
private async safeDeleteFile(deleteUserLogger: Logger, file: File): Promise<boolean> {
const paths = await gr.platformServices.storage.enumeratePathsForFile(getFilePath(file));
if (paths.length === 0) deleteUserLogger.warn({ file, paths }, "No paths found for File");
else if (!(await this.deleteS3Paths(deleteUserLogger, paths))) {
deleteUserLogger.error({ file, paths }, "Failed to delete paths");
return false;
}
return await this.safeDBDelete(deleteUserLogger, (await this.repos).file, file);
}
/** Delette version and potential related File with {@link safeDeleteFile} */
private async safeDeleteVersion(
deleteUserLogger: Logger,
version: FileVersion,
file?: false | File,
): Promise<boolean> {
if (file !== false) {
file = file || (await findFileOfVersion(await this.repos, version));
if (file && !(await this.safeDeleteFile(deleteUserLogger, file))) return false;
}
return await this.safeDBDelete(deleteUserLogger, (await this.repos).fileVersion, version);
}
/**
* Attempt to delete the DriveFile from the database, including:
* - related FileVersion and File entities,
* - all paths in storage
* - any search document
*
* @warn Does not check the DriveFile has no children
*/
private async safeDeleteSingleDriveFile(
deleteUserLogger: Logger,
item: DriveFile,
): Promise<boolean> {
let canDeleteItem = true;
const versionsAssets = await loadRawVersionsOfItemForDeletion(await this.repos, item, false);
for (const { version, file } of versionsAssets) {
if (
!(await this.safeDeleteVersion(deleteUserLogger, version, item.is_directory ? false : file))
)
canDeleteItem = false;
}
try {
await (await this.repos).search.driveFile.service.remove([item as never]);
} catch (err) {
// canDeleteItem = false; // Search documents would be filtered out by the missing DriveFile, so continue
deleteUserLogger.error({ err, item }, "Error deleting drive item search entry");
}
if (canDeleteItem)
if (!(await this.safeDBDelete(deleteUserLogger, (await this.repos).driveFile, item)))
return false;
return canDeleteItem;
}
/**
* This call deletes all data related to a user. This means anything uploaded by the
* user, including shared items, (but not things shared by another with the user).
*
* The call is meant to be called repeatedly until it succeeds, and should resist
* concurrent calls eventually succeeding (though avoid if possible, it's not productive).
* It should also resist being interrupted by a timeout mid-way, and resuming on a later
* call.
*
* Multiple phases are involved, some must succeed to continue, some failures are ignored
* as the output should be filtered in other ways.
*
* - Depth-first recurse DriveFiles by parent_id
* - For files
* - For each FileVersion:
* - Delete all storage blobs
* - Delete related File
* - Delete related FileVersion
* - Attempt to remove from search index
* - Delete DriveFile
* - For directories: delete if all children were deleted
*
* - Locate `DriveFile`s, `FileVersion`s then `File`s that have the creator_id
* set to the user being deleted, and delete them individually in a similar
* way to above
* - Delete company users and external users
* - Set user's `delete_process_started_epoch` field to 0 if succesful
*
* @param user {@link User} entity to delete
* @returns `true` if the user was completely deleted
*/
public async deleteUser(user: User): Promise<boolean> {
/*
//TODO: NONONO
const deleteUserLogger = adminLogger.child({ adminOp: "DeleteUser", user: user.id });
// todo: do test with wrong s3 delete path to ignore specifically only non existing errors
Phases:
- Recurse drive item
- if file
- Each version
- delete all s3
- then file
- then the version
- delete search document
- delete item
- if folder - delete if empty
- search items by creator
- search files by userid
- versions ?
- search leftover s3
// search for file by user id after deletion
// search s3 for prefix company/userid
*/
const deleteUserLogger = adminLogger.child({ adminOp: "DeleteUser", user });
const result = await descendDriveItemsDepthFirstRandomOrder(
await this.repos,
"user_" + user.id,
async (item, children, _parents) => {
let canDeleteItem = true;
if (!item.is_directory) {
const versionsAssets = await loadRawVersionsOfItemForDeletion(
await this.repos,
item,
true,
if (children === undefined || children.every(x => !!x)) {
if (item.is_directory && children.length == 0)
deleteUserLogger.warn({ item }, "Deleting empty directory");
if (!(await this.safeDeleteSingleDriveFile(deleteUserLogger, item))) return false;
} else {
deleteUserLogger.error(
{ item },
"Not deleting directory as at least one child failed to delete",
);
for (const { version, file, paths } of versionsAssets) {
if (paths.length > 0 && !(await this.deleteS3Paths(deleteUserLogger, paths))) {
deleteUserLogger.error({ paths }, "Failed to delete paths");
canDeleteItem = false;
} else {
try {
if (file) {
const result = await (await this.repos).file.remove(file);
if (!result)
// No error but nothing deleted, just move on
deleteUserLogger.warn({ file, result }, "Failed to delete file");
}
} catch (err) {
canDeleteItem = false;
deleteUserLogger.error({ err, file }, "Error deleting file");
}
if (canDeleteItem && version)
try {
if (version) {
const result = await (await this.repos).fileVersion.remove(version);
if (!result)
// No error but nothing deleted, just move on
deleteUserLogger.warn({ version, result }, "Failed to delete version");
}
} catch (err) {
canDeleteItem = false;
deleteUserLogger.error({ err, version }, "Error deleting version");
}
}
}
return false;
}
if (canDeleteItem) {
if (children === undefined || children.every(x => !!x)) {
if (item.is_directory && children.length == 0)
deleteUserLogger.warn({ item }, "Deleting empty directory");
try {
await (await this.repos).search.driveFile.service.remove([item as never]);
} catch (err) {
canDeleteItem = false;
deleteUserLogger.error({ err, item }, "Error deleting drive item search entry");
}
if (canDeleteItem)
try {
const result = await (await this.repos).driveFile.remove(item);
if (!result)
// No error but nothing deleted, just move on
deleteUserLogger.warn({ item, result }, "Failed to delete drive item");
} catch (err) {
canDeleteItem = false;
deleteUserLogger.error({ err, item }, "Error deleting drive item");
}
} else {
deleteUserLogger.error(
{ item },
"Not deleting directory as at least one child failed to delete",
);
canDeleteItem = false;
}
}
return canDeleteItem;
return true;
},
);
// Stop if deleting root DriveFiles didn't succeed
if (!result.every(x => !!x)) return false;
//TODO: error checking and such
// Stray items out of parent_id etc tree attached to the user in other ways
let canContinueDeleting = true;
const driveItemsByCreator = (
await (await this.repos).driveFile.find({ creator: user.id })
).getEntities();
deleteUserLogger.info(
{ canContinueDeleting, driveItemsByCreator: driveItemsByCreator.length },
"Stray driveItemsByCreator",
);
for (const creatorDriveFile of driveItemsByCreator)
if (!(await this.safeDeleteSingleDriveFile(deleteUserLogger, creatorDriveFile)))
canContinueDeleting = false;
if (!canContinueDeleting) return false;
const versionsByCreator = (
await (await this.repos).fileVersion.find({ creator_id: user.id })
).getEntities();
deleteUserLogger.info(
{ canContinueDeleting, versionsByCreator: versionsByCreator.length },
"Stray versionsByCreator",
);
for (const creatorVersion of versionsByCreator)
if (!(await this.safeDeleteVersion(deleteUserLogger, creatorVersion)))
canContinueDeleting = false;
if (!canContinueDeleting) return false;
deleteUserLogger.info(
{ canContinueDeleting, versionsByCreator: versionsByCreator.length },
"Stray versionsByCreator",
);
const filesByCreator = (await (await this.repos).file.find({ user_id: user.id })).getEntities();
deleteUserLogger.info(
{ canContinueDeleting, filesByCreator: filesByCreator.length },
"Stray filesByCreator",
);
for (const creatorFile of filesByCreator)
if (!(await this.safeDeleteFile(deleteUserLogger, creatorFile))) canContinueDeleting = false;
if (!canContinueDeleting) return false;
// Remove stray S3 data by prefix folder, then de-associate from company if ok
const companyUsers = (
await (await this.repos).companyUser.find({ user_id: user.id })
).getEntities();
deleteUserLogger.info(
{ canContinueDeleting, companyUsers: companyUsers.length },
"companyUsers",
);
for (const companyUser of companyUsers) {
const paths = await gr.platformServices.storage.enumeratePathsForFile(
getUserPath(user.id, companyUser.group_id),
);
if (!(await this.deleteS3Paths(deleteUserLogger, paths))) canContinueDeleting = false;
else if (
!(await this.safeDBDelete(deleteUserLogger, (await this.repos).companyUser, companyUser))
) {
deleteUserLogger.warn({ companyUser }, "Failed to delete database entry");
canContinueDeleting = false;
}
}
const externalUsers = (
await (await this.repos).externalUser.find({ user_id: user.id })
).getEntities();
deleteUserLogger.info(
{ canContinueDeleting, externalUsers: externalUsers.length },
"externalUsers",
);
for (const externalUser of externalUsers)
if (
!(await this.safeDBDelete(deleteUserLogger, (await this.repos).externalUser, externalUser))
)
canContinueDeleting = false;
await (await this.repos).search.user.service.remove([user as never]);
user.delete_process_started_epoch = 0;
await (await this.repos).user.save(user);
return true;
if (canContinueDeleting) {
const latestUser = await (await this.repos).user.findOne({ id: user.id });
deleteUserLogger.info("User deletion complete, zeroing delete_process_started_epoch");
latestUser.delete_process_started_epoch = 0;
await (await this.repos).user.save(latestUser);
}
return canContinueDeleting;
}
}
tdrive/backend/node/src/core/platform/services/admin/utils.ts
+12 -3
View File
@@ -70,7 +70,7 @@ export async function buildUserDeletionRepositories(
db: DatabaseServiceAPI = gr.database,
search: SearchServiceAPI = gr.platformServices.search,
) {
return {
const result = {
driveFile: await db.getRepository<DriveFile>(DriveFileTYPE, DriveFile),
file: await db.getRepository<File>(FileTYPE, File),
fileVersion: await db.getRepository<FileVersion>(FileVersionTYPE, FileVersion),
@@ -81,13 +81,22 @@ export async function buildUserDeletionRepositories(
company: await db.getRepository<Company>(CompanyType, Company),
// This one is not typical because it's only valid for remote account type:
session: gr.services.console.getSessionRepo() as Repository<Session> | null,
//TODO: checkout what to do with session and user_online (seen in prod)
//TODO: checkout what to do with user_online (seen in prod)
search: {
driveFile: await search.getRepository<DriveFile>(DriveFileTYPE, DriveFile),
user: await search.getRepository<User>(UserTYPE, User),
},
};
await Promise.all(Object.values(result).map(item => "init" in item && item.init()));
return result;
}
export async function findFileOfVersion(
repos: TUserDeletionRepos,
version: FileVersion,
): Promise<File> {
return await repos.file.findOne({ id: version.file_metadata.external_id });
}
/** Return versions, sorted by oldest first, and related DB and S3 entries for a given DriveFile */
@@ -106,7 +115,7 @@ export async function loadRawVersionsOfItemForDeletion(
(await repos.fileVersion.find({ drive_item_id: item.id }, { sort: { date_added: "asc" } }))
.getEntities()
.map(async version => {
const file = await repos.file.findOne({ id: version.file_metadata.external_id });
const file = await findFileOfVersion(repos, version);
if (!loadStoragePaths) return { version, file };
const paths = file
? await gr.platformServices.storage.enumeratePathsForFile(getFilePath(file))
tdrive/backend/node/src/core/platform/services/admin/web/delete-user-routes.ts
+14 -9
View File
@@ -14,30 +14,35 @@ function authenticateAdminQuery(request: FastifyRequest, reply: FastifyReply) {
return true;
}
type TUserDeleteQueryBody = TQueryBody & { userId: string };
type TUserDeleteQueryBody = TQueryBody & { userId: string; deleteData: boolean };
function getUserIfValidQuery(request: FastifyRequest, reply: FastifyReply) {
if (!authenticateAdminQuery(request, reply)) return false;
if (!authenticateAdminQuery(request, reply)) return { userId: "", deleteData: false };
const body = request.body as TUserDeleteQueryBody;
if (!body.userId?.length) {
reply.status(400).send();
return false;
return { userId: "", deleteData: false };
}
return body.userId;
return body;
}
const routes: FastifyPluginCallback = async (fastify: FastifyInstance, _opts, next) => {
const urlRoot = "/api/user/delete";
const config = getConfig();
const controller = new AdminDeleteUserController();
if (config?.endpointSecret?.length) {
fastify.post("/user/delete", async (request, reply) => {
const userId = getUserIfValidQuery(request, reply);
fastify.post(urlRoot, async (request, reply) => {
const { userId, deleteData } = getUserIfValidQuery(request, reply);
if (!userId) return false;
return reply.send({ status: await controller.deleteUser(userId) });
if (userId == "e2e_simulate_timeout")
return new Promise(() => {
return;
});
return { status: await controller.deleteUser(userId, deleteData) };
});
fastify.post("/user/delete/pending", async (request, reply) => {
fastify.post(`${urlRoot}/pending`, async (request, reply) => {
if (!authenticateAdminQuery(request, reply)) return false;
return reply.send(await controller.listUsersPendingDeletion());
return await controller.listUsersPendingDeletion();
});
}
next();
tdrive/backend/node/src/core/platform/services/storage/connectors/S3/s3-service.ts
+27 -3
View File
@@ -15,7 +15,13 @@ export type S3Configuration = {
useSSL: boolean;
accessKey: string;
secretKey: string;
/** If `true`, S3 file removal is disabled (see exception of {@link overrideDisableRemoveForUserAccountDeletion}) */
disableRemove: boolean;
/**
* If {@link disableRemove} is `true`, but the deletion is in the context of a user account deletion by an administrator,
* then override and delete from S3 if `overrideDisableRemoveForUserAccountDeletion` is also `true`.
*/
overrideDisableRemoveForUserAccountDeletion: boolean;
};
export default class S3ConnectorService implements StorageConnectorAPI {
@@ -31,6 +37,12 @@ export default class S3ConnectorService implements StorageConnectorAPI {
if (confCopy.useSSL && typeof confCopy.useSSL === "string") {
confCopy.useSSL = !(!confCopy.useSSL || confCopy.useSSL === "false");
}
if (
confCopy.overrideDisableRemoveForUserAccountDeletion &&
typeof confCopy.overrideDisableRemoveForUserAccountDeletion === "string"
)
confCopy.overrideDisableRemoveForUserAccountDeletion =
confCopy.overrideDisableRemoveForUserAccountDeletion === "true";
this.client = new Minio.Client(confCopy);
this.minioConfiguration = confCopy;
this.id = this.minioConfiguration.id;
@@ -114,16 +126,28 @@ export default class S3ConnectorService implements StorageConnectorAPI {
return this.client.getObject(this.minioConfiguration.bucket, path);
}
async remove(path: string): Promise<boolean> {
async remove(
path: string,
_options?: undefined,
_context?: undefined,
deletionCause?: "admin:user_account_deletion",
): Promise<boolean> {
try {
if (this.minioConfiguration.disableRemove) {
if (
this.minioConfiguration.disableRemove &&
(!this.minioConfiguration.overrideDisableRemoveForUserAccountDeletion ||
deletionCause !== "admin:user_account_deletion")
) {
logger.info(`File ${path} wasn't removed, file removal is disabled in configuration`);
return true;
} else {
// This call never fails, whether path exists or not
await this.client.removeObject(this.minioConfiguration.bucket, path);
return true;
}
} catch (err) {}
} catch (err) {
logger.error({ err, path }, "Error deleting S3 path");
}
return false;
}
tdrive/backend/node/src/core/platform/services/storage/default-storage-strategy.ts
+2 -1
View File
@@ -48,7 +48,8 @@ export class DefaultStorageStrategy implements StorageConnectorAPI {
path: string,
options?: DeleteOptions,
context?: ExecutionContext,
deletionCause?: undefined | "admin:user_account_deletion",
): Promise<boolean> => {
return this.connector.remove(path, options, context);
return this.connector.remove(path, options, context, deletionCause);
};
}
tdrive/backend/node/src/core/platform/services/storage/oneof-storage-strategy.ts
+9 -2
View File
@@ -172,8 +172,15 @@ export class OneOfStorageStrategy implements StorageConnectorAPI {
* @param path - The path of the file to be removed.
* @param options
*/
remove = async (path: string, options?: DeleteOptions): Promise<boolean> => {
return Promise.all(this.storages.map(storage => storage.remove(path, options))).then(array => {
remove = async (
path: string,
options?: DeleteOptions,
context?: undefined,
deletionCause?: "admin:user_account_deletion",
): Promise<boolean> => {
return Promise.all(
this.storages.map(storage => storage.remove(path, options, context, deletionCause)),
).then(array => {
return array.reduce((a, b) => a && b);
});
};
tdrive/backend/node/src/core/platform/services/storage/provider.ts
+10 -1
View File
@@ -61,8 +61,17 @@ export interface StorageConnectorAPI extends IServiceDiagnosticProvider {
* Remove a path
*
* @param path
* @param deletionCause In case of exceptional uses of deletion, eg. legal compliance, a reason
* for the deletion can be provided here and evaluated by the StorageConnector.
*
* In normal use, pass `undefined`
*/
remove(path: string, options?: DeleteOptions, context?: ExecutionContext): Promise<boolean>;
remove(
path: string,
options?: DeleteOptions,
context?: ExecutionContext,
deletionCause?: undefined | "admin:user_account_deletion",
): Promise<boolean>;
/**
* Enumerate all physical storage paths related to the provided file path
tdrive/backend/node/src/core/platform/services/storage/storage-service.ts
+10 -3
View File
@@ -172,12 +172,18 @@ export default class StorageService extends TdriveService<StorageAPI> implements
return stream;
}
async remove(path: string, options?: DeleteOptions) {
async remove(
path: string,
options?: DeleteOptions,
context?: undefined,
deletionCause?: undefined | "admin:user_account_deletion",
) {
try {
for (let count = 1; count <= (options?.totalChunks || 1); count++) {
const chunk = options?.totalChunks ? `${path}/chunk${count}` : path;
await this.getConnector().remove(chunk);
await this.getConnector().remove(chunk, options, context, deletionCause);
}
if (!options) await this.getConnector().remove(path, options, context, deletionCause);
return true;
} catch (err) {
logger.error("Unable to remove file %s", err);
@@ -207,7 +213,8 @@ export default class StorageService extends TdriveService<StorageAPI> implements
* "useSSL": false,
* "accessKey": "ABCD",
* "secretKey": "x1yz",
* "disableRemove": false
* "disableRemove": false,
* "overrideDisableRemoveForUserAccountDeletion": true,
* },
* "local": {
* "path": "/tdrive"