📁 Changed TDrive root folder (#16)

📁 Changed TDrive root folder

tdrive/backend/node/src/services/files/entities/file.ts

@@ -0,0 +1,90 @@
+import { Type } from "class-transformer";
+import _ from "lodash";
+import { Column, Entity } from "../../../core/platform/services/database/services/orm/decorators";
+
+@Entity("files", {
+  primaryKey: [["company_id"], "id"],
+  type: "files",
+})
+export class File {
+  @Type(() => String)
+  @Column("company_id", "uuid")
+  company_id: string;
+
+  @Type(() => String)
+  @Column("id", "uuid", { generator: "uuid" })
+  id: string;
+
+  @Type(() => String)
+  @Column("user_id", "encoded_string")
+  user_id: string;
+
+  @Column("application_id", "encoded_string")
+  application_id: null | string;
+
+  @Column("encryption_key", "encoded_string")
+  encryption_key: string;
+
+  @Column("updated_at", "number", { onUpsert: _ => new Date().getTime() })
+  updated_at: number;
+
+  @Column("created_at", "number", { onUpsert: d => d || new Date().getTime() })
+  created_at: number;
+
+  @Column("metadata", "encoded_json")
+  metadata: null | {
+    name?: string;
+    mime?: string;
+    thumbnails_status?: "done" | "error" | "waiting";
+  };
+
+  @Column("thumbnails", "encoded_json")
+  thumbnails: Thumbnail[];
+
+  @Column("upload_data", "encoded_json")
+  upload_data: null | {
+    size: number;
+    chunks: number;
+  };
+
+  getPublicObject(): PublicFile {
+    return _.pick(
+      this,
+      "company_id",
+      "id",
+      "user_id",
+      "application_id",
+      "updated_at",
+      "created_at",
+      "metadata",
+      "thumbnails",
+      "upload_data",
+    );
+  }
+}
+
+export type PublicFile = Pick<
+  File,
+  | "company_id"
+  | "id"
+  | "user_id"
+  | "application_id"
+  | "updated_at"
+  | "created_at"
+  | "metadata"
+  | "thumbnails"
+  | "upload_data"
+>;
+
+export type Thumbnail = {
+  index: number;
+  id: string;
+
+  type: string;
+  size: number;
+  width: number;
+  height: number;
+
+  url: string;
+  full_url?: string;
+};

tdrive/backend/node/src/services/files/index.ts

@@ -0,0 +1,23 @@
+import { Prefix, TdriveService } from "../../core/platform/framework";
+import WebServerAPI from "../../core/platform/services/webserver/provider";
+import web from "./web";
+
+@Prefix("/internal/services/files/v1")
+export default class FilesService extends TdriveService<undefined> {
+  version = "1";
+  name = "files";
+
+  public async doInit(): Promise<this> {
+    const fastify = this.context.getProvider<WebServerAPI>("webserver").getServer();
+    fastify.register((instance, _opts, next) => {
+      web(instance, { prefix: this.prefix });
+      next();
+    });
+    return this;
+  }
+
+  // TODO: remove
+  api(): undefined {
+    return undefined;
+  }
+}

tdrive/backend/node/src/services/files/services/index.ts

@@ -0,0 +1,231 @@
+import { randomBytes } from "crypto";
+import { Readable } from "stream";
+import { Multipart } from "fastify-multipart";
+import { UploadOptions } from "../types";
+import { File } from "../entities/file";
+import Repository from "../../../../src/core/platform/services/database/services/orm/repository/repository";
+import { CompanyExecutionContext } from "../web/types";
+import { logger } from "../../../core/platform/framework";
+import { getDownloadRoute, getThumbnailRoute } from "../web/routes";
+import {
+  CrudException,
+  DeleteResult,
+  ExecutionContext,
+} from "../../../core/platform/framework/api/crud-service";
+import gr from "../../global-resolver";
+
+export class FileServiceImpl {
+  version: "1";
+  repository: Repository<File>;
+  private algorithm = "aes-256-cbc";
+  private max_preview_file_size = 50000000;
+
+  async init(): Promise<this> {
+    try {
+      await Promise.all([(this.repository = await gr.database.getRepository<File>("files", File))]);
+    } catch (err) {
+      logger.error("Error while initializing files service", err);
+    }
+    return this;
+  }
+
+  async save(
+    id: string,
+    file: Multipart,
+    options: UploadOptions,
+    context: CompanyExecutionContext,
+  ): Promise<File> {
+    const userId = context.user?.id;
+    const applicationId: string | null = context.user?.application_id || null;
+
+    let entity: File = null;
+    if (id) {
+      entity = await this.repository.findOne(
+        {
+          company_id: context.company.id,
+          id: id,
+        },
+        {},
+        context,
+      );
+      if (!entity) {
+        throw new Error(`This file ${id} does not exist`);
+      }
+    }
+
+    if (!entity) {
+      entity = new File();
+      entity.company_id = `${context.company.id}`;
+      entity.metadata = null;
+      entity.thumbnails = [];
+
+      const iv = randomBytes(8).toString("hex");
+      const secret_key = randomBytes(16).toString("hex");
+      entity.encryption_key = `${secret_key}.${iv}`;
+
+      entity.user_id = userId;
+      entity.application_id = applicationId;
+      entity.upload_data = null;
+
+      this.repository.save(entity, context);
+    }
+
+    if (file) {
+      // Detect a new file upload
+      // Only applications can overwrite a file.
+      // Users alone can only write an empty file.
+      if (applicationId || !entity.upload_data?.size || context.user.server_request) {
+        if (
+          //If there was any change to the file
+          entity.upload_data?.size !== options.totalSize ||
+          entity.metadata?.name !== options.filename
+        ) {
+          entity.metadata = {
+            name: options.filename,
+            mime: options.type,
+            thumbnails_status: "done",
+          };
+          entity.upload_data = {
+            size: options.totalSize,
+            chunks: options.totalChunks || 1,
+          };
+          this.repository.save(entity, context);
+        }
+      }
+
+      let totalUploadedSize = 0;
+      file.file.on("data", function (chunk) {
+        totalUploadedSize += chunk.length;
+      });
+      await gr.platformServices.storage.write(getFilePath(entity), file.file, {
+        chunkNumber: options.chunkNumber,
+        encryptionAlgo: this.algorithm,
+        encryptionKey: entity.encryption_key,
+      });
+
+      if (entity.upload_data.chunks === 1 && totalUploadedSize) {
+        entity.upload_data.size = totalUploadedSize;
+        await this.repository.save(entity, context);
+      }
+    }
+
+    return entity;
+  }
+
+  async exists(id: string, companyId: string, context?: CompanyExecutionContext): Promise<boolean> {
+    const entity = await this.getFile({ id, company_id: companyId }, context);
+    return !!entity;
+  }
+
+  async download(
+    id: string,
+    context: CompanyExecutionContext,
+  ): Promise<{ file: Readable; name: string; mime: string; size: number }> {
+    const entity = await this.get(id, context);
+    if (!entity) {
+      throw "File not found";
+    }
+
+    const readable = await gr.platformServices.storage.read(getFilePath(entity), {
+      totalChunks: entity.upload_data.chunks,
+      encryptionAlgo: this.algorithm,
+      encryptionKey: entity.encryption_key,
+    });
+
+    return {
+      file: readable,
+      name: entity.metadata.name,
+      mime: entity.metadata.mime,
+      size: entity.upload_data.size,
+    };
+  }
+
+  async thumbnail(
+    id: string,
+    index: string,
+    context: CompanyExecutionContext,
+  ): Promise<{ file: Readable; type: string; size: number }> {
+    const entity = await this.get(id, context);
+
+    if (!entity) {
+      throw "File not found";
+    }
+
+    const thumbnail = entity.thumbnails[parseInt(index)];
+    if (!thumbnail) {
+      throw `Thumbnail ${parseInt(index)} not found`;
+    }
+
+    const thumbnailPath = `${getFilePath(entity)}/thumbnails/${thumbnail.id}`;
+
+    const readable = await gr.platformServices.storage.read(thumbnailPath, {
+      encryptionAlgo: this.algorithm,
+      encryptionKey: entity.encryption_key,
+    });
+
+    return {
+      file: readable,
+      type: thumbnail.type,
+      size: thumbnail.size,
+    };
+  }
+
+  get(id: string, context: CompanyExecutionContext): Promise<File> {
+    if (!id || !context.company.id) {
+      return null;
+    }
+    return this.getFile({ id, company_id: context.company.id }, context);
+  }
+
+  async getFile(
+    pk: Pick<File, "company_id" | "id">,
+    context?: ExecutionContext,
+    options?: {
+      waitForThumbnail?: boolean;
+    },
+  ): Promise<File> {
+    let entity = await this.repository.findOne(pk, {}, context);
+
+    if (options?.waitForThumbnail) {
+      for (let i = 1; i < 100; i++) {
+        if (entity.metadata.thumbnails_status === "done") {
+          break;
+        }
+        await new Promise(r => setTimeout(r, i * 200));
+        entity = await this.repository.findOne(pk, {}, context);
+      }
+    }
+
+    return entity;
+  }
+
+  getThumbnailRoute(file: File, index: string) {
+    return getThumbnailRoute(file, index);
+  }
+
+  getDownloadRoute(file: File) {
+    return getDownloadRoute(file);
+  }
+
+  async delete(id: string, context: CompanyExecutionContext): Promise<DeleteResult<File>> {
+    const fileToDelete = await this.get(id, context);
+
+    if (!fileToDelete) {
+      throw new CrudException("File not found", 404);
+    }
+
+    await this.repository.remove(fileToDelete, context);
+
+    const path = getFilePath(fileToDelete);
+
+    await gr.platformServices.storage.remove(path, {
+      totalChunks: fileToDelete.upload_data.chunks,
+    });
+
+    return new DeleteResult("files", fileToDelete, true);
+  }
+}
+
+function getFilePath(entity: File): string {
+  return `/tdrive/files/${entity.company_id}/${entity.user_id || "anonymous"}/${entity.id}`;
+}

tdrive/backend/node/src/services/files/types.ts

@@ -0,0 +1,8 @@
+export type UploadOptions = {
+  filename: string;
+  type: string;
+  totalSize: number;
+  totalChunks: number;
+  chunkNumber: number;
+  waitForThumbnail: boolean;
+};

tdrive/backend/node/src/services/files/utils.ts

@@ -0,0 +1,24 @@
+import { File, PublicFile } from "./entities/file";
+
+export const formatPublicFile = (file: Partial<File | PublicFile>): PublicFile => {
+  if ((file as Partial<File>).getPublicObject) file = (file as Partial<File>).getPublicObject();
+  return {
+    ...file,
+    thumbnails: [
+      ...file.thumbnails.map(thumbnail => ({
+        ...thumbnail,
+        full_url: thumbnail.url.match(/https?:\/\//)
+          ? "/internal/services/files/v1/" + thumbnail.url.replace(/^\//, "")
+          : thumbnail.url,
+      })),
+    ],
+  } as PublicFile;
+};
+
+export const fileIsMedia = (file: Partial<File>): boolean => {
+  return (
+    file.metadata?.mime?.startsWith("video/") ||
+    file.metadata?.mime?.startsWith("audio/") ||
+    file.metadata?.mime?.startsWith("image/")
+  );
+};

tdrive/backend/node/src/services/files/web/controllers/files.ts

@@ -0,0 +1,113 @@
+import { FastifyReply, FastifyRequest } from "fastify";
+import { Multipart } from "fastify-multipart";
+import { ResourceDeleteResponse } from "../../../../utils/types";
+import { CompanyExecutionContext } from "../types";
+import { UploadOptions } from "../../types";
+import { PublicFile } from "../../entities/file";
+import gr from "../../../global-resolver";
+
+export class FileController {
+  async save(
+    request: FastifyRequest<{
+      Params: { company_id: string; id: string };
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      Querystring: any;
+    }>,
+  ): Promise<{ resource: PublicFile }> {
+    const context = getCompanyExecutionContext(request);
+
+    let file: null | Multipart = null;
+    if (request.isMultipart()) {
+      file = await request.file();
+    }
+    const q = request.query;
+    const options: UploadOptions = {
+      totalChunks: parseInt(q.resumableTotalChunks || q.total_chunks) || 1,
+      totalSize: parseInt(q.resumableTotalSize || q.total_size) || 0,
+      chunkNumber: parseInt(q.resumableChunkNumber || q.chunk_number) || 1,
+      filename: q.resumableFilename || q.filename || file?.filename || undefined,
+      type: q.resumableType || q.type || file?.mimetype || undefined,
+      waitForThumbnail: q.thumbnail_sync,
+    };
+
+    const id = request.params.id;
+    const result = await gr.services.files.save(id, file, options, context);
+
+    return {
+      resource: result.getPublicObject(),
+    };
+  }
+
+  async download(
+    request: FastifyRequest<{ Params: { company_id: string; id: string } }>,
+    response: FastifyReply,
+  ): Promise<void> {
+    const context = getCompanyExecutionContext(request);
+    const params = request.params;
+    const data = await gr.services.files.download(params.id, context);
+    const filename = data.name.replace(/[^a-zA-Z0-9 -_.]/g, "");
+
+    response.header("Content-disposition", `attachment; filename="${filename}"`);
+    if (data.size) response.header("Content-Length", data.size);
+    response.type(data.mime);
+    response.send(data.file);
+  }
+
+  async thumbnail(
+    request: FastifyRequest<{ Params: { company_id: string; id: string; index: string } }>,
+    response: FastifyReply,
+  ): Promise<void> {
+    const context = getCompanyExecutionContext(request);
+    const params = request.params;
+    try {
+      const data = await gr.services.files.thumbnail(params.id, params.index, context);
+
+      response.header("Content-disposition", "inline");
+      response.expires(new Date(new Date().getTime() + 1000 * 60 * 60 * 24 * 365));
+      if (data.size) response.header("Content-Length", data.size);
+      response.type(data.type);
+      response.send(data.file);
+    } catch (err) {
+      console.log(err);
+      response.statusCode = 500;
+      response.send("");
+    }
+  }
+
+  async get(
+    request: FastifyRequest<{ Params: { company_id: string; id: string } }>,
+  ): Promise<{ resource: PublicFile }> {
+    const context = getCompanyExecutionContext(request);
+    const params = request.params;
+    const resource = await gr.services.files.get(params.id, context);
+
+    return { resource: resource.getPublicObject() };
+  }
+
+  async delete(
+    request: FastifyRequest<{ Params: { company_id: string; id: string } }>,
+  ): Promise<ResourceDeleteResponse> {
+    const params = request.params;
+    const context = getCompanyExecutionContext(request);
+
+    const deleteResult = await gr.services.files.delete(params.id, context);
+
+    return { status: deleteResult.deleted ? "success" : "error" };
+  }
+}
+
+function getCompanyExecutionContext(
+  request: FastifyRequest<{
+    Params: { company_id: string };
+  }>,
+): CompanyExecutionContext {
+  return {
+    user: request.currentUser,
+
+    company: { id: request.params.company_id },
+    url: request.url,
+    method: request.routerMethod,
+    reqId: request.id,
+    transport: "http",
+  };
+}

tdrive/backend/node/src/services/files/web/controllers/index.ts

@@ -0,0 +1 @@
+export * from "./files";

tdrive/backend/node/src/services/files/web/index.ts

@@ -0,0 +1,12 @@
+import { FastifyInstance, FastifyRegisterOptions } from "fastify";
+import fastifyCaching from "@fastify/caching";
+import routes from "./routes";
+
+export default (
+  fastify: FastifyInstance,
+  options: FastifyRegisterOptions<{ prefix: string }>,
+): void => {
+  fastify.log.debug("Configuring /internal/services/files/v1 routes");
+  fastify.register(fastifyCaching, { expiresIn: 31536000, privacy: fastifyCaching.privacy.PUBLIC });
+  fastify.register(routes, options);
+};

tdrive/backend/node/src/services/files/web/routes.ts

@@ -0,0 +1,61 @@
+import { FastifyInstance, FastifyPluginCallback } from "fastify";
+import { FileController } from "./controllers";
+import { File } from "../entities/file";
+
+const filesUrl = "/companies/:company_id/files";
+
+const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next) => {
+  const fileController = new FileController();
+
+  fastify.route({
+    method: "POST",
+    url: filesUrl,
+    preValidation: [fastify.authenticate],
+    handler: fileController.save.bind(fileController),
+  });
+
+  fastify.route({
+    method: "POST",
+    url: `${filesUrl}/:id`,
+    preValidation: [fastify.authenticate],
+    handler: fileController.save.bind(fileController),
+  });
+
+  fastify.route({
+    method: "GET",
+    url: `${filesUrl}/:id/download`,
+    handler: fileController.download.bind(fileController),
+  });
+
+  fastify.route({
+    method: "GET",
+    url: `${filesUrl}/:id`,
+    preValidation: [fastify.authenticate],
+    handler: fileController.get.bind(fileController),
+  });
+
+  fastify.route({
+    method: "GET",
+    url: `${filesUrl}/:id/thumbnails/:index`,
+    handler: fileController.thumbnail.bind(fileController),
+  });
+
+  fastify.route({
+    method: "DELETE",
+    url: `${filesUrl}/:id`,
+    preValidation: [fastify.authenticate],
+    handler: fileController.delete.bind(fileController),
+  });
+
+  next();
+};
+
+export const getDownloadRoute = (file: File) => {
+  return filesUrl.replace(":company_id", file.company_id) + `/${file.id}/download`;
+};
+
+export const getThumbnailRoute = (file: File, index: string) => {
+  return `/internal/services/files/v1/companies/${file.company_id}/files/${file.id}/thumbnails/${index}`;
+};
+
+export default routes;

tdrive/backend/node/src/services/files/web/schemas.ts

@@ -0,0 +1,13 @@
+export const filesSchema = {
+  type: "object",
+  properties: {
+    preferences: {
+      type: "object",
+    },
+    id: { type: "string" },
+    name: { type: "string" },
+    size: { type: "number" },
+    width: { type: "number" },
+    height: { type: "number" },
+  },
+};

tdrive/backend/node/src/services/files/web/types.ts

@@ -0,0 +1,5 @@
+import { ExecutionContext } from "../../../core/platform/framework/api/crud-service";
+
+export interface CompanyExecutionContext extends ExecutionContext {
+  company: { id: string };
+}