✨ Simple User Quota (#367)

* ✨Simple User Quota
* 👷Fix build with OpenSearch

tdrive/backend/node/config/custom-environment-variables.json

@@ -138,7 +138,10 @@
     "featureSharedDrive": "ENABLE_FEATURE_SHARED_DRIVE",
     "rootAdmins": "DRIVE_ROOT_ADMINS",
     "defaultLanguage": "DRIVE_DEFAULT_LANGUAGE",
+    "defaultCompany": "DRIVE_DEFAULT_COMPANY",
+    "defaultUserQuota": "DRIVE_DEFAULT_USER_QUOTA",
     "featureSearchUsers": "ENABLE_FEATURE_SEARCH_USERS",
-    "featureDisplayEmail": "ENABLE_FEATURE_DISPLAY_EMAIL"
+    "featureDisplayEmail": "ENABLE_FEATURE_DISPLAY_EMAIL",
+    "featureUserQuota": "ENABLE_FEATURE_USER_QUOTA"
   }
 }

tdrive/backend/node/config/default.json

@@ -138,7 +138,10 @@
   "drive": {
     "featureSharedDrive": true,
     "featureSearchUsers": true,
-    "featureDisplayEmail": true
+    "featureDisplayEmail": true,
+    "featureUserQuota": false,
+    "defaultCompany": "00000000-0000-4000-0000-000000000000",
+    "defaultUserQuota": 200000000
   },
   "applications": {
     "grid": [

tdrive/backend/node/package.json

@@ -67,7 +67,7 @@
     "@types/eslint": "^7.2.3",
     "@types/fluent-ffmpeg": "^2.1.20",
     "@types/html-to-text": "^8.1.1",
-    "@types/jest": "^29.1.0",
+    "@types/jest": "^29.5.12",
     "@types/lodash": "^4.14.165",
     "@types/node": "^18.11",
     "@types/node-cron": "^3.0.0",
@@ -88,13 +88,13 @@
     "eslint-plugin-prettier": "^4.2.1",
     "eslint-plugin-unused-imports": "^2.0.0",
     "form-auto-content": "^2.2.0",
-    "jest": "^29.1.0",
+    "jest": "^29.7.0",
     "jest-mock-extended": "^3.0.4",
     "nodemon": "^2.0.22",
     "pino-pretty": "^10.0.0",
     "prettier": "^2.1.2",
     "rimraf": "^3.0.2",
-    "ts-jest": "^29.1.0",
+    "ts-jest": "^29.1.2",
     "ts-node": "^9.0.0",
     "ts-node-dev": "^1.1.8",
     "tsc-watch": "^4.2.9",

tdrive/backend/node/src/core/platform/services/database/services/orm/connectors/postgres/postgres.ts

@@ -124,8 +124,19 @@ export class PostgresConnector extends AbstractConnector<PostgresConnectionOptio
 
   private async alterTablePrimaryKey(entity: EntityDefinition) {
     if (entity.options.primaryKey) {
-      const query = `ALTER TABLE "${entity.name}" ADD PRIMARY KEY (
-        ${entity.options.primaryKey.join(", ")});`;
+      const query = `
+        do $$
+        begin
+        IF NOT EXISTS 
+          (SELECT constraint_name 
+          FROM information_schema.table_constraints 
+          WHERE table_name = '${entity.name}' 
+          and constraint_type = 'PRIMARY KEY') 
+        THEN
+          ALTER TABLE "${entity.name}" ADD PRIMARY KEY (
+          ${entity.options.primaryKey.join(", ")});
+        END IF;
+        end $$;`;
       try {
         await this.client.query(query);
       } catch (err) {

tdrive/backend/node/src/services/documents/services/index.ts

@@ -1,6 +1,10 @@
 import SearchRepository from "../../../core/platform/services/search/repository";
 import { getLogger, logger, TdriveLogger } from "../../../core/platform/framework";
-import { CrudException, ListResult } from "../../../core/platform/framework/api/crud-service";
+import {
+  CrudException,
+  ExecutionContext,
+  ListResult,
+} from "../../../core/platform/framework/api/crud-service";
 import Repository, {
   comparisonType,
   inType,
@@ -111,6 +115,14 @@ export class DocumentsService {
     }
   };
 
+  userQuota = async (context: CompanyExecutionContext): Promise<number> => {
+    const children = await this.repository.find({
+      parent_id: "user_" + context.user.id,
+      company_id: context.company.id,
+    });
+    return children.getEntities().reduce((sum, child) => sum + child.size, 0);
+  };
+
   /**
    * Fetches a drive element
    *
@@ -1021,12 +1033,11 @@ export class DocumentsService {
   }
 
   getTab = async (tabId: string, context: CompanyExecutionContext): Promise<DriveTdriveTab> => {
-    const tab = await this.driveTdriveTabRepository.findOne(
+    return await this.driveTdriveTabRepository.findOne(
       { company_id: context.company.id, tab_id: tabId },
       {},
       context,
     );
-    return tab;
   };
 
   setTab = async (

tdrive/backend/node/src/services/documents/web/routes.ts

@@ -1,7 +1,7 @@
 import { FastifyInstance, FastifyPluginCallback } from "fastify";
 import { DocumentsController } from "./controllers";
 import { createDocumentSchema, createVersionSchema } from "./schemas";
-import profilerPlugin from "../../../utils/profiler";
+// import profilerPlugin from "../../../utils/profiler";
 
 const baseUrl = "/companies/:company_id";
 const serviceUrl = `${baseUrl}/item`;
@@ -9,10 +9,10 @@ const serviceUrl = `${baseUrl}/item`;
 const routes: FastifyPluginCallback = (fastify: FastifyInstance, _options, next) => {
   const documentsController = new DocumentsController();
 
-  fastify.register(profilerPlugin, {
-    active: documentsController.profilingEnabled,
-    outputDir: "profiles",
-  });
+  // fastify.register(profilerPlugin, {
+  //   active: documentsController.profilingEnabled,
+  //   outputDir: "profiles",
+  // });
 
   fastify.route({
     method: "GET",

tdrive/backend/node/src/services/files/services/index.ts

@@ -311,7 +311,5 @@ export class FileServiceImpl {
 }
 
 function getFilePath(entity: File): string {
-  return `${gr.platformServices.storage.getHomeDir()}/files/${entity.company_id}/${
-    entity.user_id || "anonymous"
-  }/${entity.id}`;
+  return `/files/${entity.company_id}/${entity.user_id || "anonymous"}/${entity.id}`;
 }

tdrive/backend/node/src/services/tags/web/routes.ts

@@ -20,7 +20,7 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next)
       { id: request.currentUser.id },
     );
 
-    const authorization = user.role === "owner" || user.role === "admin" ? true : false;
+    const authorization = user.role === "owner" || user.role === "admin";
 
     if (!authorization) {
       throw fastify.httpErrors.unauthorized("Only owner and Company Administrator have permission");

tdrive/backend/node/src/services/user/services/users/service.ts

@@ -33,6 +33,10 @@ import { getPublicUserRoom, getUserRoom } from "../../realtime";
 import NodeCache from "node-cache";
 import gr from "../../../global-resolver";
 import { formatUser } from "../../../../utils/users";
+import { getDefaultDriveItem } from "../../../documents/utils";
+import { CompanyExecutionContext } from "../../../documents/types";
+import { TYPE as DriveFileType, DriveFile } from "../../../documents/entities/drive-file";
+import config from "config";
 
 export class UserServiceImpl {
   version: "1";
@@ -40,6 +44,7 @@ export class UserServiceImpl {
   searchRepository: SearchRepository<User>;
   companyUserRepository: Repository<CompanyUser>;
   extUserRepository: Repository<ExternalUser>;
+  driveFileRepository: Repository<DriveFile>;
   private deviceRepository: Repository<Device>;
   private cache: NodeCache;
 
@@ -56,6 +61,7 @@ export class UserServiceImpl {
     );
 
     this.deviceRepository = await gr.database.getRepository<Device>(DeviceType, Device);
+    this.driveFileRepository = await gr.database.getRepository<DriveFile>(DriveFileType, DriveFile);
 
     this.cache = new NodeCache({ stdTTL: 0.2, checkperiod: 120 });
 

tdrive/backend/node/src/services/user/utils.ts

@@ -64,6 +64,9 @@ export function formatCompany(
       [CompanyFeaturesEnum.COMPANY_DISPLAY_EMAIL]: JSON.parse(
         config.get("drive.featureDisplayEmail") || "true",
       ),
+      [CompanyFeaturesEnum.COMPANY_USER_QUOTA]: JSON.parse(
+        config.get("drive.featureUserQuota") || "false",
+      ),
     },
     {
       ...(res.plan?.features || {}),

tdrive/backend/node/src/services/user/web/controller.ts

@@ -27,6 +27,7 @@ import {
   UserObject,
   UserParameters,
   CompanyUsersParameters,
+  UserQuota,
 } from "./types";
 import Company from "../entities/company";
 import CompanyUser from "../entities/company_user";
@@ -35,6 +36,7 @@ import { getCompanyRooms, getUserRooms } from "../realtime";
 import { formatCompany, getCompanyStats } from "../utils";
 import { formatUser } from "../../../utils/users";
 import gr from "../../global-resolver";
+import config from "config";
 
 export class UsersCrudController
   implements
@@ -318,6 +320,40 @@ export class UsersCrudController
       resources: [],
     };
   }
+
+  async qouta(
+    request: FastifyRequest<{
+      Params: UserParameters;
+      Querystring: CompanyUsersParameters;
+    }>,
+    _reply: FastifyReply,
+  ): Promise<UserQuota> {
+    const context = getExecutionContext(request);
+
+    let id = request.params.id;
+    if (request.params.id === "me") {
+      id = context.user.id;
+    }
+
+    if (id != context.user.id) {
+      //if admin or application wants to know user quota, it's not implemented yet
+      throw new Error("Not implemented yes");
+    }
+
+    const quota = await gr.services.documents.documents.userQuota({
+      ...context,
+      company: { id: request.query.companyId || config.get("drive.defaultCompany") },
+    });
+
+    const total: number = config.has("drive.defaultUserQuota")
+      ? config.get("drive.defaultUserQuota")
+      : NaN;
+    return {
+      total: total,
+      remaining: isNaN(total) ? NaN : total - quota,
+      used: quota,
+    } as UserQuota;
+  }
 }
 
 function getExecutionContext(request: FastifyRequest): ExecutionContext {

tdrive/backend/node/src/services/user/web/routes.ts

@@ -4,6 +4,7 @@ import {
   deleteDeviceSchema,
   getCompanySchema,
   getDevicesSchema,
+  getQuotaSchema,
   getUserCompaniesSchema,
   getUserSchema,
   getUsersSchema,
@@ -33,6 +34,15 @@ const routes: FastifyPluginCallback = (fastify: FastifyInstance, options, next)
     handler: usersController.get.bind(usersController),
   });
 
+  fastify.route({
+    method: "GET",
+    url: `${usersUrl}/:id/quota`,
+    preHandler: accessControl,
+    preValidation: fastify.authenticate,
+    schema: getQuotaSchema,
+    handler: usersController.qouta.bind(usersController),
+  });
+
   fastify.route({
     method: "POST",
     url: `${usersUrl}/me/preferences`,

tdrive/backend/node/src/services/user/web/schemas.ts

@@ -95,6 +95,7 @@ export const companyObjectSchema = {
             [CompanyFeaturesEnum.COMPANY_SEARCH_USERS]: { type: "boolean" },
             [CompanyFeaturesEnum.COMPANY_SHARED_DRIVE]: { type: "boolean" },
             [CompanyFeaturesEnum.COMPANY_DISPLAY_EMAIL]: { type: "boolean" },
+            [CompanyFeaturesEnum.COMPANY_USER_QUOTA]: { type: "boolean" },
             guests: { type: "number" }, // to rename or delete
             members: { type: "number" }, //  to rename or delete
             storage: { type: "number" }, //  to rename or delete
@@ -143,6 +144,34 @@ export const getUserSchema = {
   },
 };
 
+export const getQuotaSchema = {
+  type: "object",
+  properties: {
+    companyId: { type: "string" },
+  },
+  response: {
+    "2xx": {
+      type: "object",
+      properties: {
+        used: { type: "number" },
+        remaining: { type: "number" },
+        total: { type: "number" },
+      },
+    },
+  },
+  tags: ["User"],
+  params: {
+    type: "object",
+    description: "Users",
+    properties: {
+      id: {
+        description: "User ID",
+        type: "string",
+      },
+    },
+  },
+};
+
 export const setUserPreferencesSchema = {
   request: {
     properties: {

tdrive/backend/node/src/services/user/web/types.ts

@@ -14,7 +14,7 @@ export interface UserParameters {
 }
 
 export interface CompanyUsersParameters {
-  /* user id */
+  /* company id */
   companyId: string;
 }
 
@@ -86,6 +86,7 @@ export enum CompanyFeaturesEnum {
   COMPANY_SEARCH_USERS = "company:search_users",
   COMPANY_SHARED_DRIVE = "company:shared_drive",
   COMPANY_DISPLAY_EMAIL = "company:display_email",
+  COMPANY_USER_QUOTA = "company:user_quota",
 }
 
 export type CompanyFeaturesObject = {
@@ -98,6 +99,7 @@ export type CompanyFeaturesObject = {
   [CompanyFeaturesEnum.COMPANY_SEARCH_USERS]?: boolean;
   [CompanyFeaturesEnum.COMPANY_SHARED_DRIVE]?: boolean;
   [CompanyFeaturesEnum.COMPANY_DISPLAY_EMAIL]?: boolean;
+  [CompanyFeaturesEnum.COMPANY_USER_QUOTA]?: boolean;
 };
 
 export type CompanyLimitsObject = {
@@ -144,3 +146,9 @@ export interface RegisterDeviceParams {
 export interface DeregisterDeviceParams {
   value: "string";
 }
+
+export interface UserQuota {
+  used: number;
+  remaining: number;
+  total: number;
+}

tdrive/backend/node/test/e2e/common/common_test_helpers.ts

@@ -9,10 +9,16 @@ import { v1 as uuidv1 } from "uuid";
 import { TestDbService } from "../utils.prepare.db";
 import { DriveFile } from "../../../src/services/documents/entities/drive-file";
 import { FileVersion } from "../../../src/services/documents/entities/file-version";
-import { AccessTokenMockClass, DriveItemDetailsMockClass, SearchResultMockClass } from "./entities/mock_entities";
+import {
+    AccessTokenMockClass,
+    DriveItemDetailsMockClass,
+    SearchResultMockClass,
+    UserQuotaMockClass
+} from "./entities/mock_entities";
 import { logger } from "../../../src/core/platform/framework";
 import { expect } from "@jest/globals";
 import { publicAccessLevel } from "../../../src/services/documents/types";
+import { UserQuota } from "../../../src/services/user/web/types";
 
 export default class TestHelpers {
 
@@ -191,22 +197,23 @@ export default class TestHelpers {
       parent_id = "root",
     ) {
         const file = await this.uploadRandomFile();
-        const item = {
-            name: file.metadata.name,
-            parent_id: parent_id,
-            company_id: file.company_id,
-        };
 
-        const version = {
-            file_metadata: {
-                name: file.metadata.name,
-                size: file.upload_data?.size,
-                thumbnails: [],
-                external_id: file.id
-            }
-        }
+        const doc = await this.createDocumentFromFile(file, parent_id);
 
-        const doc = await this.createDocument(item, version);
+        expect(doc).toBeDefined();
+        expect(doc).not.toBeNull();
+        expect(doc.parent_id).toEqual(parent_id)
+
+        return doc;
+    };
+
+    async createDocumentFromFilename(
+      file_name: "sample.png",
+      parent_id = "root",
+    ) {
+        const file = await this.uploadFile(file_name);
+
+        const doc = await this.createDocumentFromFile(file, parent_id);
 
         expect(doc).toBeDefined();
         expect(doc).not.toBeNull();
@@ -338,5 +345,16 @@ export default class TestHelpers {
           response.body)
     };
 
+    async quota() {
+        const url = "/internal/services/users/v1/users";
+
+        const response =  await this.platform.app.inject({
+            method: "GET",
+            headers: { "authorization": `Bearer ${this.jwt}` },
+            url: `${url}/${this.user.id}/quota?companyId=${this.platform.workspace.company_id}`,
+        })
+
+        return deserialize<UserQuota>(UserQuotaMockClass, response.body)
+    }
 }
 

tdrive/backend/node/test/e2e/common/entities/mock_entities.ts

@@ -1,4 +1,5 @@
 import {DriveFileAccessLevel, publicAccessLevel} from "../../../../src/services/documents/types";
+import { UserQuota } from "../../../../src/services/user/web/types";
 
 export type MockAccessInformation = {
     public?: {
@@ -45,6 +46,12 @@ export class SearchResultMockClass {
     entities: DriveFileMockClass[];
 }
 
+export class UserQuotaMockClass implements UserQuota{
+    remaining: number;
+    total: number;
+    used: number;
+}
+
 export class AccessTokenMockClass {
     access_token: {
         time: 0;

tdrive/backend/node/test/e2e/documents/public-links.spec.ts

@@ -4,7 +4,6 @@ import { AccessInformation, DriveFile } from "../../../src/services/documents/en
 import { FileVersion } from "../../../src/services/documents/entities/file-version";
 import { DriveFileAccessLevel, DriveItemDetails } from "../../../src/services/documents/types";
 import { init, TestPlatform } from "../setup";
-import { TestDbService } from "../utils.prepare.db";
 import { e2e_createDocument, e2e_updateDocument } from "./utils";
 import TestHelpers from "../common/common_test_helpers";
 import { AccessTokenMockClass } from "../common/entities/mock_entities";

tdrive/backend/node/test/e2e/users/users.quota.spec.ts

@@ -0,0 +1,69 @@
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect} from "@jest/globals";
+import { init, TestPlatform } from "../setup";
+import TestHelpers from "../common/common_test_helpers";
+
+describe("The /users/quota API", () => {
+  let platform: TestPlatform;
+  let currentUser: TestHelpers;
+
+  beforeEach(async () => {
+    platform = await init();
+    currentUser = await TestHelpers.getInstance(platform);
+  }, 30000000);
+
+  afterEach(async () => {
+    await platform.tearDown();
+    platform = null;
+  });
+
+  beforeAll(async () => {
+    const platform = await init({
+      services: [
+        "database",
+        "search",
+        "message-queue",
+        "websocket",
+        "applications",
+        "webserver",
+        "user",
+        "auth",
+        "storage",
+        "counter",
+        "console",
+        "workspaces",
+        "statistics",
+        "platform-services",
+      ],
+    });
+  }, 30000000);
+
+  afterAll(async () => {
+  });
+
+
+  test("should reutrn 200 with available quota", async () => {
+    //given
+    const userQuota = 200000000;
+    const doc = await currentUser.createDocumentFromFilename("sample.png", "user_" + currentUser.user.id)
+
+    //when
+    const quota = await currentUser.quota();
+
+    expect(quota.total).toBe(userQuota);
+    expect(quota.remaining).toBe(userQuota - doc.size); //198346196 //198342406
+    expect(quota.used).toBe(doc.size);
+  }, 30000000);
+
+  test("should return 200 with all empty space", async () => {
+    //given
+    const userQuota = 200000000;
+
+    //when
+    const quota = await currentUser.quota();
+
+    expect(quota.total).toBe(userQuota);
+    expect(quota.remaining).toBe(userQuota); //198346196 //198342406
+    expect(quota.used).toBe(0);
+  });
+
+});

tdrive/backend/node/test/unit/core/services/database/services/orm/connectors/postgres/postgres.test.ts

@@ -54,7 +54,7 @@ describe('The Postgres Connector module', () => {
     expect(normalizeWhitespace(dbQuerySpy.mock.calls[2][0])).toBe(normalizeWhitespace("SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_name = $1"));
     expect(dbQuerySpy.mock.calls[2][1]).toStrictEqual(["test_table"])
     expect(normalizeWhitespace(dbQuerySpy.mock.calls[3][0])).toBe(`ALTER TABLE "test_table" ADD COLUMN id UUID, ADD COLUMN is_in_trash BOOLEAN, ADD COLUMN tags TEXT, ADD COLUMN added BIGINT`)
-    expect(normalizeWhitespace(dbQuerySpy.mock.calls[4][0])).toBe(`ALTER TABLE "test_table" ADD PRIMARY KEY ( company_id, id);`)
+    expect(normalizeWhitespace(dbQuerySpy.mock.calls[4][0])).toBe(`do $$ begin IF NOT EXISTS (SELECT constraint_name FROM information_schema.table_constraints WHERE table_name = 'test_table' and constraint_type = 'PRIMARY KEY') THEN ALTER TABLE "test_table" ADD PRIMARY KEY ( company_id, id); END IF; end $$;`)
     expect(normalizeWhitespace(dbQuerySpy.mock.calls[5][0])).toBe(`CREATE INDEX IF NOT EXISTS index_test_table_company_id_parent_id ON "test_table" ((company_id), parent_id)`)
     expect(normalizeWhitespace(dbQuerySpy.mock.calls[6][0])).toBe(`CREATE INDEX IF NOT EXISTS index_test_table_company_id_is_in_trash ON "test_table" ((company_id), is_in_trash)`)
   });