eduvia/workavia-drive
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

🐞 Add antivirus inside Twake Drive (#725)

Browse Source 
🐞 Add antivirus inside Twake Drive (#725)
This commit is contained in:
Montassar Ghanmy
2024-11-19 17:39:18 +01:00
committed by GitHub
parent cc53cd9a94
commit 00f819fb6d
49 changed files with 1341 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tdrive/backend/node/test/e2e/av/av.spec.ts
+131
View File
@@ -0,0 +1,131 @@
import "./load_test_config";
import "reflect-metadata";
import { afterAll, beforeEach, describe, expect, it, jest } from "@jest/globals";
import { init, TestPlatform } from "../setup";
import { deserialize } from "class-transformer";
import UserApi from "../common/user-api";
import { DriveItemDetailsMockClass } from "../common/entities/mock_entities";
import { DocumentsEngine } from "../../../src/services/documents/services/engine";
import { e2e_createDocumentFile, e2e_createVersion } from "../documents/utils";
import { ResourceUpdateResponse } from "../../../src/utils/types";
import { File } from "../../../src/services/files/entities/file";
import { FileVersion } from "../../../src/services/documents/entities/file-version";
describe("The documents antivirus", () => {
let platform: TestPlatform;
const notifyDocumentAVScanAlert = jest.spyOn(
DocumentsEngine.prototype,
"notifyDocumentAVScanAlert",
);
beforeEach(async () => {
platform = await init({
services: [
"webserver",
"database",
"applications",
"search",
"storage",
"message-queue",
"user",
"files",
"auth",
"statistics",
"platform-services",
"documents",
],
});
});
afterAll(async () => {
await platform?.tearDown();
// @ts-ignore
platform = null;
});
describe("On document create", () => {
it("Should scan the document and detect it as safe", async () => {
// Create an admin user
const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
const document = await oneUser.uploadFileAndCreateDocument("../../common/assets/sample.doc");
expect(document).toBeDefined();
expect(document.av_status).toBe("scanning");
await new Promise(resolve => setTimeout(resolve, 5000));
const documentResponse = await oneUser.getDocument(document.id);
const deserializedDocument = deserialize<DriveItemDetailsMockClass>(
DriveItemDetailsMockClass,
documentResponse.body,
);
expect(deserializedDocument).toBeDefined();
expect(deserializedDocument.item.av_status).toBe("safe");
});
it.skip("Should scan the document and detect it as malicious", async () => {
// Create an admin user
const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
const document = await oneUser.uploadTestMalAndCreateDocument("test-malware.txt");
expect(document).toBeDefined();
expect(document.av_status).toBe("scanning");
});
it("Should skip the scan if the document is too large", async () => {
// Create an admin user
const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
// 2.8 MB file > 1 MB limit
const document = await oneUser.uploadFileAndCreateDocument("../../common/assets/sample.mp4");
expect(document).toBeDefined();
expect(document.av_status).toBe("skipped");
expect(notifyDocumentAVScanAlert).toHaveBeenCalled();
});
});
describe("On version creation", () => {
it("Should scan the document and detect it as safe.", async () => {
// Create an admin user
const oneUser = await UserApi.getInstance(platform, true, { companyRole: "admin" });
// Create a default document for the user
const document = await oneUser.uploadFileAndCreateDocument("../../common/assets/sample.doc");
// Upload a file and deserialize the response
const fileUploadResponse = await e2e_createDocumentFile(platform);
const fileUploadResult = deserialize<ResourceUpdateResponse<File>>(
ResourceUpdateResponse,
fileUploadResponse.body,
);
console.log("🚀🚀 document:: ", document);
console.log("🚀🚀 fileUploadResponseS:: ", fileUploadResponse.body);
// Prepare metadata with the uploaded file's ID
const fileMetadata = { external_id: fileUploadResult.resource.id };
// Create a new version of the document with the uploaded file metadata
const versionResponse = await e2e_createVersion(
platform,
document.id,
{ filename: "file2", file_metadata: fileMetadata },
oneUser.jwt,
);
const versionResult = deserialize<FileVersion>(FileVersion, versionResponse.body);
expect(versionResult).toBeDefined();
console.log("🚀🚀 VERSION RESULT IS:: ", versionResponse.body);
// Retrieve the document and verify the antivirus status
const documentResponse = await oneUser.getDocument(versionResult.drive_item_id);
const deserializedDocument = deserialize<DriveItemDetailsMockClass>(
DriveItemDetailsMockClass,
documentResponse.body,
);
console.log("🚀🚀 RESP IS:: ", documentResponse.body);
// Ensure the document has been scanned and is no longer marked as "uploaded"
expect(deserializedDocument.item.av_status).not.toBe("uploaded");
});
});
});
tdrive/backend/node/test/e2e/av/config/runtime.json
+12
View File
@@ -0,0 +1,12 @@
{
"drive": {
"featureAntivirus": true
},
"av": {
"host": "av",
"port": 3310,
"debugMode": false,
"timeout": 2000,
"maxFileSize": 1048576
}
}
tdrive/backend/node/test/e2e/av/load_test_config.ts
+9
View File
@@ -0,0 +1,9 @@
// @ts-ignore
import path from "path";
// @ts-ignore
import config from "config";
// @ts-ignore
const ourConfigDir = path.join(__dirname, 'config');
let configs = config.util.loadFileConfigs(ourConfigDir);
config.util.extendDeep(config, configs);
tdrive/backend/node/test/e2e/common/entities/mock_entities.ts
+1
View File
@@ -32,6 +32,7 @@ export class DriveFileMockClass {
creator: string;
is_directory: boolean;
scope: "personal" | "shared";
av_status: string;
created_by: Record<string, any>;
shared_by: Record<string, any>;
}
tdrive/backend/node/test/e2e/common/user-api.ts
+158 -133
View File
@@ -29,7 +29,6 @@ import { Response } from "light-my-request";
* in the application.
*/
export default class UserApi {
private static readonly DOC_URL = "/internal/services/documents/v1";
static readonly ALL_FILES = [
@@ -38,7 +37,7 @@ export default class UserApi {
"sample.pdf",
"sample.doc",
"sample.zip",
"sample.mp4"
"sample.mp4",
];
platform: TestPlatform;
@@ -51,9 +50,7 @@ export default class UserApi {
api: Api;
session: string;
private constructor(
platform: TestPlatform
) {
private constructor(platform: TestPlatform) {
this.platform = platform;
}
@@ -66,12 +63,14 @@ export default class UserApi {
company_id: this.workspace.company_id,
};
this.user = await this.dbService.createUser([workspacePK], options, uuidv1());
this.anonymous = await this.dbService.createUser([workspacePK],
this.anonymous = await this.dbService.createUser(
[workspacePK],
{
...options,
identity_provider: "anonymous",
},
uuidv1());
uuidv1(),
);
} else {
this.user = this.platform.currentUser;
}
@@ -128,7 +127,7 @@ export default class UserApi {
events: {
"http://schemas.openid.net/event/backchannel-logout": {},
},
}
},
};
const verifierMock = jest.spyOn(OidcJwtVerifier.prototype, "verifyLogoutToken");
verifierMock.mockImplementation(() => {
@@ -140,28 +139,34 @@ export default class UserApi {
});
}
public static async getInstance(platform: TestPlatform, newUser = false, options?: {}): Promise<UserApi> {
public static async getInstance(
platform: TestPlatform,
newUser = false,
options?: {},
): Promise<UserApi> {
const helpers = new UserApi(platform);
await helpers.init(newUser, options);
return helpers;
}
async uploadRandomFile() {
return await this.uploadFile(UserApi.ALL_FILES[Math.floor((Math.random() * UserApi.ALL_FILES.length))]);
return await this.uploadFile(
UserApi.ALL_FILES[Math.floor(Math.random() * UserApi.ALL_FILES.length)],
);
}
private async injectUploadRequest(readable: Readable | string) {
if (typeof readable === "string")
readable = Readable.from(readable);
private async injectUploadRequest(readable: Readable | string, filename?: string) {
if (typeof readable === "string") readable = Readable.from(readable);
const url = "/internal/services/files/v1";
const form = formAutoContent({ file: readable });
form.headers["authorization"] = `Bearer ${this.jwt}`;
return await this.platform.app.inject({
method: "POST",
url: `${url}/companies/${this.platform.workspace.company_id}/files?thumbnail_sync=0`,
...form
url: `${url}/companies/${this.platform.workspace.company_id}/files?thumbnail_sync=0${
filename ? `&filename=${filename}` : ""
}`,
...form,
});
}
@@ -173,38 +178,64 @@ export default class UserApi {
if (filesUploadRaw.statusCode == 200) {
const filesUpload: ResourceUpdateResponse<File> = deserialize<ResourceUpdateResponse<File>>(
ResourceUpdateResponse,
filesUploadRaw.body
filesUploadRaw.body,
);
return filesUpload.resource;
} else this.throwServerError(filesUploadRaw.statusCode);
}
private throwServerError(code: number) {
throw new Error("Error code: " + code)
throw new Error("Error code: " + code);
}
public getJWTTokenForUser(userId: string): string {
const payload = {
sub: userId,
role: ""
role: "",
};
return this.platform.authService.sign(payload);
}
async uploadFileAndCreateDocument(
filename: string,
parent_id = "root"
) {
async uploadEicarTestFile(filename: string) {
// EICAR test file content
const eicarContent = "X5O!P%@AP[4\\PZX54(P^)7CC)7}$EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*";
// Create a readable stream from the EICAR content
const eicarStream = new Readable();
eicarStream.push(eicarContent);
eicarStream.push(null); // End of the stream
// Upload using the stream
const filesUploadRaw = await this.injectUploadRequest(eicarStream, filename);
if (filesUploadRaw.statusCode === 200) {
const filesUpload = deserialize<ResourceUpdateResponse<File>>(
ResourceUpdateResponse,
filesUploadRaw.body,
);
console.log("UPLOADED FILE IS: ", filesUpload.resource);
return filesUpload.resource;
} else {
this.throwServerError(filesUploadRaw.statusCode);
}
}
async uploadFileAndCreateDocument(filename: string, parent_id = "root") {
return this.uploadFile(filename).then(f => this.createDocumentFromFile(f, parent_id));
};
}
async uploadTestMalAndCreateDocument(filename: string, parent_id = "root") {
return this.uploadEicarTestFile(filename).then(f => this.createDocumentFromFile(f, parent_id));
}
async uploadRandomFileAndCreateDocument(parent_id = "root") {
return this.uploadRandomFile().then(f => this.createDocumentFromFile(f, parent_id));
};
}
async uploadAllFilesAndCreateDocuments(parent_id = "root") {
return await Promise.all(UserApi.ALL_FILES.map(f => this.uploadFileAndCreateDocument(f, parent_id)));
};
return await Promise.all(
UserApi.ALL_FILES.map(f => this.uploadFileAndCreateDocument(f, parent_id)),
);
}
async uploadAllFilesOneByOne(parent_id = "root") {
const files: Array<DriveFile> = [];
@@ -214,16 +245,19 @@ export default class UserApi {
files.push(doc);
}
return files;
};
}
async createDirectory(parent = "root", overrides?: Partial<DriveFile>) {
const directory = await this.createDocument({
company_id: this.platform.workspace.company_id,
name: "Test Folder Name",
parent_id: parent,
is_directory: true,
...overrides
}, {});
const directory = await this.createDocument(
{
company_id: this.platform.workspace.company_id,
name: "Test Folder Name",
parent_id: parent,
is_directory: true,
...overrides,
},
{},
);
expect(directory).toBeDefined();
expect(directory).not.toBeNull();
expect(directory.id).toBeDefined();
@@ -245,25 +279,28 @@ export default class UserApi {
}
/** Gets the public link access token then `impersonateWithJWT` as an anonymous user with that link */
async impersonatePublicLinkAccessOf<T>(item: Partial<DriveFile> & { id: string }, cb: () => Promise<T>): Promise<T> {
async impersonatePublicLinkAccessOf<T>(
item: Partial<DriveFile> & { id: string },
cb: () => Promise<T>,
): Promise<T> {
const publicToken = await this.getPublicLinkAccessToken(item);
expect(publicToken?.value?.length ?? "").toBeGreaterThan(0);
return this.impersonateWithJWT(publicToken?.value, cb);
}
async createDocument(
item: Partial<DriveFile>,
version: Partial<FileVersion>
) {
const response = await this.api.post(`${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item`,
async createDocument(item: Partial<DriveFile>, version: Partial<FileVersion>) {
const response = await this.api.post(
`${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item`,
{
item,
version
}, {
authorization: `Bearer ${this.jwt}`
});
version,
},
{
authorization: `Bearer ${this.jwt}`,
},
);
return deserialize<DriveFile>(DriveFile, response.body);
};
}
async createDefaultDocument(overrides?: Partial<DriveFile>): Promise<DriveFile> {
const scope: "personal" | "shared" = "shared";
@@ -276,28 +313,38 @@ export default class UserApi {
};
return await this.createDocument(item, {});
};
}
async shareWithPublicLink(doc: Partial<DriveFile> & { id: string }, accessLevel: publicAccessLevel) {
async shareWithPublicLink(
doc: Partial<DriveFile> & { id: string },
accessLevel: publicAccessLevel,
) {
return await this.updateDocument(doc.id, {
...doc,
access_info: {
...doc.access_info!,
public: {
...doc.access_info?.public!,
level: accessLevel
}
}
level: accessLevel,
},
},
});
}
async shareWithPublicLinkWithOkCheck(doc: Partial<DriveFile> & { id: string }, accessLevel: publicAccessLevel) {
const shareResponse = await this.shareWithPublicLink(doc, accessLevel);
async shareWithPublicLinkWithOkCheck(
doc: Partial<DriveFile> & { id: string },
accessLevel: publicAccessLevel,
) {
const shareResponse = await this.shareWithPublicLink(doc, accessLevel);
expect(shareResponse.statusCode).toBe(200);
return deserialize<DriveFile>(DriveFile, shareResponse.body);
}
async shareWithPermissions(doc: Partial<DriveFile> & { id: string }, toUserId: string, permissions: DriveFileAccessLevel) {
async shareWithPermissions(
doc: Partial<DriveFile> & { id: string },
toUserId: string,
permissions: DriveFileAccessLevel,
) {
doc.access_info.entities.push({
type: "user",
id: toUserId,
@@ -316,21 +363,19 @@ export default class UserApi {
payload: {
company_id: doc.company_id,
document_id: doc.id,
token: doc.access_info.public?.token
}
token: doc.access_info.public?.token,
},
});
const { access_token } = deserialize<AccessTokenMockClass>(
AccessTokenMockClass,
accessRes.body
accessRes.body,
);
expect(access_token).toBeDefined();
return access_token;
}
async createRandomDocument(
parent_id = "root"
) {
async createRandomDocument(parent_id = "root") {
const file = await this.uploadRandomFile();
const doc = await this.createDocumentFromFile(file, parent_id);
@@ -340,7 +385,7 @@ export default class UserApi {
expect(doc.parent_id).toEqual(parent_id);
return doc;
};
}
async createDocumentFromFilename(
file_name:
@@ -361,16 +406,13 @@ export default class UserApi {
expect(doc.parent_id).toEqual(parent_id);
return doc;
};
}
async createDocumentFromFile(
file: File,
parent_id = "root"
) {
async createDocumentFromFile(file: File, parent_id = "root") {
const item = {
name: file.metadata.name,
parent_id: parent_id,
company_id: file.company_id
company_id: file.company_id,
};
const version = {
@@ -378,35 +420,31 @@ export default class UserApi {
name: file.metadata.name,
size: file.upload_data?.size,
thumbnails: [],
external_id: file.id
}
external_id: file.id,
},
};
return await this.createDocument(item, version);
};
}
async updateDocument(
id: string | "root" | "trash" | "shared_with_me",
item: Partial<DriveFile>
) {
async updateDocument(id: string | "root" | "trash" | "shared_with_me", item: Partial<DriveFile>) {
return await this.api.post(
`${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/${id}`,
item,
{
authorization: `Bearer ${this.jwt}`
});
};
authorization: `Bearer ${this.jwt}`,
},
);
}
async beginEditingDocument(
driveFileId: string,
editorApplicationId: string,
): Promise<Response> {
async beginEditingDocument(driveFileId: string, editorApplicationId: string): Promise<Response> {
return await this.api.post(
`${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/${driveFileId}/editing_session`,
{ editorApplicationId },
{
authorization: `Bearer ${this.jwt}`
});
authorization: `Bearer ${this.jwt}`,
},
);
}
async updateEditingDocument(
@@ -415,7 +453,7 @@ export default class UserApi {
userId: string | null = null,
): Promise<Response> {
const fullPath = `${__dirname}/assets/${UserApi.ALL_FILES[0]}`;
const readable= Readable.from(fs.createReadStream(fullPath));
const readable = Readable.from(fs.createReadStream(fullPath));
const form = formAutoContent({ file: readable });
form.headers["authorization"] = `Bearer ${this.jwt}`;
let queryString = keepEditing ? "keepEditing=true" : "";
@@ -423,23 +461,23 @@ export default class UserApi {
queryString += `${queryString.length ? "&" : ""}userId=${encodeURIComponent(userId)}`;
return await this.platform.app.inject({
method: "POST",
url: `${UserApi.DOC_URL}/editing_session/${encodeURIComponent(editingSessionKey)}${queryString ? "?" : ""}${queryString}`,
url: `${UserApi.DOC_URL}/editing_session/${encodeURIComponent(editingSessionKey)}${
queryString ? "?" : ""
}${queryString}`,
headers: {
authorization: `Bearer ${this.jwt}`
authorization: `Bearer ${this.jwt}`,
},
...form,
});
}
async cancelEditingDocument(
editingSessionKey: string,
): Promise<Response> {
async cancelEditingDocument(editingSessionKey: string): Promise<Response> {
return await this.platform.app.inject({
method: "DELETE",
url: `${UserApi.DOC_URL}/editing_session/${editingSessionKey}`,
headers: {
authorization: `Bearer ${this.jwt}`
}
authorization: `Bearer ${this.jwt}`,
},
});
}
@@ -449,65 +487,56 @@ export default class UserApi {
): Promise<string> {
const result = await this.beginEditingDocument(driveFileId, editorApplicationId);
expect(result.statusCode).toBe(200);
const {editingSessionKey} = result.json();
const { editingSessionKey } = result.json();
expect(editingSessionKey).toBeTruthy();
return editingSessionKey;
}
async searchDocument(
payload: Record<string, any>
) {
async searchDocument(payload: Record<string, any>) {
const response = await this.platform.app.inject({
method: "POST",
url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/search`,
headers: {
authorization: `Bearer ${this.jwt}`
authorization: `Bearer ${this.jwt}`,
},
payload
payload,
});
return deserialize<SearchResultMockClass>(
SearchResultMockClass,
response.body);
};
return deserialize<SearchResultMockClass>(SearchResultMockClass, response.body);
}
async browseDocuments(
id: string,
payload: Record<string, any> = {}
) {
async browseDocuments(id: string, payload: Record<string, any> = {}) {
const response = await this.platform.app.inject({
method: "POST",
url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/browse/${id}`,
headers: {
authorization: `Bearer ${this.jwt}`
authorization: `Bearer ${this.jwt}`,
},
payload
payload,
});
return deserialize<DriveItemDetailsMockClass>(
DriveItemDetailsMockClass,
response.body);
};
return deserialize<DriveItemDetailsMockClass>(DriveItemDetailsMockClass, response.body);
}
async getDocument(id: string | "root" | "trash" | "shared_with_me") {
return await this.platform.app.inject({
method: "GET",
url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/${id}`,
headers: {
authorization: `Bearer ${this.jwt}`
}
authorization: `Bearer ${this.jwt}`,
},
});
};
}
async zipDocument(id: string | "root" | "trash" | "shared_with_me") {
return await this.platform.app.inject({
method: "GET",
url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/download/zip?items=${id}`,
headers: {
authorization: `Bearer ${this.jwt}`
}
authorization: `Bearer ${this.jwt}`,
},
});
};
}
async getDocumentOKCheck(id: string | "root" | "trash" | "shared_with_me") {
const response = await this.getDocument(id);
@@ -515,42 +544,38 @@ export default class UserApi {
const doc = deserialize<DriveItemDetailsMockClass>(DriveItemDetailsMockClass, response.body);
expect(doc.item?.id).toBe(id);
return doc;
};
}
async getDocumentByEditingKey(editing_session_key: string) {
return await this.platform.app.inject({
method: "GET",
url: `${UserApi.DOC_URL}/editing_session/${encodeURIComponent(editing_session_key)}`,
headers: {
authorization: `Bearer ${this.jwt}`
}
authorization: `Bearer ${this.jwt}`,
},
});
};
}
async sharedWithMeDocuments(
payload: Record<string, any>
) {
async sharedWithMeDocuments(payload: Record<string, any>) {
const response = await this.platform.app.inject({
method: "POST",
url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/browse/shared_with_me`,
headers: {
authorization: `Bearer ${this.jwt}`
authorization: `Bearer ${this.jwt}`,
},
payload
payload,
});
return deserialize<DriveItemDetailsMockClass>(
DriveItemDetailsMockClass,
response.body);
};
return deserialize<DriveItemDetailsMockClass>(DriveItemDetailsMockClass, response.body);
}
async quota() {
const url = "/internal/services/users/v1/users";
const response = await this.platform.app.inject({
method: "GET",
headers: { "authorization": `Bearer ${this.jwt}` },
url: `${url}/${this.user.id}/quota?companyId=${this.platform.workspace.company_id}`
headers: { authorization: `Bearer ${this.jwt}` },
url: `${url}/${this.user.id}/quota?companyId=${this.platform.workspace.company_id}`,
});
return deserialize<UserQuota>(UserQuotaMockClass, response.body);
@@ -560,7 +585,7 @@ export default class UserApi {
return await this.platform.app.inject({
method: "DELETE",
url: `${UserApi.DOC_URL}/companies/${this.platform.workspace.company_id}/item/${id}`,
headers: { "authorization": `Bearer ${this.jwt}` },
headers: { authorization: `Bearer ${this.jwt}` },
});
}
}
tdrive/backend/node/test/e2e/documents/documents-pagination-sorting.spec.ts
+3
View File
@@ -35,6 +35,8 @@ describe("The Documents Browser Window and API", () => {
for (const file of files) {
await currentUser.shareWithPermissions(file, anotherUser.user.id, "read");
}
// for opensearch to index the files
await new Promise(resolve => setTimeout(resolve, 3000));
});
afterAll(async () => {
@@ -147,6 +149,7 @@ describe("The Documents Browser Window and API", () => {
it("Should paginate shared with me ", async () => {
let page_token: any = "1";
const limitStr = "2";
let docs = await anotherUser.browseDocuments(sharedWIthMeFolder, {
paginate: { page_token, limitStr },
});