🌟 Data migration tool from nextcloud (#299)
- created server configuration - implemented downloading files from nextcloud to a local directory - build of the docker image added to the GutHub workflow - Twake client added - Creating user in Twake Drive from ldap - Ldap client with ldapsearch(TODO make it work with ldapjs) - Jest debug tests
This commit is contained in:
Anton Shepilov
@@ -0,0 +1,66 @@
|
||||
import express, { Express, Request, Response } from "express";
|
||||
import { NextcloudMigration, NextcloudMigrationConfiguration } from './nextcloud_migration.js';
|
||||
|
||||
const app: Express = express();
|
||||
const port = process.env.SERVER_PORT || 3000;
|
||||
|
||||
app.use(express.json());
|
||||
app.use(express.urlencoded());
|
||||
|
||||
const config: NextcloudMigrationConfiguration = {
|
||||
ldap: {
|
||||
baseDn: process.env.LDAP_BASE!,
|
||||
url: process.env.LDAP_URL!,
|
||||
},
|
||||
tmpDir: process.env.TMP_DIR || '/tmp',
|
||||
nextcloudUrl: process.env.NEXTCLOUD_URL!,
|
||||
drive: {
|
||||
url: process.env.TWAKE_DRIVE_URL!,
|
||||
credentials: {
|
||||
appId: process.env.TWAKE_DRIVE_APP_ID!,
|
||||
secret: process.env.TWAKE_DRIVE_SECRET!,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!config.ldap.baseDn) {
|
||||
throw new Error("LDAP base has to be set")
|
||||
}
|
||||
if (!config.ldap.url) {
|
||||
throw new Error("LDAP url has to be set")
|
||||
}
|
||||
if (!config.drive.url) {
|
||||
throw new Error("Twake Drive url host has to be set")
|
||||
}
|
||||
if (!config.drive.credentials.appId) {
|
||||
throw new Error("Twake Drive application identifier host has to be set")
|
||||
}
|
||||
if (!config.nextcloudUrl) {
|
||||
throw new Error("Nextcloud url has to be set")
|
||||
}
|
||||
|
||||
app.get("/", (req: Request, res: Response) => {
|
||||
res.send("Hello, to run the the migration process you should send post request.");
|
||||
});
|
||||
|
||||
const nextcloud = new NextcloudMigration(config);
|
||||
|
||||
app.post("/", async (req: Request, res: Response) => {
|
||||
const params = req.body;
|
||||
console.log(`Got request for data synchronization with params: ${params}`)
|
||||
if (!params || !params["username"] || !params.password) {
|
||||
res.status(400).send("Username and password for nextcloud are required");
|
||||
}
|
||||
try {
|
||||
await nextcloud.migrate(params.username, params.password);
|
||||
res.status(200).send("Sync DONE ✅");
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
res.status(500).send("Error during synchronization:: " + e.message)
|
||||
}
|
||||
|
||||
});
|
||||
|
||||
app.listen(port, () => {
|
||||
console.log(`[server]: Server is running at http://localhost:${port}`);
|
||||
});
|
||||
@@ -0,0 +1,101 @@
|
||||
import ldap, { SearchEntry, SearchOptions } from 'ldapjs';
|
||||
|
||||
export type LdapConfiguration = {
|
||||
url: string,
|
||||
baseDn: string,
|
||||
}
|
||||
|
||||
export type User = {
|
||||
firstName: string,
|
||||
lastName: string,
|
||||
email: string,
|
||||
uid: string
|
||||
}
|
||||
|
||||
// Doesn't work, fix it later, somehow none of the events is called for the search request
|
||||
export class LdapUser {
|
||||
|
||||
private config: LdapConfiguration;
|
||||
|
||||
private client?: ldap.Client;
|
||||
|
||||
constructor(config: LdapConfiguration) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
async auth(username: string, password: string) {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.client?.bind(username, password, (error) => {
|
||||
if (error) {
|
||||
reject(new Error("Authentication error"));
|
||||
} else {
|
||||
console.log("Successfully authenticated in LDAP")
|
||||
resolve(this.client);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async connect() {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!this.client) {
|
||||
this.client = ldap.createClient({
|
||||
url: this.config.url,
|
||||
reconnect: true
|
||||
});
|
||||
this.client.on('connect', (res) => {
|
||||
console.log("Connected to LDAP")
|
||||
resolve(this.auth("", ""));
|
||||
})
|
||||
this.client.on('connectionError', (error) => {
|
||||
console.log("Error connecting to LDAP")
|
||||
reject(error);
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async find(username: string): Promise<User> {
|
||||
const search = await this.search(username);
|
||||
return new Promise<User>((resolve, reject) => {
|
||||
search.on('error', (err) => {
|
||||
console.log("ERROR");
|
||||
console.log(err);
|
||||
});
|
||||
search.on('searchRequest', (searchRequest) => {
|
||||
console.log('searchRequest: ', searchRequest.messageId);
|
||||
});
|
||||
search.on('searchEntry', (entry) => {
|
||||
console.log('entry: ' + JSON.stringify(entry));
|
||||
});
|
||||
search.on('searchReference', (referral) => {
|
||||
console.log('referral: ' + referral.uris.join());
|
||||
});
|
||||
search.on('end', (err) => {
|
||||
console.log("END");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async search(username: string): Promise<ldap.SearchCallbackResponse> {
|
||||
return new Promise<ldap.SearchCallbackResponse>((resolve, reject) => {
|
||||
const opts = {
|
||||
filter: `(objectClass=*)`,
|
||||
attributes: ['cn', 'sn'],
|
||||
scope: 'sub',
|
||||
} as SearchOptions;
|
||||
console.log(`Search in ${this.config.baseDn} with options`);
|
||||
// Perform search
|
||||
this.client?.search(this.config.baseDn, opts, (error, res) => {
|
||||
if (error) {
|
||||
console.error("Search error", error);
|
||||
reject(error)
|
||||
} else {
|
||||
console.log("returning search callback");
|
||||
resolve(res);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
import { exec } from 'child_process';
|
||||
import fs from 'fs';
|
||||
import { LdapUser } from './shell_ldap_user.js';
|
||||
import { User } from './ldap_user.js';
|
||||
import { TwakeDriveClient } from './twake_client.js';
|
||||
|
||||
export type NextcloudMigrationConfiguration = {
|
||||
ldap: {
|
||||
baseDn: string,
|
||||
url: string,
|
||||
},
|
||||
drive: {
|
||||
url: string,
|
||||
credentials: {
|
||||
appId: string,
|
||||
secret: string,
|
||||
}
|
||||
},
|
||||
tmpDir: string,
|
||||
nextcloudUrl: string
|
||||
}
|
||||
|
||||
export class NextcloudMigration {
|
||||
|
||||
private config: NextcloudMigrationConfiguration;
|
||||
|
||||
private ldap: LdapUser;
|
||||
|
||||
private driveClient: TwakeDriveClient;
|
||||
|
||||
constructor(config: NextcloudMigrationConfiguration) {
|
||||
this.config = config;
|
||||
this.ldap = new LdapUser(config.ldap);
|
||||
this.driveClient = new TwakeDriveClient(this.config.drive);
|
||||
}
|
||||
|
||||
async migrate(username: string, password: string) {
|
||||
const dir = this.createTmpDir(username);
|
||||
try {
|
||||
// await this.download(username, password, dir);
|
||||
const user = await this.getLDAPUser(username);
|
||||
//create user if needed Twake Drive
|
||||
await this.driveClient.createUser(user);
|
||||
//upload files to the Twake Drive
|
||||
} catch (e) {
|
||||
console.error('Error downloading files from next cloud', e);
|
||||
throw e;
|
||||
} finally {
|
||||
this.deleteDir(dir);
|
||||
}
|
||||
}
|
||||
|
||||
async download(username: string, password: string, dir: string) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let cmd = `nextcloudcmd -s --non-interactive -u '${username}' -p '${password}' ${dir} ${this.config.nextcloudUrl}`;
|
||||
console.log('Start downloading data from Nextcloud');
|
||||
exec(cmd, (error, stdout, stderr) => {
|
||||
if (stderr) {
|
||||
console.log('ERROR: ' + stderr);
|
||||
}
|
||||
if (stdout) {
|
||||
console.log('OUT: ' + stdout);
|
||||
}
|
||||
if (error) {
|
||||
console.log(`ERROR running sync for the user: ${error.message}`);
|
||||
reject(error.message);
|
||||
} else {
|
||||
console.log('Download finished');
|
||||
resolve('');
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async getLDAPUser(username: string): Promise<User> {
|
||||
const user = await this.ldap.find(username);
|
||||
if (!user.email) {
|
||||
throw new Error(`User ${username} not found`);
|
||||
}
|
||||
return user;
|
||||
}
|
||||
|
||||
createTmpDir(username: string) {
|
||||
console.log('Creating tmp directory for the user data');
|
||||
const dir = this.config.tmpDir + '/' + username + '_' + new Date().getTime();
|
||||
if (!fs.existsSync(dir)) {
|
||||
console.log(`Creating directory ${dir} ...`);
|
||||
fs.mkdirSync(dir);
|
||||
console.log(`Directory ${dir} created`);
|
||||
} else {
|
||||
this.deleteDir(dir);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
|
||||
deleteDir(dir: string) {
|
||||
console.log(`Deleting directory ${dir} ...`);
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
console.log(`Directory ${dir} deleted`);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import { LdapConfiguration, User } from './ldap_user';
|
||||
import { exec } from 'child_process';
|
||||
import ldif from 'ldif';
|
||||
|
||||
export class LdapUser {
|
||||
|
||||
private config: LdapConfiguration;
|
||||
|
||||
constructor(config: LdapConfiguration) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
async find(username: string): Promise<User> {
|
||||
return new Promise((resolve, reject) => {
|
||||
let cmd = `ldapsearch -x -H ${this.config.url} -b '${this.config.baseDn}' '(uid=${username})'`;
|
||||
console.log("Executing command to get data from LDAP for " + username);
|
||||
exec(cmd, (error, stdout, stderr) => {
|
||||
if (stderr) {
|
||||
console.log("ERROR: " + stderr);
|
||||
}
|
||||
if (error) {
|
||||
console.log(`ERROR running sync for the user: ${error.message}`);
|
||||
reject(new Error(error.message));
|
||||
} else {
|
||||
if (stdout) {
|
||||
try {
|
||||
if (stdout.lastIndexOf("# search result") > 0) {
|
||||
stdout = stdout.substring(0, stdout.lastIndexOf("# search result"))
|
||||
}
|
||||
let obj = ldif.parse(stdout).shift().toObject({});
|
||||
console.log(obj);
|
||||
resolve({
|
||||
lastName: obj.attributes.sn,
|
||||
firstName: obj.attributes.givenName,
|
||||
email: obj.attributes.mail,
|
||||
uid: obj.attributes.uid} as User);
|
||||
} catch (e) {
|
||||
console.error(e)
|
||||
resolve({ } as User);
|
||||
}
|
||||
} else {
|
||||
console.log("No user");
|
||||
resolve({ } as User);
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import axios, { AxiosError } from 'axios';
|
||||
import { User } from './ldap_user';
|
||||
|
||||
type TwakeClientConfiguration = {
|
||||
url: string,
|
||||
credentials: {
|
||||
appId: string,
|
||||
secret: string,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export class TwakeDriveClient {
|
||||
|
||||
private config: TwakeClientConfiguration;
|
||||
|
||||
constructor(config: TwakeClientConfiguration) {
|
||||
this.config = config;
|
||||
//remove the trailing '/'
|
||||
this.config.url = this.config.url.replace(/\/$/, '');
|
||||
}
|
||||
|
||||
async createUser(user: User) {
|
||||
const client = await this.client();
|
||||
try {
|
||||
const response = await client.post(this.config.url + "/api/sync",
|
||||
{
|
||||
first_name: user.firstName,
|
||||
last_name: user.lastName,
|
||||
email: user.email
|
||||
});
|
||||
return response.data;
|
||||
} catch(e){
|
||||
console.log(`Error for ${JSON.stringify(user)}: ${e.message}, body: ${e.response?.data?.message}`);
|
||||
throw e;
|
||||
};
|
||||
}
|
||||
|
||||
private async client() {
|
||||
return axios.create({
|
||||
baseURL: this.config.url,
|
||||
headers: {
|
||||
Authorization: `Bearer ${await (this.accessToken())}`,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
private async accessToken(): Promise<string> {
|
||||
try {
|
||||
const response = await axios.post<IApiServiceApplicationTokenRequestParams, { data: IApiServiceApplicationTokenResponse }>(
|
||||
`${this.config.url}/api/console/v1/login`,
|
||||
{
|
||||
id: this.config.credentials.appId,
|
||||
secret: this.config.credentials.secret,
|
||||
},
|
||||
{
|
||||
headers: {
|
||||
Authorization: `Basic ${Buffer.from(`${this.config.credentials.appId}:${this.config.credentials.secret}`).toString('base64')}`,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
const {
|
||||
resource: {
|
||||
access_token: { value },
|
||||
},
|
||||
} = response.data;
|
||||
|
||||
return value;
|
||||
} catch (error) {
|
||||
console.error('failed to get application token', error);
|
||||
console.info('Using token ', this.config.credentials.appId, this.config.credentials.secret);
|
||||
console.info(`POST ${this.config.url}/api/console/v1/login`);
|
||||
console.info(`Basic ${Buffer.from(`${this.config.credentials.appId}:${this.config.credentials.secret}`).toString('base64')}`);
|
||||
throw new Error("Unable to get access to token, see precious errors for details.");
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
type TwakeDriveUser = {
|
||||
first_name: string;
|
||||
last_name: string;
|
||||
email: string;
|
||||
}
|
||||
|
||||
interface IApiServiceApplicationTokenRequestParams {
|
||||
id: string;
|
||||
secret: string;
|
||||
}
|
||||
|
||||
interface IApiServiceApplicationTokenResponse {
|
||||
resource: {
|
||||
access_token: {
|
||||
time: number;
|
||||
expiration: number;
|
||||
value: string;
|
||||
type: string;
|
||||
};
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user