From 2d23f456459d55a0c7d87ace99bc2947a2f26e1e Mon Sep 17 00:00:00 2001 From: Yash Mittal Date: Mon, 2 Oct 2023 21:12:35 +0530 Subject: [PATCH 1/2] feat(wip): add support for onedev as a provider --- src/git/onedev.ts | 293 ++++++++++++++++++++++++++++++++++++++++ src/routes/pipelines.ts | 30 ++-- src/wrappers/onedev.ts | 17 +++ 3 files changed, 325 insertions(+), 15 deletions(-) create mode 100644 src/git/onedev.ts create mode 100644 src/wrappers/onedev.ts diff --git a/src/git/onedev.ts b/src/git/onedev.ts new file mode 100644 index 00000000..eea542c3 --- /dev/null +++ b/src/git/onedev.ts @@ -0,0 +1,293 @@ +import debug from 'debug'; +import * as crypto from "crypto" +import { Repo } from "./repo"; +import { IWebhook, IRepository, IWebhookR, IDeploykeyR } from './types'; +import gitUrlParse from 'git-url-parse'; +import axios from 'axios'; +debug('app:kubero:onedev:api') + +export class OneDevApi extends Repo { + private onedev: any; + + constructor(baseURL: string, username: string, token: string) { + super("onedev"); + this.onedev.clientOptions = { + auth: { + baseURL: baseURL, + username: username, + password: token + } + }; + } + + private async getProjectIdFromURL(oneDevUrl: string) { + const parts = oneDevUrl.split('/'); + let projectId = ''; + + for (let i = 1; i < parts.length; ++i) { + if (parts[i].startsWith('~')) return projectId.slice(1); + projectId += '/' + parts[i]; + } + + return projectId.slice(1); + } + + + protected async getRepository(gitrepo: string): Promise { + let ret: IRepository = { + status: 500, + statusText: 'error', + data: { + owner: 'unknown', + name: 'unknown', + admin: false, + push: false, + } + } + + let repo = this.getProjectIdFromURL(gitrepo) + + + let res = await axios.get(`${this.onedev.}`) + this.onedev.repos.repoGet(owner, repo) + .catch((error: any) => { + console.log(error) + return ret; + }) + + ret = { + status: res.status, + statusText: 'found', + data: { + id: res.data.id, + node_id: res.data.node_id, + name: res.data.name, + description: res.data.description, + owner: res.data.owner.login, + private: res.data.private, + ssh_url: res.data.ssh_url, + language: res.data.language, + homepage: res.data.homepage, + admin: res.data.permissions.admin, + push: res.data.permissions.push, + visibility: res.data.visibility, + default_branch: res.data.default_branch, + } + } + return ret; + + } + + protected async addWebhook(owner: string, repo: string, url: string, secret: string): Promise { + + let ret: IWebhookR = { + status: 500, + statusText: 'error', + data: { + id: 0, + active: false, + created_at: '2020-01-01T00:00:00Z', + url: '', + insecure: true, + events: [], + } + } + + //https://try.gitea.io/api/swagger#/repository/repoListHooks + const webhooksList = await this.onedev.repos.repoListHooks(owner, repo) + .catch((error: any) => { + console.log(error) + return ret; + }) + + // try to find the webhook + for (let webhook of webhooksList.data) { + if (webhook.config.url === url && + webhook.config.content_type === 'json' && + webhook.active === true) { + ret = { + status: 422, + statusText: 'found', + data: webhook, + } + return ret; + } + } + //console.log(webhooksList) + + // create the webhook since it does not exist + try { + + //https://try.gitea.io/api/swagger#/repository/repoCreateHook + let res = await this.onedev.repos.repoCreateHook(owner, repo, { + active: true, + config: { + url: url, + content_type: "json", + secret: secret, + insecure_ssl: '0' + }, + events: [ + "push", + "pull_request" + ], + type: "gitea" + }); + + ret = { + status: res.status, + statusText: 'created', + data: { + id: res.data.id, + active: res.data.active, + created_at: res.data.created_at, + url: res.data.url, + insecure: res.data.config.insecure_ssl, + events: res.data.events, + } + } + } catch (e) { + console.log(e) + } + return ret; + } + + + protected async addDeployKey(owner: string, repo: string): Promise { + + const keyPair = this.createDeployKeyPair(); + + const title: string = "bot@kubero." + crypto.randomBytes(4).toString('hex'); + + let ret: IDeploykeyR = { + status: 500, + statusText: 'error', + data: { + id: 0, + title: title, + verified: false, + created_at: '2020-01-01T00:00:00Z', + url: '', + read_only: true, + pub: keyPair.pubKeyBase64, + priv: keyPair.privKeyBase64 + } + } + + try { + //https://try.gitea.io/api/swagger#/repository/repoCreateKey + let res = await this.onedev.repos.repoCreateKey(owner, repo, { + title: title, + key: keyPair.pubKey, + read_only: true + }); + + ret = { + status: res.status, + statusText: 'created', + data: { + id: res.data.id, + title: res.data.title, + verified: res.data.verified, + created_at: res.data.created_at, + url: res.data.url, + read_only: res.data.read_only, + pub: keyPair.pubKeyBase64, + priv: keyPair.privKeyBase64 + } + } + } catch (e) { + console.log(e) + } + + return ret + } + + public getWebhook(event: string, delivery: string, signature: string, body: any): IWebhook | boolean { + //https://docs.github.com/en/developers/webhooks-and-events/webhooks/securing-your-webhooks + let secret = process.env.KUBERO_WEBHOOK_SECRET as string; + let hash = 'sha256=' + crypto.createHmac('sha256', secret).update(JSON.stringify(body, null, ' ')).digest('hex') + + let verified = false; + if (hash === signature) { + debug.debug('Gitea webhook signature is valid for event: ' + delivery); + verified = true; + } else { + debug.log('ERROR: invalid signature for event: ' + delivery); + debug.log('Hash: ' + hash); + debug.log('Signature: ' + signature); + verified = false; + return false; + } + + let branch: string = 'main'; + let ssh_url: string = ''; + let action; + if (body.pull_request == undefined) { + let ref = body.ref + let refs = ref.split('/') + branch = refs[refs.length - 1] + ssh_url = body.repository.ssh_url + } else if (body.pull_request != undefined) { + action = body.action, + branch = body.pull_request.head.ref + ssh_url = body.pull_request.head.repo.ssh_url + } else { + ssh_url = body.repository.ssh_url + } + + try { + let webhook: IWebhook = { + repoprovider: 'gitea', + action: action, + event: event, + delivery: delivery, + body: body, + branch: branch, + verified: verified, + repo: { + ssh_url: ssh_url, + } + } + + return webhook; + } catch (error) { + console.log(error) + return false; + } + } + + // public async listRepos(): Promise { + // let ret: string[] = []; + // try { + // const repos = await this.onedev.user.userCurrentListRepos() + // for (let repo of repos.data) { + // ret.push(repo.ssh_url) + // } + // } catch (error) { + // console.log(error) + // } + // return ret; + // } + + public async getBranches(gitrepo: string): Promise { + // https://try.gitea.io/api/swagger#/repository/repoListBranches + let ret: string[] = []; + + //let repo = "template-nodeapp" + //let owner = "gicara" + + let { repo, owner } = this.parseRepo(gitrepo) + try { + const branches = await this.onedev.repos.repoListBranches(owner, repo) + for (let branch of branches.data) { + ret.push(branch.name) + } + } catch (error) { + console.log(error) + } + + return ret; + } + +} diff --git a/src/routes/pipelines.ts b/src/routes/pipelines.ts index 53a0b35a..ecd23a22 100644 --- a/src/routes/pipelines.ts +++ b/src/routes/pipelines.ts @@ -1,6 +1,6 @@ import express, { Request, Response } from 'express'; import { Auth } from '../modules/auth'; -import { gitLink } from '../types'; +import { IgitLink } from '../types'; import { IApp, IPipeline } from '../types'; import { App } from '../modules/application'; import { Webhooks } from '@octokit/webhooks'; @@ -12,7 +12,7 @@ auth.init(); export const authMiddleware = auth.getAuthMiddleware(); export const bearerMiddleware = auth.getBearerMiddleware(); -Router.post('/cli/pipelines',bearerMiddleware, async function (req: Request, res: Response) { +Router.post('/cli/pipelines', bearerMiddleware, async function (req: Request, res: Response) { // #swagger.tags = ['Pipeline'] // #swagger.summary = 'Create a new pipeline' // #swagger.parameters['body'] = { in: 'body', description: 'Pipeline object', required: true, type: 'object' } @@ -25,10 +25,10 @@ Router.post('/cli/pipelines',bearerMiddleware, async function (req: Request, res }] */ const con = await req.app.locals.kubero.connectRepo( - req.body.git.repository.provider.toLowerCase(), - req.body.git.repository.ssh_url); + req.body.git.repository.provider.toLowerCase(), + req.body.git.repository.ssh_url); - let git: gitLink = { + let git: IgitLink = { keys: { priv: "Zm9v", pub: "YmFy" @@ -45,8 +45,8 @@ Router.post('/cli/pipelines',bearerMiddleware, async function (req: Request, res console.log("ERROR: connecting Gitrepository", con.error); } else { git.keys = con.keys.data, - git.webhook = con.webhook.data, - git.repository = con.repository.data + git.webhook = con.webhook.data, + git.repository = con.repository.data } const buildpackList = req.app.locals.kubero.getBuildpacks() @@ -67,7 +67,7 @@ Router.post('/cli/pipelines',bearerMiddleware, async function (req: Request, res res.send(pipeline); }); -Router.post('/pipelines',authMiddleware, async function (req: Request, res: Response) { +Router.post('/pipelines', authMiddleware, async function (req: Request, res: Response) { // #swagger.tags = ['UI'] // #swagger.summary = 'Create a new pipeline' // #swagger.parameters['body'] = { in: 'body', description: 'Pipeline object', required: true, type: 'object' } @@ -86,7 +86,7 @@ Router.post('/pipelines',authMiddleware, async function (req: Request, res: Resp }); -Router.put('/pipelines/:pipeline',authMiddleware, async function (req: Request, res: Response) { +Router.put('/pipelines/:pipeline', authMiddleware, async function (req: Request, res: Response) { // #swagger.tags = ['UI'] // #swagger.summary = 'Edit a pipeline' // #swagger.parameters['body'] = { in: 'body', description: 'Pipeline object', required: true, type: 'object' } @@ -122,9 +122,9 @@ Router.get('/pipelines', authMiddleware, async function (req: Request, res: Resp // #swagger.tags = ['UI'] // #swagger.summary = 'Get a list of available pipelines' let pipelines = await req.app.locals.kubero.listPipelines() - .catch((err: any) => { - console.log(err) - }); + .catch((err: any) => { + console.log(err) + }); res.send(pipelines); }); @@ -153,7 +153,7 @@ Router.delete('/pipelines/:pipeline', authMiddleware, async function (req: Reque // #swagger.tags = ['UI'] // #swagger.summary = 'Delete a pipeline' await req.app.locals.kubero.deletePipeline(encodeURI(req.params.pipeline)); - res.send("pipeline "+encodeURI(req.params.pipeline)+" deleted"); + res.send("pipeline " + encodeURI(req.params.pipeline) + " deleted"); }); Router.delete('/cli/pipelines/:pipeline', bearerMiddleware, async function (req: Request, res: Response) { @@ -167,7 +167,7 @@ Router.delete('/cli/pipelines/:pipeline', bearerMiddleware, async function (req: } }] */ await req.app.locals.kubero.deletePipeline(encodeURI(req.params.pipeline)); - res.send("pipeline "+encodeURI(req.params.pipeline)+" deleted"); + res.send("pipeline " + encodeURI(req.params.pipeline) + " deleted"); }); Router.delete('/pipelines/:pipeline/:phase/:app', authMiddleware, async function (req: Request, res: Response) { @@ -194,7 +194,7 @@ Router.delete('/cli/pipelines/:pipeline/:phase/:app', bearerMiddleware, async fu const phase = encodeURI(req.params.phase); const app = encodeURI(req.params.app); const response = { - message: "deleted "+pipeline+" "+phase+" "+app, + message: "deleted " + pipeline + " " + phase + " " + app, pipeline: pipeline, phase: phase, app: app diff --git a/src/wrappers/onedev.ts b/src/wrappers/onedev.ts new file mode 100644 index 00000000..22cebe1c --- /dev/null +++ b/src/wrappers/onedev.ts @@ -0,0 +1,17 @@ +export class OneDevApi { + private baseURL: string; + private username: string; + private token: string; + + constructor(baseURL: string, username: string, token: string) { + this.baseURL = baseURL; + this.username = username; + this.token = token; + } + + + public async repoGet() { + + } + +} \ No newline at end of file From 472350ceee5c693d3bfadf8070d57e7dab9521dc Mon Sep 17 00:00:00 2001 From: Yash Mittal Date: Wed, 4 Oct 2023 23:21:14 +0530 Subject: [PATCH 2/2] feat: half of the functions --- src/git/onedev.ts | 114 +++++++++++++++-------------------- src/wrappers/onedev.ts | 85 +++++++++++++++++++++++--- src/wrappers/types.onedev.ts | 20 ++++++ 3 files changed, 145 insertions(+), 74 deletions(-) create mode 100644 src/wrappers/types.onedev.ts diff --git a/src/git/onedev.ts b/src/git/onedev.ts index eea542c3..9a48056e 100644 --- a/src/git/onedev.ts +++ b/src/git/onedev.ts @@ -2,34 +2,44 @@ import debug from 'debug'; import * as crypto from "crypto" import { Repo } from "./repo"; import { IWebhook, IRepository, IWebhookR, IDeploykeyR } from './types'; -import gitUrlParse from 'git-url-parse'; import axios from 'axios'; +import { OneDevWrapper } from '../wrappers/onedev'; debug('app:kubero:onedev:api') export class OneDevApi extends Repo { - private onedev: any; + private onedev: OneDevWrapper; constructor(baseURL: string, username: string, token: string) { super("onedev"); - this.onedev.clientOptions = { - auth: { - baseURL: baseURL, - username: username, - password: token - } - }; + this.onedev = new OneDevWrapper(baseURL, username, token); } - private async getProjectIdFromURL(oneDevUrl: string) { + private async getProjectIdFromURL(oneDevUrl: string): Promise { + let projectNameWithParents = ''; const parts = oneDevUrl.split('/'); - let projectId = ''; - for (let i = 1; i < parts.length; ++i) { - if (parts[i].startsWith('~')) return projectId.slice(1); - projectId += '/' + parts[i]; + for (let i = 1; i < parts.length; ++i) { // starting from 1 since 0th element would be baseURL + if (parts[i].startsWith('~')) break; + projectNameWithParents += '/' + parts[i]; } - return projectId.slice(1); + projectNameWithParents = projectNameWithParents.slice(1); + + // getting projectIds of all the parents since there can be multiple projects with a single name + let parentId: number | null = null; + + projectNameWithParents.split('/').forEach(async (projectName: string, idx: number): Promise => { + // no need of try-catch here since the wrapper handles that + const projects = await this.onedev.getProjectsFromName(projectName, parentId); // since the parentId of a top level project is null + console.log('projects', projects); + + if (!projects || projects.length === 0) throw new Error(`Project with name ${projectName} and parentId ${parentId} not found`); + else if (projects.length > 1) throw new Error(`Multilple projects with name ${projectName} and parentId ${parentId} found, kindly provide the projectId directly.`); + + parentId = projects[0].id; + }); + + return parentId; } @@ -45,37 +55,32 @@ export class OneDevApi extends Repo { } } - let repo = this.getProjectIdFromURL(gitrepo) + const projectId = await this.getProjectIdFromURL(gitrepo); + if (projectId === null || projectId === undefined) { + ret.status = 404; + ret.statusText = 'not found'; + return ret; + }; - let res = await axios.get(`${this.onedev.}`) - this.onedev.repos.repoGet(owner, repo) - .catch((error: any) => { - console.log(error) - return ret; - }) + const projectInfo = await this.onedev.getProjectInfoByProjectId(projectId); + // TODO: Need to discuss this with kubero's maintainer and if possible review onedev's API with them to make sure we get everything we need ret = { - status: res.status, + status: 200, statusText: 'found', data: { - id: res.data.id, - node_id: res.data.node_id, - name: res.data.name, - description: res.data.description, - owner: res.data.owner.login, - private: res.data.private, - ssh_url: res.data.ssh_url, - language: res.data.language, - homepage: res.data.homepage, - admin: res.data.permissions.admin, - push: res.data.permissions.push, - visibility: res.data.visibility, - default_branch: res.data.default_branch, + id: projectId, + name: projectInfo.name, + description: projectInfo.description, + owner: this.onedev.username, + push: true, + admin: true, + default_branch: await this.onedev.getRepositoryDefaultBranch(projectId), } } - return ret; + return ret; } protected async addWebhook(owner: string, repo: string, url: string, secret: string): Promise { @@ -166,7 +171,7 @@ export class OneDevApi extends Repo { id: 0, title: title, verified: false, - created_at: '2020-01-01T00:00:00Z', + created_at: new Date().toISOString(), url: '', read_only: true, pub: keyPair.pubKeyBase64, @@ -257,37 +262,12 @@ export class OneDevApi extends Repo { } } - // public async listRepos(): Promise { - // let ret: string[] = []; - // try { - // const repos = await this.onedev.user.userCurrentListRepos() - // for (let repo of repos.data) { - // ret.push(repo.ssh_url) - // } - // } catch (error) { - // console.log(error) - // } - // return ret; - // } - public async getBranches(gitrepo: string): Promise { - // https://try.gitea.io/api/swagger#/repository/repoListBranches - let ret: string[] = []; - - //let repo = "template-nodeapp" - //let owner = "gicara" - - let { repo, owner } = this.parseRepo(gitrepo) - try { - const branches = await this.onedev.repos.repoListBranches(owner, repo) - for (let branch of branches.data) { - ret.push(branch.name) - } - } catch (error) { - console.log(error) - } + // no need of try-catch here since the wrapper takes care of that + let projectId = await this.getProjectIdFromURL(gitrepo); + if (projectId === null || projectId === undefined) throw new Error('Failed to get projectId for project'); - return ret; + return await this.onedev.getProjectBranches(projectId as number); } } diff --git a/src/wrappers/onedev.ts b/src/wrappers/onedev.ts index 22cebe1c..66e07f31 100644 --- a/src/wrappers/onedev.ts +++ b/src/wrappers/onedev.ts @@ -1,17 +1,88 @@ -export class OneDevApi { - private baseURL: string; - private username: string; - private token: string; +import axios, { Axios, AxiosResponse } from 'axios'; +import { OneDevProjectINfo } from './types.onedev'; + +export class OneDevWrapper { + public baseURL: string; + public username: string; + protected password: string; constructor(baseURL: string, username: string, token: string) { this.baseURL = baseURL; this.username = username; - this.token = token; + this.password = token; } - public async repoGet() { + public async getProjectsFromName(projectName: string, parentId: number | null | undefined = undefined): Promise> { + let projectInfo: AxiosResponse; + + try { + projectInfo = await axios.get(`${this.baseURL}/~api/projects`, { + params: { + query: `"Name" is ${projectName}`, + offset: 0, + count: 100 + }, + auth: { + username: this.username, + password: this.password + } + }); + } catch (err) { + console.error(`Error fetching project id from proect name via API call: ${err}`); + throw new Error(`Failed to get project id for project ${projectName}`); + } + + if (projectInfo.status !== 200) { + throw new Error(`Failed to get project info for project ${projectName} with status code ${projectInfo.status}}`); + } + + return projectInfo.data.filter((project: OneDevProjectINfo) => (parentId !== undefined) ? (project.parentId === parentId) : true); + } + + public async getProjectBranches(projectId: number): Promise { + let branches: AxiosResponse; + try { + branches = await axios.get(`${this.baseURL}/~api/repositories/${projectId}/branches`); + } catch (err) { + console.error('Error while fetching branches: ', err); + throw new Error(`Failed to get branches for project ${projectId}`) + } + + if (branches.status !== 200) throw new Error(`Failed to get branches for project ${projectId}`); + return branches.data as string[]; + } + + public async getProjectInfoByProjectId(projectId: number): Promise { + let repo: AxiosResponse; + try { + repo = await axios.get(`${this.baseURL}/~api/projects/${projectId}`); + } catch (err) { + console.error('Error while fetching repository: ', err); + throw new Error(`Failed to get repository for project ${projectId}`) + } + if (repo.status !== 200) throw new Error(`Failed to get repository for project ${projectId}`); + return repo.data as OneDevProjectINfo; } -} \ No newline at end of file + public async getRepositoryDefaultBranch(projectId: number): Promise { + let defaultBranchResp: AxiosResponse; + try { + defaultBranchResp = await axios.get(`${this.baseURL}/~api/repositories/${projectId}/default-branch`); + } catch (err) { + console.error('Error fetching default branch for project: ', err); + throw new Error('Error fetching default branch for project with id ' + projectId); + } + + if (defaultBranchResp.status !== 200) throw new Error('Error fetching default branch for project with id ' + projectId); + return defaultBranchResp.data as string; + } + + public async addSHHKeys(projectId: number) { + + } + + + +} diff --git a/src/wrappers/types.onedev.ts b/src/wrappers/types.onedev.ts new file mode 100644 index 00000000..987744b3 --- /dev/null +++ b/src/wrappers/types.onedev.ts @@ -0,0 +1,20 @@ +export type OneDevProjectINfo = { + id: number; + forkedFromId: number; + parentId: number; + description: string; + createDate: string; + defaultRoleId: number; + name: string; + codeManagement: boolean; + issueManagement: boolean; + gitPackConfig: { + windowMemory: string, + packSizeLimit: string, + threads: string, + window: string + }; + codeAnalysisSetting: { + analysisFiles: string + }; +}; \ No newline at end of file