From b56c64269bf060feb9affde7408f048ee9098092 Mon Sep 17 00:00:00 2001 From: Jochen Klar Date: Fri, 3 Nov 2023 19:19:18 +0100 Subject: [PATCH 1/4] Add OpenProjectIssueProvider and OpenProjectProviderMixin --- pyproject.toml | 1 + rdmo/projects/models/integration.py | 10 + rdmo/projects/providers.py | 257 +++++++++++++++--- .../projects/issue_send_integrations.html | 2 +- .../projects/project_detail_integrations.html | 2 +- rdmo/services/providers.py | 92 ++++++- 6 files changed, 313 insertions(+), 51 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9665e8e7c5..d1d9384f89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ dependencies = [ "iso8601~=2.0", "markdown~=3.4", "pypandoc~=1.11", + "requests-toolbelt~=1.0", "rules~=3.3", ] diff --git a/rdmo/projects/models/integration.py b/rdmo/projects/models/integration.py index b8cdc52169..14fc689c81 100644 --- a/rdmo/projects/models/integration.py +++ b/rdmo/projects/models/integration.py @@ -36,6 +36,12 @@ def provider(self): def get_absolute_url(self): return reverse('project', kwargs={'pk': self.project.pk}) + def get_option_value(self, key): + try: + return self.options.get(key=key).value + except IntegrationOption.DoesNotExist: + return None + def save_options(self, options): for field in self.provider.fields: try: @@ -77,3 +83,7 @@ class Meta: def __str__(self): return f'{self.integration.project.title} / {self.integration.provider_key} / {self.key} = {self.value}' + + @property + def title(self): + return self.key.title().replace('_', ' ') diff --git a/rdmo/projects/providers.py b/rdmo/projects/providers.py index a891ffc8ee..9efa2933c8 100644 --- a/rdmo/projects/providers.py +++ b/rdmo/projects/providers.py @@ -2,13 +2,19 @@ import json from urllib.parse import quote +from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.http import Http404, HttpResponse, HttpResponseRedirect from django.shortcuts import render from django.utils.translation import gettext_lazy as _ from rdmo.core.plugins import Plugin -from rdmo.services.providers import GitHubProviderMixin, GitLabProviderMixin, OauthProviderMixin +from rdmo.services.providers import ( + GitHubProviderMixin, + GitLabProviderMixin, + OauthProviderMixin, + OpenProjectProviderMixin, +) class IssueProvider(Plugin): @@ -37,17 +43,13 @@ def send_issue(self, request, issue, integration, subject, message, attachments) return self.post(request, url, data) - def post_success(self, request, response): + def update_issue(self, request, remote_url): from rdmo.projects.models import Integration, Issue, IssueResource - # get the upstream url of the issue - remote_url = self.get_issue_url(response) - # get the issue_id and integration_id from the session issue_id = self.pop_from_session(request, 'issue_id') integration_id = self.pop_from_session(request, 'integration_id') - # update the issue in rdmo try: issue = Issue.objects.get(pk=issue_id) issue.status = Issue.ISSUE_STATUS_IN_PROGRESS @@ -60,6 +62,13 @@ def post_success(self, request, response): except ObjectDoesNotExist: pass + def post_success(self, request, response): + # get the upstream url of the issue + remote_url = self.get_issue_url(response) + + # update the issue in rdmo + self.update_issue(request, remote_url) + return HttpResponseRedirect(remote_url) def get_post_url(self, request, issue, integration, subject, message, attachments): @@ -78,20 +87,8 @@ class GitHubIssueProvider(GitHubProviderMixin, OauthIssueProvider): description = _('This integration allow the creation of issues in arbitrary GitHub repositories. ' 'The upload of attachments is not supported by GitHub.') - def get_repo(self, integration): - try: - return integration.options.get(key='repo').value - except ObjectDoesNotExist: - return None - - def get_secret(self, integration): - try: - return integration.options.get(key='secret').value - except ObjectDoesNotExist: - return None - def get_post_url(self, request, issue, integration, subject, message, attachments): - repo = self.get_repo(integration) + repo = integration.get_option_value('repo') if repo: return f'https://api.github.com/repos/{repo}/issues' @@ -105,7 +102,7 @@ def get_issue_url(self, response): return response.json().get('html_url') def webhook(self, request, integration): - secret = self.get_secret(integration) + secret = integration.get_option_value('secret') header_signature = request.headers.get('X-Hub-Signature') if (secret is not None) and (header_signature is not None): @@ -147,7 +144,7 @@ def fields(self): { 'key': 'secret', 'placeholder': 'Secret (random) string', - 'help': _('The secret for a GitHub webhook to close a task.'), + 'help': _('The secret for a GitHub webhook to close a task (optional).'), 'required': False, 'secret': True } @@ -163,20 +160,8 @@ def description(self): return _(f'This integration allow the creation of issues in arbitrary repositories on {self.gitlab_url}. ' 'The upload of attachments is not supported by GitLab.') - def get_repo(self, integration): - try: - return integration.options.get(key='repo').value - except ObjectDoesNotExist: - return None - - def get_secret(self, integration): - try: - return integration.options.get(key='secret').value - except ObjectDoesNotExist: - return None - def get_post_url(self, request, issue, integration, subject, message, attachments): - repo = self.get_repo(integration) + repo = integration.get_option_value('repo') if repo: return '{}/api/v4/projects/{}/issues'.format(self.gitlab_url, quote(repo, safe='')) @@ -190,7 +175,7 @@ def get_issue_url(self, response): return response.json().get('web_url') def webhook(self, request, integration): - secret = self.get_secret(integration) + secret = integration.get_option_value('secret') header_token = request.headers.get('X-Gitlab-Token') if (secret is not None) and (header_token is not None) and (header_token == secret): @@ -229,7 +214,207 @@ def fields(self): { 'key': 'secret', 'placeholder': 'Secret (random) string', - 'help': _('The secret for a GitLab webhook to close a task.'), + 'help': _('The secret for a GitLab webhook to close a task (optional).'), + 'required': False, + 'secret': True + } + ] + + +class OpenProjectIssueProvider(OpenProjectProviderMixin, OauthIssueProvider): + add_label = _('Add OpenProject integration') + send_label = _('Send to OpenProject') + + status_map = { + 'New': 'open', + 'To be scheduled': 'in_progress', + 'Scheduled': 'in_progress', + 'In progress': 'in_progress', + 'Closed': 'closed', + 'On hold': 'in_progress', + 'Rejected': 'closed' + } + + @property + def description(self): + return _(f'This integration allow the creation of issues on {self.openproject_url}.') + + def send_issue(self, request, issue, integration, subject, message, attachments): + self.store_in_session(request, 'issue_id', issue.id) + self.store_in_session(request, 'integration_id', integration.id) + self.store_in_session(request, 'project_name', integration.get_option_value('project_name')) + self.store_in_session(request, 'work_package_type', integration.get_option_value('work_package_type')) + self.store_in_session(request, 'subject', subject) + self.store_in_session(request, 'message', message) + self.store_in_session(request, 'attachments', attachments) + + return self.get_project_id(request) + + def get_project_id(self, request): + project_name = self.pop_from_session(request, 'project_name') + query = quote(json.dumps([{ + 'name_and_identifier': { + 'operator': '=', + 'values': [project_name] + } + }])) + url = f'{self.api_url}/projects?filters={query}' + + return self.get(request, url) + + def get_type_id(self, request): + url = f'{self.api_url}/types' + return self.get(request, url) + + def post_issue(self, request): + project_id = self.get_from_session(request, 'project_id') + type_id = self.get_from_session(request, 'type_id') + url = f'{self.api_url}/projects/{project_id}/work_packages' + data = { + '_links': { + 'type': { + 'href': f'/api/v3/types/{type_id}' + } + }, + 'subject': self.pop_from_session(request, 'subject'), + 'description': { + 'format': 'plain', + 'raw': self.pop_from_session(request, 'message'), + } + } + + return self.post(request, url, data) + + def post_attachment(self, request): + work_package_id = self.get_from_session(request, 'work_package_id') + attachments = self.pop_from_session(request, 'attachments') + + if attachments: + file_name, file_content, file_type = attachments[0] + url = f'{self.api_url}/work_packages/{work_package_id}/attachments' + multipart = { + 'metadata': json.dumps({'fileName': file_name }), + 'file': (file_name, file_content, file_type) + } + + self.store_in_session(request, 'attachments', attachments[1:]) + return self.post(request, url, multipart=multipart) + + else: + # there are no attachments left, get the url of the work_package + remote_url = self.get_work_package_url(work_package_id) + + # update the issue in rdmo + self.update_issue(request, remote_url) + + # redirect to the work package in open project + return HttpResponseRedirect(remote_url) + + def get_success(self, request, response): + if '/projects' in response.url: + try: + project_id = response.json()['_embedded']['elements'][0]['id'] + self.store_in_session(request, 'project_id', project_id) + return self.get_type_id(request) + + except (KeyError, IndexError): + return render(request, 'core/error.html', { + 'title': _('Integration error'), + 'errors': [_('OpenProject project could not be found.')] + }, status=200) + + elif '/types' in response.url: + try: + work_package_type = self.pop_from_session(request, 'work_package_type') + for element in response.json()['_embedded']['elements']: + if element['name'] == work_package_type: + self.store_in_session(request, 'type_id', element['id']) + return self.post_issue(request) + + except KeyError: + pass + + return render(request, 'core/error.html', { + 'title': _('Integration error'), + 'errors': [_('OpenProject work package type could not be found.')] + }, status=200) + + elif response.request.method == 'POST': + pass + + # return an error if everything failed + return render(request, 'core/error.html', { + 'title': _('Integration error'), + 'errors': [_('The Integration is not configured correctly.')] + }, status=200) + + def post_success(self, request, response): + if '/projects/' in response.url: + # get the upstream url of the issue + work_package_id = response.json()['id'] + self.store_in_session(request, 'work_package_id', work_package_id) + + # post the next attachment + return self.post_attachment(request) + + def get_work_package_url(self, work_package_id): + return f'{self.openproject_url}/work_packages/{work_package_id}' + + def webhook(self, request, integration): + secret = integration.get_option_value('secret') + header_signature = request.headers.get('X-Op-Signature') + + if (secret is not None) and (header_signature is not None): + body_signature = 'sha1=' + hmac.new(secret.encode(), request.body, 'sha1').hexdigest() + + if hmac.compare_digest(header_signature, body_signature): + try: + payload = json.loads(request.body.decode()) + action = payload.get('action') + work_package = payload.get('work_package') + + if action and work_package: + work_package_id = work_package.get('id') + work_package_url = self.get_work_package_url(work_package_id) + work_package_status = work_package.get('_links', {}).get('status', {}).get('title') + + try: + issue_resource = integration.resources.get(url=work_package_url) + status_map = self.status_map + status_map.update(settings.OPENPROJECT_PROVIDER.get('status_map', {})) + + if work_package_status in status_map: + print('-->' , status_map[work_package_status]) + issue_resource.issue.status = status_map[work_package_status] + issue_resource.issue.save() + + except ObjectDoesNotExist: + pass + + return HttpResponse(status=200) + + except json.decoder.JSONDecodeError as e: + return HttpResponse(e, status=400) + + raise Http404 + + @property + def fields(self): + return [ + { + 'key': 'project_name', + 'placeholder': 'Project', + 'help': _('The name of the OpenProject url') + }, + { + 'key': 'work_package_type', + 'placeholder': 'Work Package Type', + 'help': _('The type of workpackage to create, e.g. "Task"') + }, + { + 'key': 'secret', + 'placeholder': 'Secret (random) string', + 'help': _('The secret for a OpenProject webhook to close a task (optional).'), 'required': False, 'secret': True } diff --git a/rdmo/projects/templates/projects/issue_send_integrations.html b/rdmo/projects/templates/projects/issue_send_integrations.html index ac7d5e8137..92a43b97c7 100644 --- a/rdmo/projects/templates/projects/issue_send_integrations.html +++ b/rdmo/projects/templates/projects/issue_send_integrations.html @@ -18,7 +18,7 @@

{% trans 'Send via integration' %}

{% for option in integration.options.all %} {% if not option.secret %} -

{{ option.key.title }}: {{ option.value }}

+

{{ option.title }}: {{ option.value }}

{% endif %} {% endfor %} diff --git a/rdmo/projects/templates/projects/project_detail_integrations.html b/rdmo/projects/templates/projects/project_detail_integrations.html index a48d2ce852..08e5f34ad7 100644 --- a/rdmo/projects/templates/projects/project_detail_integrations.html +++ b/rdmo/projects/templates/projects/project_detail_integrations.html @@ -32,7 +32,7 @@

{% trans 'Integrations' %}

{% for option in integration.options.all %} {% if not option.secret %} - {{ option.key.title }}: {{ option.value }}
+ {{ option.title }}: {{ option.value }}
{% endif %} {% endfor %} diff --git a/rdmo/services/providers.py b/rdmo/services/providers.py index 41b6530224..8819f60860 100644 --- a/rdmo/services/providers.py +++ b/rdmo/services/providers.py @@ -9,6 +9,7 @@ from django.utils.translation import gettext_lazy as _ import requests +from requests_toolbelt.multipart.encoder import MultipartEncoder logger = logging.getLogger(__name__) @@ -25,14 +26,14 @@ def get(self, request, url): response = requests.get(url, headers=self.get_authorization_headers(access_token)) if response.status_code == 401: - logger.warn('get forbidden: %s (%s)', response.content, response.status_code) + logger.warning('get forbidden: %s (%s)', response.content, response.status_code) else: try: response.raise_for_status() return self.get_success(request, response) except requests.HTTPError: - logger.warn('get error: %s (%s)', response.content, response.status_code) + logger.warning('get error: %s (%s)', response.content, response.status_code) return render(request, 'core/error.html', { 'title': _('OAuth error'), @@ -40,27 +41,35 @@ def get(self, request, url): }, status=200) # if the above did not work authorize first - self.store_in_session(request, 'request', ('get', url, {})) + self.store_in_session(request, 'request', ('get', url)) return self.authorize(request) - def post(self, request, url, data): + def post(self, request, url, json=None, files=None, multipart=None): # get access token from the session access_token = self.get_from_session(request, 'access_token') if access_token: # if the access_token is available post to the upstream service - logger.debug('post: %s %s', url, data) - - response = requests.post(url, json=data, headers=self.get_authorization_headers(access_token)) + logger.debug('post: %s %s', url, json, files) + + if multipart is not None: + multipart_encoder = MultipartEncoder(fields=multipart) + headers = self.get_authorization_headers(access_token) + headers['Content-Type'] = multipart_encoder.content_type + response = requests.post(url, data=multipart_encoder, headers=headers) + elif files is not None: + response = requests.post(url, files=files, headers=self.get_authorization_headers(access_token)) + else: + response = requests.post(url, json=json, headers=self.get_authorization_headers(access_token)) if response.status_code == 401: - logger.warn('post forbidden: %s (%s)', response.content, response.status_code) + logger.warning('post forbidden: %s (%s)', response.content, response.status_code) else: try: response.raise_for_status() return self.post_success(request, response) except requests.HTTPError: - logger.warn('post error: %s (%s)', response.content, response.status_code) + logger.warning('post error: %s (%s)', response.content, response.status_code) return render(request, 'core/error.html', { 'title': _('OAuth error'), @@ -68,7 +77,7 @@ def post(self, request, url, data): }, status=200) # if the above did not work authorize first - self.store_in_session(request, 'request', ('post', url, data)) + self.store_in_session(request, 'request', ('post', url, json, files, multipart)) return self.authorize(request) def authorize(self, request): @@ -105,11 +114,11 @@ def callback(self, request): # get post data from session try: - method, url, data = self.pop_from_session(request, 'request') + method, *args = self.pop_from_session(request, 'request') if method == 'get': - return self.get(request, url) + return self.get(request, *args) elif method == 'post': - return self.post(request, url, data) + return self.post(request, *args) except ValueError: pass @@ -201,6 +210,9 @@ def get_callback_params(self, request): 'code': request.GET.get('code') } + def get_error_message(self, response): + return response.json().get('message') + class GitLabProviderMixin(OauthProviderMixin): @@ -251,3 +263,57 @@ def get_callback_params(self, request): 'grant_type': 'authorization_code', 'redirect_uri': request.build_absolute_uri(self.redirect_path) } + + +class OpenProjectProviderMixin(OauthProviderMixin): + + @property + def openproject_url(self): + return settings.OPENPROJECT_PROVIDER['openproject_url'].strip('/') + + @property + def authorize_url(self): + return f'{self.openproject_url}/oauth/authorize' + + @property + def token_url(self): + return f'{self.openproject_url}/oauth/token' + + @property + def api_url(self): + return f'{self.openproject_url}/api/v3' + + @property + def client_id(self): + return settings.OPENPROJECT_PROVIDER['client_id'] + + @property + def client_secret(self): + return settings.OPENPROJECT_PROVIDER['client_secret'] + + @property + def redirect_path(self): + return reverse('oauth_callback', args=['openproject']) + + def get_authorize_params(self, request, state): + return { + 'authorize_url': self.authorize_url, + 'client_id': self.client_id, + 'redirect_uri': request.build_absolute_uri(self.redirect_path), + 'response_type': 'code', + 'scope': 'api_v3', + 'state': state, + } + + def get_callback_data(self, request): + return { + 'token_url': self.token_url, + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'code': request.GET.get('code'), + 'grant_type': 'authorization_code', + 'redirect_uri': request.build_absolute_uri(self.redirect_path) + } + + def get_error_message(self, response): + return response.json().get('message') From a9ee9306e12873785afb06221b38a17ccead8415 Mon Sep 17 00:00:00 2001 From: Jochen Klar Date: Thu, 16 Nov 2023 19:41:34 +0100 Subject: [PATCH 2/4] Move GitHubIssueProvider, GitLabIssueProvider, and OpenProjectIssueProvider to seperate repositories --- pyproject.toml | 1 - rdmo/projects/forms.py | 4 +- rdmo/projects/providers.py | 356 +------------------------------------ rdmo/services/providers.py | 151 ---------------- 4 files changed, 6 insertions(+), 506 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d1d9384f89..9665e8e7c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,6 @@ dependencies = [ "iso8601~=2.0", "markdown~=3.4", "pypandoc~=1.11", - "requests-toolbelt~=1.0", "rules~=3.3", ] diff --git a/rdmo/projects/forms.py b/rdmo/projects/forms.py index 4ac7d42ab2..c1ff1145a1 100644 --- a/rdmo/projects/forms.py +++ b/rdmo/projects/forms.py @@ -278,8 +278,10 @@ def __init__(self, *args, **kwargs): if field.get('placeholder'): attrs = {'placeholder': field.get('placeholder')} + self.fields[field.get('key')] = forms.CharField(widget=forms.TextInput(attrs=attrs), - initial=initial, required=field.get('required', True)) + initial=initial, required=field.get('required', True), + help_text=field.get('help')) def save(self): # the the project and the provider_key diff --git a/rdmo/projects/providers.py b/rdmo/projects/providers.py index 9efa2933c8..1c11db028a 100644 --- a/rdmo/projects/providers.py +++ b/rdmo/projects/providers.py @@ -1,20 +1,10 @@ -import hmac -import json -from urllib.parse import quote - -from django.conf import settings from django.core.exceptions import ObjectDoesNotExist -from django.http import Http404, HttpResponse, HttpResponseRedirect +from django.http import HttpResponseRedirect from django.shortcuts import render from django.utils.translation import gettext_lazy as _ from rdmo.core.plugins import Plugin -from rdmo.services.providers import ( - GitHubProviderMixin, - GitLabProviderMixin, - OauthProviderMixin, - OpenProjectProviderMixin, -) +from rdmo.services.providers import OauthProviderMixin class IssueProvider(Plugin): @@ -38,7 +28,7 @@ def send_issue(self, request, issue, integration, subject, message, attachments) if url is None or data is None: return render(request, 'core/error.html', { 'title': _('Integration error'), - 'errors': [_('The Integration is not configured correctly.') % message] + 'errors': [_('The Integration is not configured correctly.')] }, status=200) return self.post(request, url, data) @@ -79,343 +69,3 @@ def get_post_data(self, request, issue, integration, subject, message, attachmen def get_issue_url(self, response): raise NotImplementedError - - -class GitHubIssueProvider(GitHubProviderMixin, OauthIssueProvider): - add_label = _('Add GitHub integration') - send_label = _('Send to GitHub') - description = _('This integration allow the creation of issues in arbitrary GitHub repositories. ' - 'The upload of attachments is not supported by GitHub.') - - def get_post_url(self, request, issue, integration, subject, message, attachments): - repo = integration.get_option_value('repo') - if repo: - return f'https://api.github.com/repos/{repo}/issues' - - def get_post_data(self, request, issue, integration, subject, message, attachments): - return { - 'title': subject, - 'body': message - } - - def get_issue_url(self, response): - return response.json().get('html_url') - - def webhook(self, request, integration): - secret = integration.get_option_value('secret') - header_signature = request.headers.get('X-Hub-Signature') - - if (secret is not None) and (header_signature is not None): - body_signature = 'sha1=' + hmac.new(secret.encode(), request.body, 'sha1').hexdigest() - - if hmac.compare_digest(header_signature, body_signature): - try: - payload = json.loads(request.body.decode()) - action = payload.get('action') - issue_url = payload.get('issue', {}).get('html_url') - - if action and issue_url: - try: - issue_resource = integration.resources.get(url=issue_url) - if action == 'closed': - issue_resource.issue.status = issue_resource.issue.ISSUE_STATUS_CLOSED - else: - issue_resource.issue.status = issue_resource.issue.ISSUE_STATUS_IN_PROGRESS - - issue_resource.issue.save() - except ObjectDoesNotExist: - pass - - return HttpResponse(status=200) - - except json.decoder.JSONDecodeError as e: - return HttpResponse(e, status=400) - - raise Http404 - - @property - def fields(self): - return [ - { - 'key': 'repo', - 'placeholder': 'user_name/repo_name', - 'help': _('The GitHub repository to send issues to.') - }, - { - 'key': 'secret', - 'placeholder': 'Secret (random) string', - 'help': _('The secret for a GitHub webhook to close a task (optional).'), - 'required': False, - 'secret': True - } - ] - - -class GitLabIssueProvider(GitLabProviderMixin, OauthIssueProvider): - add_label = _('Add GitLab integration') - send_label = _('Send to GitLab') - - @property - def description(self): - return _(f'This integration allow the creation of issues in arbitrary repositories on {self.gitlab_url}. ' - 'The upload of attachments is not supported by GitLab.') - - def get_post_url(self, request, issue, integration, subject, message, attachments): - repo = integration.get_option_value('repo') - if repo: - return '{}/api/v4/projects/{}/issues'.format(self.gitlab_url, quote(repo, safe='')) - - def get_post_data(self, request, issue, integration, subject, message, attachments): - return { - 'title': subject, - 'description': message - } - - def get_issue_url(self, response): - return response.json().get('web_url') - - def webhook(self, request, integration): - secret = integration.get_option_value('secret') - header_token = request.headers.get('X-Gitlab-Token') - - if (secret is not None) and (header_token is not None) and (header_token == secret): - try: - payload = json.loads(request.body.decode()) - state = payload.get('object_attributes', {}).get('state') - issue_url = payload.get('object_attributes', {}).get('url') - - if state and issue_url: - try: - issue_resource = integration.resources.get(url=issue_url) - if state == 'closed': - issue_resource.issue.status = issue_resource.issue.ISSUE_STATUS_CLOSED - else: - issue_resource.issue.status = issue_resource.issue.ISSUE_STATUS_IN_PROGRESS - - issue_resource.issue.save() - except ObjectDoesNotExist: - pass - - return HttpResponse(status=200) - - except json.decoder.JSONDecodeError as e: - return HttpResponse(e, status=400) - - raise Http404 - - @property - def fields(self): - return [ - { - 'key': 'repo', - 'placeholder': 'user_name/repo_name', - 'help': _('The GitLab repository to send issues to.') - }, - { - 'key': 'secret', - 'placeholder': 'Secret (random) string', - 'help': _('The secret for a GitLab webhook to close a task (optional).'), - 'required': False, - 'secret': True - } - ] - - -class OpenProjectIssueProvider(OpenProjectProviderMixin, OauthIssueProvider): - add_label = _('Add OpenProject integration') - send_label = _('Send to OpenProject') - - status_map = { - 'New': 'open', - 'To be scheduled': 'in_progress', - 'Scheduled': 'in_progress', - 'In progress': 'in_progress', - 'Closed': 'closed', - 'On hold': 'in_progress', - 'Rejected': 'closed' - } - - @property - def description(self): - return _(f'This integration allow the creation of issues on {self.openproject_url}.') - - def send_issue(self, request, issue, integration, subject, message, attachments): - self.store_in_session(request, 'issue_id', issue.id) - self.store_in_session(request, 'integration_id', integration.id) - self.store_in_session(request, 'project_name', integration.get_option_value('project_name')) - self.store_in_session(request, 'work_package_type', integration.get_option_value('work_package_type')) - self.store_in_session(request, 'subject', subject) - self.store_in_session(request, 'message', message) - self.store_in_session(request, 'attachments', attachments) - - return self.get_project_id(request) - - def get_project_id(self, request): - project_name = self.pop_from_session(request, 'project_name') - query = quote(json.dumps([{ - 'name_and_identifier': { - 'operator': '=', - 'values': [project_name] - } - }])) - url = f'{self.api_url}/projects?filters={query}' - - return self.get(request, url) - - def get_type_id(self, request): - url = f'{self.api_url}/types' - return self.get(request, url) - - def post_issue(self, request): - project_id = self.get_from_session(request, 'project_id') - type_id = self.get_from_session(request, 'type_id') - url = f'{self.api_url}/projects/{project_id}/work_packages' - data = { - '_links': { - 'type': { - 'href': f'/api/v3/types/{type_id}' - } - }, - 'subject': self.pop_from_session(request, 'subject'), - 'description': { - 'format': 'plain', - 'raw': self.pop_from_session(request, 'message'), - } - } - - return self.post(request, url, data) - - def post_attachment(self, request): - work_package_id = self.get_from_session(request, 'work_package_id') - attachments = self.pop_from_session(request, 'attachments') - - if attachments: - file_name, file_content, file_type = attachments[0] - url = f'{self.api_url}/work_packages/{work_package_id}/attachments' - multipart = { - 'metadata': json.dumps({'fileName': file_name }), - 'file': (file_name, file_content, file_type) - } - - self.store_in_session(request, 'attachments', attachments[1:]) - return self.post(request, url, multipart=multipart) - - else: - # there are no attachments left, get the url of the work_package - remote_url = self.get_work_package_url(work_package_id) - - # update the issue in rdmo - self.update_issue(request, remote_url) - - # redirect to the work package in open project - return HttpResponseRedirect(remote_url) - - def get_success(self, request, response): - if '/projects' in response.url: - try: - project_id = response.json()['_embedded']['elements'][0]['id'] - self.store_in_session(request, 'project_id', project_id) - return self.get_type_id(request) - - except (KeyError, IndexError): - return render(request, 'core/error.html', { - 'title': _('Integration error'), - 'errors': [_('OpenProject project could not be found.')] - }, status=200) - - elif '/types' in response.url: - try: - work_package_type = self.pop_from_session(request, 'work_package_type') - for element in response.json()['_embedded']['elements']: - if element['name'] == work_package_type: - self.store_in_session(request, 'type_id', element['id']) - return self.post_issue(request) - - except KeyError: - pass - - return render(request, 'core/error.html', { - 'title': _('Integration error'), - 'errors': [_('OpenProject work package type could not be found.')] - }, status=200) - - elif response.request.method == 'POST': - pass - - # return an error if everything failed - return render(request, 'core/error.html', { - 'title': _('Integration error'), - 'errors': [_('The Integration is not configured correctly.')] - }, status=200) - - def post_success(self, request, response): - if '/projects/' in response.url: - # get the upstream url of the issue - work_package_id = response.json()['id'] - self.store_in_session(request, 'work_package_id', work_package_id) - - # post the next attachment - return self.post_attachment(request) - - def get_work_package_url(self, work_package_id): - return f'{self.openproject_url}/work_packages/{work_package_id}' - - def webhook(self, request, integration): - secret = integration.get_option_value('secret') - header_signature = request.headers.get('X-Op-Signature') - - if (secret is not None) and (header_signature is not None): - body_signature = 'sha1=' + hmac.new(secret.encode(), request.body, 'sha1').hexdigest() - - if hmac.compare_digest(header_signature, body_signature): - try: - payload = json.loads(request.body.decode()) - action = payload.get('action') - work_package = payload.get('work_package') - - if action and work_package: - work_package_id = work_package.get('id') - work_package_url = self.get_work_package_url(work_package_id) - work_package_status = work_package.get('_links', {}).get('status', {}).get('title') - - try: - issue_resource = integration.resources.get(url=work_package_url) - status_map = self.status_map - status_map.update(settings.OPENPROJECT_PROVIDER.get('status_map', {})) - - if work_package_status in status_map: - print('-->' , status_map[work_package_status]) - issue_resource.issue.status = status_map[work_package_status] - issue_resource.issue.save() - - except ObjectDoesNotExist: - pass - - return HttpResponse(status=200) - - except json.decoder.JSONDecodeError as e: - return HttpResponse(e, status=400) - - raise Http404 - - @property - def fields(self): - return [ - { - 'key': 'project_name', - 'placeholder': 'Project', - 'help': _('The name of the OpenProject url') - }, - { - 'key': 'work_package_type', - 'placeholder': 'Work Package Type', - 'help': _('The type of workpackage to create, e.g. "Task"') - }, - { - 'key': 'secret', - 'placeholder': 'Secret (random) string', - 'help': _('The secret for a OpenProject webhook to close a task (optional).'), - 'required': False, - 'secret': True - } - ] diff --git a/rdmo/services/providers.py b/rdmo/services/providers.py index 8819f60860..d8262161dd 100644 --- a/rdmo/services/providers.py +++ b/rdmo/services/providers.py @@ -1,10 +1,8 @@ import logging from urllib.parse import urlencode -from django.conf import settings from django.http import HttpResponseRedirect from django.shortcuts import render -from django.urls import reverse from django.utils.crypto import get_random_string from django.utils.translation import gettext_lazy as _ @@ -168,152 +166,3 @@ def get_callback_data(self, request): def get_error_message(self, response): return response.json().get('error') - - -class GitHubProviderMixin(OauthProviderMixin): - authorize_url = 'https://github.com/login/oauth/authorize' - token_url = 'https://github.com/login/oauth/access_token' - api_url = 'https://api.github.com' - - @property - def client_id(self): - return settings.GITHUB_PROVIDER['client_id'] - - @property - def client_secret(self): - return settings.GITHUB_PROVIDER['client_secret'] - - @property - def redirect_path(self): - return reverse('oauth_callback', args=['github']) - - def get_authorization_headers(self, access_token): - return { - 'Authorization': f'token {access_token}', - 'Accept': 'application/vnd.github.v3+json' - } - - def get_authorize_params(self, request, state): - return { - 'authorize_url': self.authorize_url, - 'client_id': self.client_id, - 'redirect_uri': request.build_absolute_uri(self.redirect_path), - 'scope': 'repo', - 'state': state, - } - - def get_callback_params(self, request): - return { - 'token_url': self.token_url, - 'client_id': self.client_id, - 'client_secret': self.client_secret, - 'code': request.GET.get('code') - } - - def get_error_message(self, response): - return response.json().get('message') - - -class GitLabProviderMixin(OauthProviderMixin): - - @property - def gitlab_url(self): - return settings.GITLAB_PROVIDER['gitlab_url'].strip('/') - - @property - def authorize_url(self): - return f'{self.gitlab_url}/oauth/authorize' - - @property - def token_url(self): - return f'{self.gitlab_url}/oauth/token' - - @property - def api_url(self): - return f'{self.gitlab_url}/api/v4' - - @property - def client_id(self): - return settings.GITLAB_PROVIDER['client_id'] - - @property - def client_secret(self): - return settings.GITLAB_PROVIDER['client_secret'] - - @property - def redirect_path(self): - return reverse('oauth_callback', args=['gitlab']) - - def get_authorize_params(self, request, state): - return { - 'authorize_url': self.authorize_url, - 'client_id': self.client_id, - 'redirect_uri': request.build_absolute_uri(self.redirect_path), - 'response_type': 'code', - 'scope': 'api', - 'state': state, - } - - def get_callback_params(self, request): - return { - 'token_url': self.token_url, - 'client_id': self.client_id, - 'client_secret': self.client_secret, - 'code': request.GET.get('code'), - 'grant_type': 'authorization_code', - 'redirect_uri': request.build_absolute_uri(self.redirect_path) - } - - -class OpenProjectProviderMixin(OauthProviderMixin): - - @property - def openproject_url(self): - return settings.OPENPROJECT_PROVIDER['openproject_url'].strip('/') - - @property - def authorize_url(self): - return f'{self.openproject_url}/oauth/authorize' - - @property - def token_url(self): - return f'{self.openproject_url}/oauth/token' - - @property - def api_url(self): - return f'{self.openproject_url}/api/v3' - - @property - def client_id(self): - return settings.OPENPROJECT_PROVIDER['client_id'] - - @property - def client_secret(self): - return settings.OPENPROJECT_PROVIDER['client_secret'] - - @property - def redirect_path(self): - return reverse('oauth_callback', args=['openproject']) - - def get_authorize_params(self, request, state): - return { - 'authorize_url': self.authorize_url, - 'client_id': self.client_id, - 'redirect_uri': request.build_absolute_uri(self.redirect_path), - 'response_type': 'code', - 'scope': 'api_v3', - 'state': state, - } - - def get_callback_data(self, request): - return { - 'token_url': self.token_url, - 'client_id': self.client_id, - 'client_secret': self.client_secret, - 'code': request.GET.get('code'), - 'grant_type': 'authorization_code', - 'redirect_uri': request.build_absolute_uri(self.redirect_path) - } - - def get_error_message(self, response): - return response.json().get('message') From 599913ec3adb3a9ca6ac691c02679f20cebcd328 Mon Sep 17 00:00:00 2001 From: Jochen Klar Date: Fri, 17 Nov 2023 11:27:56 +0100 Subject: [PATCH 3/4] Fix pyproject.toml --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9665e8e7c5..d1d9384f89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ dependencies = [ "iso8601~=2.0", "markdown~=3.4", "pypandoc~=1.11", + "requests-toolbelt~=1.0", "rules~=3.3", ] From ff08f121e594ecac4d2f01e1bd27230aad2475e7 Mon Sep 17 00:00:00 2001 From: Jochen Klar Date: Fri, 17 Nov 2023 12:43:45 +0100 Subject: [PATCH 4/4] Add SimpleIssueProvider and fix tests for integrations --- rdmo/projects/providers.py | 64 ++++++++++++++++++- rdmo/projects/tests/test_view_integration.py | 49 +++++++++----- rdmo/projects/tests/test_view_issue.py | 12 +++- .../tests/test_viewset_project_integration.py | 30 ++++----- testing/config/settings/base.py | 7 +- testing/fixtures/projects.json | 14 ++-- 6 files changed, 126 insertions(+), 50 deletions(-) diff --git a/rdmo/projects/providers.py b/rdmo/projects/providers.py index 1c11db028a..267706bfc7 100644 --- a/rdmo/projects/providers.py +++ b/rdmo/projects/providers.py @@ -1,5 +1,7 @@ +import json + from django.core.exceptions import ObjectDoesNotExist -from django.http import HttpResponseRedirect +from django.http import Http404, HttpResponse, HttpResponseRedirect from django.shortcuts import render from django.utils.translation import gettext_lazy as _ @@ -12,7 +14,7 @@ class IssueProvider(Plugin): def send_issue(self, request, issue, integration, subject, message, attachments): raise NotImplementedError - def webhook(self, request, options, payload): + def webhook(self, request, integration): raise NotImplementedError @@ -69,3 +71,61 @@ def get_post_data(self, request, issue, integration, subject, message, attachmen def get_issue_url(self, response): raise NotImplementedError + + +class SimpleIssueProvider(OauthIssueProvider): + + add_label = _('Add Simple integration') + send_label = _('Send to Simple') + description = _('This integration allow the creation of issues in arbitrary Simple repositories. ' + 'The upload of attachments is not supported.') + + @property + def fields(self): + return [ + { + 'key': 'project_url', + 'placeholder': 'https://example.com/projects/', + 'help': _('The URL of the project to send tasks to.') + }, + { + 'key': 'secret', + 'placeholder': 'Secret (random) string', + 'help': _('The secret for a webhook to close a task (optional).'), + 'required': False, + 'secret': True + } + ] + + def get(self, request, url): + raise NotImplementedError + + def post(self, request, url, json=None, files=None, multipart=None): + raise NotImplementedError + + def webhook(self, request, integration): + secret = integration.get_option_value('secret') + header_signature = request.headers.get('X-Secret') + if secret == header_signature: + try: + payload = json.loads(request.body.decode()) + except json.decoder.JSONDecodeError as e: + return HttpResponse(e, status=400) + + action = payload.get('action') + url = payload.get('url') + + try: + issue_resource = integration.resources.get(url=url) + if action == 'closed': + issue_resource.issue.status = issue_resource.issue.ISSUE_STATUS_CLOSED + else: + issue_resource.issue.status = issue_resource.issue.ISSUE_STATUS_IN_PROGRESS + + issue_resource.issue.save() + except ObjectDoesNotExist: + pass + + return HttpResponse(status=200) + + raise Http404 diff --git a/rdmo/projects/tests/test_view_integration.py b/rdmo/projects/tests/test_view_integration.py index fe626862c0..cb20690f8d 100644 --- a/rdmo/projects/tests/test_view_integration.py +++ b/rdmo/projects/tests/test_view_integration.py @@ -1,6 +1,3 @@ -import hmac -import json - import pytest from django.urls import reverse @@ -42,7 +39,7 @@ def test_integration_create_get(db, client, username, password, project_id): client.login(username=username, password=password) - url = reverse('integration_create', args=[project_id, 'github']) + url = reverse('integration_create', args=[project_id, 'simple']) response = client.get(url) if project_id in add_integration_permission_map.get(username, []): @@ -58,9 +55,9 @@ def test_integration_create_get(db, client, username, password, project_id): def test_integration_create_post(db, client, username, password, project_id): client.login(username=username, password=password) - url = reverse('integration_create', args=[project_id, 'github']) + url = reverse('integration_create', args=[project_id, 'simple']) data = { - 'repo': 'example/example1' + 'project_url': 'https://example.com/projects/1' } response = client.post(url, data) @@ -69,8 +66,8 @@ def test_integration_create_post(db, client, username, password, project_id): values = Integration.objects.order_by('id').last().options.values('key', 'value', 'secret') assert sorted(values, key=lambda obj: obj['key']) == [ { - 'key': 'repo', - 'value': 'example/example1', + 'key': 'project_url', + 'value': 'https://example.com/projects/1', 'secret': False }, { @@ -115,7 +112,7 @@ def test_integration_update_post(db, client, username, password, project_id, int url = reverse('integration_update', args=[project_id, integration_id]) data = { - 'repo': 'example/example2', + 'project_url': 'https://example.com/projects/2', 'secret': 'super_secret' } response = client.post(url, data) @@ -127,8 +124,8 @@ def test_integration_update_post(db, client, username, password, project_id, int .options.values('key', 'value', 'secret') assert sorted(values, key=lambda obj: obj['key']) == [ { - 'key': 'repo', - 'value': 'example/example2', + 'key': 'project_url', + 'value': 'https://example.com/projects/2', 'secret': False }, { @@ -207,14 +204,10 @@ def test_integration_webhook_post(db, client, project_id, integration_id): url = reverse('integration_webhook', args=[project_id, integration_id]) data = { 'action': 'closed', - 'issue': { - 'html_url': 'https://github.com/example/example/issues/1' - } + 'url': 'https://simple.example.com/issues/1' } - body = json.dumps(data) - signature = 'sha1=' + hmac.new(secret.encode(), body.encode(), 'sha1').hexdigest() - response = client.post(url, data, **{'HTTP_X_HUB_SIGNATURE': signature, 'content_type': 'application/json'}) + response = client.post(url, data, **{'HTTP_X_SECRET': secret, 'content_type': 'application/json'}) if integration: assert response.status_code == 200 @@ -224,6 +217,28 @@ def test_integration_webhook_post(db, client, project_id, integration_id): assert Issue.objects.filter(status='closed').count() == 0 +@pytest.mark.parametrize('project_id', projects) +@pytest.mark.parametrize('integration_id', integrations) +def test_integration_webhook_post_wrong_url(db, client, project_id, integration_id): + integration = Integration.objects.filter(project_id=project_id, id=integration_id).first() + + secret = 'super_duper_secret' + url = reverse('integration_webhook', args=[project_id, integration_id]) + data = { + 'action': 'closed', + 'url': 'https://simple.example.com/issues/2' + } + + response = client.post(url, data, **{'HTTP_X_SECRET': secret, 'content_type': 'application/json'}) + + if integration: + assert response.status_code == 200 + assert Issue.objects.filter(status='closed').count() == 0 + else: + assert response.status_code == 404 + assert Issue.objects.filter(status='closed').count() == 0 + + @pytest.mark.parametrize('project_id', projects) @pytest.mark.parametrize('integration_id', integrations) def test_integration_webhook_post_no_secret(db, client, project_id, integration_id): diff --git a/rdmo/projects/tests/test_view_issue.py b/rdmo/projects/tests/test_view_issue.py index 230bc2f7e5..e80a67b4a2 100644 --- a/rdmo/projects/tests/test_view_issue.py +++ b/rdmo/projects/tests/test_view_issue.py @@ -1,6 +1,9 @@ +from unittest.mock import Mock + import pytest from django.core import mail +from django.http import HttpResponseRedirect from django.urls import reverse from rdmo.core.constants import VALUE_TYPE_FILE @@ -224,7 +227,10 @@ def test_issue_send_post_attachements(db, client, files, username, password, pro @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('issue_id', issues) @pytest.mark.parametrize('project_id', projects) -def test_issue_send_post_integration(db, client, username, password, project_id, issue_id): +def test_issue_send_post_integration(db, client, mocker, username, password, project_id, issue_id): + mocked_send_issue = Mock(return_value=HttpResponseRedirect(redirect_to='https://example.com/login/oauth/authorize')) + mocker.patch('rdmo.projects.providers.SimpleIssueProvider.send_issue', mocked_send_issue) + client.login(username=username, password=password) issue = Issue.objects.filter(project_id=project_id, id=issue_id).first() @@ -240,7 +246,7 @@ def test_issue_send_post_integration(db, client, username, password, project_id, if project_id in change_issue_permission_map.get(username, []): if integration_pk in Project.objects.get(pk=project_id).integrations.values_list('id', flat=True): assert response.status_code == 302 - assert response.url.startswith('https://github.com') + assert response.url.startswith('https://example.com') else: assert response.status_code == 200 else: @@ -248,6 +254,6 @@ def test_issue_send_post_integration(db, client, username, password, project_id, assert response.status_code == 403 else: assert response.status_code == 302 - assert not response.url.startswith('https://github.com') + assert not response.url.startswith('https://example.com') else: assert response.status_code == 404 diff --git a/rdmo/projects/tests/test_viewset_project_integration.py b/rdmo/projects/tests/test_viewset_project_integration.py index 94129a3d06..d1271081cc 100644 --- a/rdmo/projects/tests/test_viewset_project_integration.py +++ b/rdmo/projects/tests/test_viewset_project_integration.py @@ -87,11 +87,11 @@ def test_create(db, client, username, password, project_id): url = reverse(urlnames['list'], args=[project_id]) data = { - 'provider_key': 'github', + 'provider_key': 'simple', 'options': [ { - 'key': 'repo', - 'value': 'example/example' + 'key': 'project_url', + 'value': 'https://example.com/projects/1' } ] } @@ -115,8 +115,8 @@ def test_create_error1(db, client, username, password, project_id): 'provider_key': 'wrong', 'options': [ { - 'key': 'repo', - 'value': 'example/example' + 'key': 'project_url', + 'value': 'https://example.com/projects/1' } ] } @@ -138,10 +138,10 @@ def test_create_error2(db, client, username, password, project_id): url = reverse(urlnames['list'], args=[project_id]) data = { - 'provider_key': 'github', + 'provider_key': 'simple', 'options': [ { - 'key': 'repo', + 'key': 'project_url', 'value': '' } ] @@ -164,11 +164,11 @@ def test_create_error3(db, client, username, password, project_id): url = reverse(urlnames['list'], args=[project_id]) data = { - 'provider_key': 'github', + 'provider_key': 'simple', 'options': [ { - 'key': 'repo', - 'value': 'example/example' + 'key': 'project_url', + 'value': 'https://example.com/projects/1' }, { 'key': 'foo', @@ -196,11 +196,11 @@ def test_update(db, client, username, password, project_id, integration_id): url = reverse(urlnames['detail'], args=[project_id, integration_id]) data = { - 'provider_key': 'github', + 'provider_key': 'simple', 'options': [ { - 'key': 'repo', - 'value': 'example/test' + 'key': 'project_url', + 'value': 'https://example.com/projects/2' } ] } @@ -210,8 +210,8 @@ def test_update(db, client, username, password, project_id, integration_id): assert response.status_code == 200 assert sorted(response.json().get('options'), key=lambda obj: obj['key']) == [ { - 'key': 'repo', - 'value': 'example/test' + 'key': 'project_url', + 'value': 'https://example.com/projects/2' }, { 'key': 'secret', diff --git a/testing/config/settings/base.py b/testing/config/settings/base.py index 270ec4ea20..aa888ab8bd 100644 --- a/testing/config/settings/base.py +++ b/testing/config/settings/base.py @@ -79,10 +79,5 @@ ] PROJECT_ISSUE_PROVIDERS = [ - ('github', _('GitHub'), 'rdmo.projects.providers.GitHubIssueProvider') + ('simple', _('Simple provider'), 'rdmo.projects.providers.SimpleIssueProvider') ] - -GITHUB_PROVIDER = { - 'client_id': '', - 'client_secret': '' -} diff --git a/testing/fixtures/projects.json b/testing/fixtures/projects.json index 791f1a7493..59bfbb2cc9 100644 --- a/testing/fixtures/projects.json +++ b/testing/fixtures/projects.json @@ -15,7 +15,7 @@ "pk": 1, "fields": { "project": 1, - "provider_key": "github" + "provider_key": "simple" } }, { @@ -23,7 +23,7 @@ "pk": 2, "fields": { "project": 2, - "provider_key": "github" + "provider_key": "simple" } }, { @@ -31,8 +31,8 @@ "pk": 1, "fields": { "integration": 1, - "key": "repo", - "value": "example/example", + "key": "project_url", + "value": "https://example.com/projects/1", "secret": false } }, @@ -51,8 +51,8 @@ "pk": 3, "fields": { "integration": 2, - "key": "repo", - "value": "example/example", + "key": "project_url", + "value": "https://example.com/projects/1", "secret": false } }, @@ -135,7 +135,7 @@ "fields": { "issue": 1, "integration": 1, - "url": "https://github.com/example/example/issues/1" + "url": "https://simple.example.com/issues/1" } }, {