From 6001430822bd786e4c67147fb8434970c08085a7 Mon Sep 17 00:00:00 2001
From: Heinz-Alexander Fuetterer
Date: Thu, 28 Sep 2023 09:47:56 +0200
Subject: [PATCH 01/47] refactor: use admin.register decorator
---
rdmo/accounts/admin.py | 10 ++++------
rdmo/conditions/admin.py | 4 +---
rdmo/domain/admin.py | 4 +---
rdmo/options/admin.py | 6 ++----
rdmo/overlays/admin.py | 4 +---
rdmo/projects/admin.py | 22 ++++++++++------------
rdmo/questions/admin.py | 12 +++++-------
rdmo/tasks/admin.py | 4 +---
rdmo/views/admin.py | 4 +---
9 files changed, 26 insertions(+), 44 deletions(-)
diff --git a/rdmo/accounts/admin.py b/rdmo/accounts/admin.py
index ac5a14342a..9c4fea9afe 100644
--- a/rdmo/accounts/admin.py
+++ b/rdmo/accounts/admin.py
@@ -5,14 +5,17 @@
from .models import AdditionalField, AdditionalFieldValue, ConsentFieldValue, Role
+@admin.register(AdditionalField)
class AdditionalFieldAdmin(admin.ModelAdmin):
pass
+@admin.register(AdditionalFieldValue)
class AdditionalFieldValueAdmin(admin.ModelAdmin):
readonly_fields = ('user', )
+@admin.register(ConsentFieldValue)
class ConsentFieldValueAdmin(admin.ModelAdmin):
readonly_fields = ('user', 'consent')
@@ -20,6 +23,7 @@ def has_add_permission(self, request, obj=None):
return False
+@admin.register(Role)
class RoleAdmin(admin.ModelAdmin):
search_fields = ('user__username', 'user__email')
list_filter = ('member', 'manager', 'editor', 'reviewer')
@@ -50,9 +54,3 @@ def editors(self, obj):
def reviewers(self, obj):
return self.render_all_sites_or_join(obj, 'reviewer')
-
-
-admin.site.register(AdditionalField, AdditionalFieldAdmin)
-admin.site.register(AdditionalFieldValue, AdditionalFieldValueAdmin)
-admin.site.register(ConsentFieldValue, ConsentFieldValueAdmin)
-admin.site.register(Role, RoleAdmin)
diff --git a/rdmo/conditions/admin.py b/rdmo/conditions/admin.py
index 17db94527c..7bcfb04697 100644
--- a/rdmo/conditions/admin.py
+++ b/rdmo/conditions/admin.py
@@ -17,6 +17,7 @@ def clean(self):
ConditionLockedValidator(self.instance)(self.cleaned_data)
+@admin.register(Condition)
class ConditionAdmin(admin.ModelAdmin):
form = ConditionAdminForm
@@ -25,6 +26,3 @@ class ConditionAdmin(admin.ModelAdmin):
readonly_fields = ('uri', )
list_filter = ('relation', )
filter_horizontal = ('editors', )
-
-
-admin.site.register(Condition, ConditionAdmin)
diff --git a/rdmo/domain/admin.py b/rdmo/domain/admin.py
index 756e41f4d3..44377ac95d 100644
--- a/rdmo/domain/admin.py
+++ b/rdmo/domain/admin.py
@@ -19,6 +19,7 @@ def clean(self):
AttributeLockedValidator(self.instance)(self.cleaned_data)
+@admin.register(Attribute)
class AttributeAdmin(admin.ModelAdmin):
form = AttributeAdminForm
@@ -37,6 +38,3 @@ def values_count(self, obj):
def projects_count(self, obj):
return obj.projects_count
-
-
-admin.site.register(Attribute, AttributeAdmin)
diff --git a/rdmo/options/admin.py b/rdmo/options/admin.py
index bf5e21c65c..f95d8787bd 100644
--- a/rdmo/options/admin.py
+++ b/rdmo/options/admin.py
@@ -39,6 +39,7 @@ class OptionSetOptionInline(admin.TabularInline):
extra = 0
+@admin.register(OptionSet)
class OptionSetAdmin(admin.ModelAdmin):
form = OptionSetAdminForm
inlines = (OptionSetOptionInline, )
@@ -49,6 +50,7 @@ class OptionSetAdmin(admin.ModelAdmin):
filter_horizontal = ('editors', 'conditions')
+@admin.register(Option)
class OptionAdmin(admin.ModelAdmin):
form = OptionAdminForm
@@ -57,7 +59,3 @@ class OptionAdmin(admin.ModelAdmin):
readonly_fields = ('uri', )
list_filter = ('editors', 'optionsets', 'additional_input')
filter_horizontal = ('editors', )
-
-
-admin.site.register(OptionSet, OptionSetAdmin)
-admin.site.register(Option, OptionAdmin)
diff --git a/rdmo/overlays/admin.py b/rdmo/overlays/admin.py
index d1267b834e..ee663ac45c 100644
--- a/rdmo/overlays/admin.py
+++ b/rdmo/overlays/admin.py
@@ -3,10 +3,8 @@
from .models import Overlay
+@admin.register(Overlay)
class OverlayAdmin(admin.ModelAdmin):
search_fields = ('user__username', 'url_name', 'current')
list_display = ('user', 'site', 'url_name', 'current')
list_filter = ('url_name', 'current')
-
-
-admin.site.register(Overlay, OverlayAdmin)
diff --git a/rdmo/projects/admin.py b/rdmo/projects/admin.py
index e8890cf19d..fcf6749b5c 100644
--- a/rdmo/projects/admin.py
+++ b/rdmo/projects/admin.py
@@ -15,6 +15,7 @@
)
+@admin.register(Project)
class ProjectAdmin(admin.ModelAdmin):
search_fields = ('title', 'user__username')
list_display = ('title', 'owners', 'updated', 'created')
@@ -32,43 +33,51 @@ def owners(self, obj):
return ', '.join([membership.user.username for membership in obj.owner_memberships])
+@admin.register(Membership)
class MembershipAdmin(admin.ModelAdmin):
search_fields = ('project__title', 'user__username', 'role')
list_display = ('project', 'user', 'role')
+@admin.register(Continuation)
class ContinuationAdmin(admin.ModelAdmin):
search_fields = ('project__title', 'user__username')
list_display = ('project', 'user', 'page')
+@admin.register(Integration)
class IntegrationAdmin(admin.ModelAdmin):
search_fields = ('project__title', 'provider_key')
list_display = ('project', 'provider_key')
+@admin.register(IntegrationOption)
class IntegrationOptionAdmin(admin.ModelAdmin):
search_fields = ('integration__project__title', 'key', 'value')
list_display = ('integration', 'key', 'value')
+@admin.register(Invite)
class InviteAdmin(admin.ModelAdmin):
search_fields = ('project__title', 'user__username', 'email', 'role')
list_display = ('project', 'user', 'email', 'token', 'timestamp')
readonly_fields = ('token', 'timestamp')
+@admin.register(Issue)
class IssueAdmin(admin.ModelAdmin):
search_fields = ('project__title', 'task', 'status')
list_display = ('project', 'task', 'status')
list_filter = ('status', )
+@admin.register(IssueResource)
class IssueResourceAdmin(admin.ModelAdmin):
search_fields = ('issue__project__title', 'url')
list_display = ('issue', 'url')
+@admin.register(Snapshot)
class SnapshotAdmin(admin.ModelAdmin):
search_fields = ('title', 'project__title', 'project__user__username')
list_display = ('title', 'project', 'owners', 'updated', 'created')
@@ -86,6 +95,7 @@ def owners(self, obj):
return ', '.join([membership.user.username for membership in obj.project.owner_memberships])
+@admin.register(Value)
class ValueAdmin(admin.ModelAdmin):
search_fields = ('attribute__uri', 'project__title', 'snapshot__title', 'project__user__username')
list_display = ('attribute', 'set_prefix', 'set_index', 'collection_index', 'project', 'snapshot_title')
@@ -94,15 +104,3 @@ class ValueAdmin(admin.ModelAdmin):
def snapshot_title(self, obj):
if obj.snapshot:
return obj.snapshot.title
-
-
-admin.site.register(Project, ProjectAdmin)
-admin.site.register(Membership, MembershipAdmin)
-admin.site.register(Continuation, ContinuationAdmin)
-admin.site.register(Integration, IntegrationAdmin)
-admin.site.register(IntegrationOption, IntegrationOptionAdmin)
-admin.site.register(Invite, InviteAdmin)
-admin.site.register(Issue, IssueAdmin)
-admin.site.register(IssueResource, IssueResourceAdmin)
-admin.site.register(Snapshot, SnapshotAdmin)
-admin.site.register(Value, ValueAdmin)
diff --git a/rdmo/questions/admin.py b/rdmo/questions/admin.py
index 9b52d07f74..1ff106b968 100644
--- a/rdmo/questions/admin.py
+++ b/rdmo/questions/admin.py
@@ -96,6 +96,7 @@ class CatalogSectionInline(admin.TabularInline):
extra = 0
+@admin.register(Catalog)
class CatalogAdmin(admin.ModelAdmin):
form = CatalogAdminForm
inlines = (CatalogSectionInline, )
@@ -119,6 +120,7 @@ class SectionPageInline(admin.TabularInline):
extra = 0
+@admin.register(Section)
class SectionAdmin(admin.ModelAdmin):
form = SectionAdminForm
inlines = (SectionPageInline, )
@@ -140,6 +142,7 @@ class PageQuestionInline(admin.TabularInline):
extra = 0
+@admin.register(Page)
class PageAdmin(admin.ModelAdmin):
form = PageAdminForm
inlines = (PageQuestionSetInline, PageQuestionInline)
@@ -162,6 +165,7 @@ class QuestionSetQuestionInline(admin.TabularInline):
extra = 0
+@admin.register(QuestionSet)
class QuestionSetAdmin(admin.ModelAdmin):
form = QuestionSetAdminForm
inlines = (QuestionSetQuestionSetInline, QuestionSetQuestionInline)
@@ -173,6 +177,7 @@ class QuestionSetAdmin(admin.ModelAdmin):
filter_horizontal = ('editors', 'conditions')
+@admin.register(Question)
class QuestionAdmin(admin.ModelAdmin):
form = QuestionAdminForm
@@ -182,10 +187,3 @@ class QuestionAdmin(admin.ModelAdmin):
list_filter = ('pages__sections__catalogs', 'pages__sections', 'pages', 'is_collection',
'widget_type', 'value_type')
filter_horizontal = ('editors', 'optionsets', 'conditions')
-
-
-admin.site.register(Catalog, CatalogAdmin)
-admin.site.register(Section, SectionAdmin)
-admin.site.register(Page, PageAdmin)
-admin.site.register(QuestionSet, QuestionSetAdmin)
-admin.site.register(Question, QuestionAdmin)
diff --git a/rdmo/tasks/admin.py b/rdmo/tasks/admin.py
index bb2c85c429..6d1acf36cc 100644
--- a/rdmo/tasks/admin.py
+++ b/rdmo/tasks/admin.py
@@ -19,6 +19,7 @@ def clean(self):
TaskLockedValidator(self.instance)(self.cleaned_data)
+@admin.register(Task)
class TaskAdmin(admin.ModelAdmin):
form = TaskAdminForm
@@ -27,6 +28,3 @@ class TaskAdmin(admin.ModelAdmin):
readonly_fields = ('uri', )
list_filter = ('available', )
filter_horizontal = ('catalogs', 'sites', 'editors', 'groups', 'conditions')
-
-
-admin.site.register(Task, TaskAdmin)
diff --git a/rdmo/views/admin.py b/rdmo/views/admin.py
index 1cfebec9af..9fedfe6d53 100644
--- a/rdmo/views/admin.py
+++ b/rdmo/views/admin.py
@@ -19,6 +19,7 @@ def clean(self):
ViewLockedValidator(self.instance)(self.cleaned_data)
+@admin.register(View)
class ViewAdmin(admin.ModelAdmin):
form = ViewAdminForm
@@ -27,6 +28,3 @@ class ViewAdmin(admin.ModelAdmin):
readonly_fields = ('uri', )
list_filter = ('available', )
filter_horizontal = ('catalogs', 'sites', 'editors', 'groups')
-
-
-admin.site.register(View, ViewAdmin)
From 345afd92322b88cb65f63764d4776cae59dbf91b Mon Sep 17 00:00:00 2001
From: Heinz-Alexander Fuetterer <35225576+afuetterer@users.noreply.github.com>
Date: Mon, 16 Oct 2023 15:31:11 +0200
Subject: [PATCH 02/47] build: pin django-mptt dependency
---
.github/dependabot.yml | 2 ++
pyproject.toml | 5 ++++-
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index ee73ab2200..02ffb752a9 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -10,6 +10,8 @@ updates:
- dependencies
- python
- type:maintenance
+ ignore:
+ - dependency-name: django-mptt # pinned, 0.15 requires Python >= 3.9
- package-ecosystem: github-actions
directory: /
schedule:
diff --git a/pyproject.toml b/pyproject.toml
index 86cf1dbcfa..9665e8e7c5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -37,6 +37,9 @@ dynamic = [
"version",
]
dependencies = [
+ # dependencies with major version on zero are declared with
+ # major.minor.patch, because they can potentially introduce breaking changes
+ # in minor version updates anytime
"defusedcsv~=2.0",
"defusedxml~=0.7.1",
"django~=4.2",
@@ -46,7 +49,7 @@ dependencies = [
"django-filter~=23.2",
"django-libsass~=0.9",
"django-mathfilters~=1.0",
- "django-mptt~=0.14.0",
+ "django-mptt==0.14.0", # pinned, 0.15 requires Python >= 3.9
"django-rest-swagger~=2.2",
"django-settings-export~=1.2",
"django-split-settings~=1.2",
From e7107080b44a1103c1ea66c7800a053e8c1f4ac8 Mon Sep 17 00:00:00 2001
From: Heinz-Alexander Fuetterer <35225576+afuetterer@users.noreply.github.com>
Date: Tue, 7 Nov 2023 14:35:17 +0100
Subject: [PATCH 03/47] fix: update verbose_name_plural in questionset
---
.../0091_alter_questionset_options.py | 20 +++++++++++++++++++
rdmo/questions/models/questionset.py | 2 +-
2 files changed, 21 insertions(+), 1 deletion(-)
create mode 100644 rdmo/questions/migrations/0091_alter_questionset_options.py
diff --git a/rdmo/questions/migrations/0091_alter_questionset_options.py b/rdmo/questions/migrations/0091_alter_questionset_options.py
new file mode 100644
index 0000000000..037cce001f
--- /dev/null
+++ b/rdmo/questions/migrations/0091_alter_questionset_options.py
@@ -0,0 +1,20 @@
+# Generated by Django 4.2.7 on 2023-11-08 05:40
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("questions", "0090_add_editors"),
+ ]
+
+ operations = [
+ migrations.AlterModelOptions(
+ name="questionset",
+ options={
+ "ordering": ("uri",),
+ "verbose_name": "Question set",
+ "verbose_name_plural": "Question sets",
+ },
+ ),
+ ]
diff --git a/rdmo/questions/models/questionset.py b/rdmo/questions/models/questionset.py
index 8e3f0026cb..64aaaebddc 100644
--- a/rdmo/questions/models/questionset.py
+++ b/rdmo/questions/models/questionset.py
@@ -185,7 +185,7 @@ class QuestionSet(Model, TranslationMixin):
class Meta:
ordering = ('uri', )
verbose_name = _('Question set')
- verbose_name_plural = _('Question set')
+ verbose_name_plural = _('Question sets')
def __str__(self):
return self.uri
From c7f625af2a6fdbbc17e1b2767595a74e702622a0 Mon Sep 17 00:00:00 2001
From: David Wallace
Date: Tue, 24 Oct 2023 12:01:39 +0200
Subject: [PATCH 04/47] fix #791 for openid_connect providers and refactor
keycloak snippet
---
.../socialaccount/snippets/provider_list.html | 24 +++++++++++++------
1 file changed, 17 insertions(+), 7 deletions(-)
diff --git a/rdmo/accounts/templates/socialaccount/snippets/provider_list.html b/rdmo/accounts/templates/socialaccount/snippets/provider_list.html
index f8f4e278cd..0d95b4b405 100644
--- a/rdmo/accounts/templates/socialaccount/snippets/provider_list.html
+++ b/rdmo/accounts/templates/socialaccount/snippets/provider_list.html
@@ -24,13 +24,23 @@
-{% elif provider.id == 'keycloak' %}
-
-
-
-
-
+{% elif provider.id == 'openid_connect' %}
+ {% if provider.app.provider_id == 'keycloak' %}
+
+
+
+
+
+ {% else %}
+
+
+ {{ provider.name }}
+
+
+{% endif %}
+
{% else %}
Date: Fri, 6 Oct 2023 15:47:19 +0200
Subject: [PATCH 05/47] Add custom {more} markdown tag (#595)
---
rdmo/core/static/core/css/base.scss | 10 ++++++++++
rdmo/core/static/core/js/utils.js | 11 +++++++++++
rdmo/core/templates/core/base.html | 3 +++
rdmo/core/utils.py | 19 +++++++++++++++----
4 files changed, 39 insertions(+), 4 deletions(-)
create mode 100644 rdmo/core/static/core/js/utils.js
diff --git a/rdmo/core/static/core/css/base.scss b/rdmo/core/static/core/css/base.scss
index 5d372c3206..af36f32d2f 100644
--- a/rdmo/core/static/core/css/base.scss
+++ b/rdmo/core/static/core/css/base.scss
@@ -477,3 +477,13 @@ li.has-warning > a.control-label > i {
text-decoration: underline;
text-decoration-style: dotted;
}
+
+.more,
+.show-less {
+ display: none;
+}
+.show-more,
+.show-less {
+ color: $link-color;
+ cursor: pointer;
+}
diff --git a/rdmo/core/static/core/js/utils.js b/rdmo/core/static/core/js/utils.js
new file mode 100644
index 0000000000..8c34a73526
--- /dev/null
+++ b/rdmo/core/static/core/js/utils.js
@@ -0,0 +1,11 @@
+function showMore(element) {
+ $(element).siblings('.more').show();
+ $(element).siblings('.show-less').show();
+ $(element).hide();
+}
+
+function showLess(element) {
+ $(element).siblings('.more').hide();
+ $(element).siblings('.show-more').show();
+ $(element).hide();
+}
diff --git a/rdmo/core/templates/core/base.html b/rdmo/core/templates/core/base.html
index fbe8dcec9a..d9508e1be9 100644
--- a/rdmo/core/templates/core/base.html
+++ b/rdmo/core/templates/core/base.html
@@ -20,6 +20,9 @@
{% endblock %}
{% block js %}
+ {% compress js %}
+
+ {% endcompress %}
{% endblock %}
{% block head %}{% endblock %}
diff --git a/rdmo/core/utils.py b/rdmo/core/utils.py
index 22d9d42b31..acfaf51fdd 100644
--- a/rdmo/core/utils.py
+++ b/rdmo/core/utils.py
@@ -384,15 +384,26 @@ def is_truthy(value):
def markdown2html(markdown_string):
- # adoption of the normal markdown function which also converts
- # `[]{}` to to
- # allow for underlined tooltips
- html = markdown(force_str(markdown_string))
+ # adoption of the normal markdown function
+ html = markdown(force_str(markdown_string)).strip()
+
+ # convert `[]{}` to to allow for underlined tooltips
html = re.sub(
r'\[(.*?)\]\{(.*?)\}',
r'\1',
html
)
+
+ # convert everything after `{more}` to to be shown/hidden on user input
+ show_string = _('show more')
+ hide_string = _('show less')
+ html = re.sub(
+ r'(\{more\})(.*?)
$',
+ f'... ({show_string})'
+ r'\2'
+ f' ({hide_string})',
+ html
+ )
return html
From 480ae0e0a9d4fd7df081c4423baf2653a203c12c Mon Sep 17 00:00:00 2001
From: Jochen Klar
Date: Fri, 6 Oct 2023 15:51:10 +0200
Subject: [PATCH 06/47] Adjust details/summary style
---
rdmo/core/static/core/css/base.scss | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/rdmo/core/static/core/css/base.scss b/rdmo/core/static/core/css/base.scss
index af36f32d2f..02bda4951e 100644
--- a/rdmo/core/static/core/css/base.scss
+++ b/rdmo/core/static/core/css/base.scss
@@ -129,9 +129,14 @@ table {
}
}
+details {
+ margin-bottom: 10px;
+}
+
summary {
- color: $link-color;
+ display: list-item;
cursor: pointer;
+ margin-bottom: 5px;
}
metadata {
From 983e7d4e1a491c463fc085a5d78e4dd780a17df8 Mon Sep 17 00:00:00 2001
From: Jochen Klar
Date: Mon, 30 Oct 2023 18:49:10 +0100
Subject: [PATCH 07/47] Add ValueConflictValidator and check for conflicts when
storing values
---
rdmo/projects/serializers/v1/__init__.py | 7 ++--
.../projects/js/project_questions/services.js | 11 +++++-
...ect_questions_form_group_autocomplete.html | 2 ++
...project_questions_form_group_checkbox.html | 3 ++
.../project_questions_form_group_date.html | 2 ++
.../project_questions_form_group_file.html | 2 ++
.../project_questions_form_group_radio.html | 2 ++
.../project_questions_form_group_range.html | 2 ++
.../project_questions_form_group_select.html | 2 ++
.../project_questions_form_group_text.html | 2 ++
...project_questions_form_group_textarea.html | 2 ++
.../project_questions_form_group_yesno.html | 2 ++
.../projects/project_questions_overview.html | 3 ++
.../project_questions_save_error.html | 2 +-
.../project_questions_value_errors.html | 21 ++++++++++++
rdmo/projects/validators.py | 34 +++++++++++++++++--
16 files changed, 93 insertions(+), 6 deletions(-)
create mode 100644 rdmo/projects/templates/projects/project_questions_value_errors.html
diff --git a/rdmo/projects/serializers/v1/__init__.py b/rdmo/projects/serializers/v1/__init__.py
index 7e65e7cb9f..a5ec382b30 100644
--- a/rdmo/projects/serializers/v1/__init__.py
+++ b/rdmo/projects/serializers/v1/__init__.py
@@ -8,7 +8,7 @@
from rdmo.services.validators import ProviderValidator
from ...models import Integration, IntegrationOption, Invite, Issue, IssueResource, Membership, Project, Snapshot, Value
-from ...validators import ValueValidator
+from ...validators import ValueConflictValidator, ValueQuotaValidator
class UserSerializer(serializers.ModelSerializer):
@@ -259,7 +259,10 @@ class Meta:
'unit',
'external_id'
)
- validators = (ValueValidator(), )
+ validators = (
+ ValueConflictValidator(),
+ ValueQuotaValidator()
+ )
class MembershipSerializer(serializers.ModelSerializer):
diff --git a/rdmo/projects/static/projects/js/project_questions/services.js b/rdmo/projects/static/projects/js/project_questions/services.js
index efa8e4ea9d..77fabffcc4 100644
--- a/rdmo/projects/static/projects/js/project_questions/services.js
+++ b/rdmo/projects/static/projects/js/project_questions/services.js
@@ -706,6 +706,9 @@ angular.module('project_questions')
};
service.storeValue = function(value, question, set_prefix, set_index, collection_index) {
+ // reset value errors
+ value.errors = []
+
if (angular.isDefined(value.removed) && value.removed) {
// remove additional_input from unselected checkboxes
value.additional_input = {};
@@ -826,10 +829,16 @@ angular.module('project_questions')
}, function (response) {
if (response.status == 500) {
service.error = response;
+ } else if (response.status == 400) {
+ service.error = true;
+ value.errors = Object.keys(response.data);
+ } else if (response.status == 404) {
+ service.error = true;
+ value.errors = ['not_found']
}
})
}
-};
+ };
service.storeValues = function() {
var promises = [];
diff --git a/rdmo/projects/templates/projects/project_questions_form_group_autocomplete.html b/rdmo/projects/templates/projects/project_questions_form_group_autocomplete.html
index e35e795114..2996f3a321 100644
--- a/rdmo/projects/templates/projects/project_questions_form_group_autocomplete.html
+++ b/rdmo/projects/templates/projects/project_questions_form_group_autocomplete.html
@@ -44,6 +44,8 @@
+
+ {% include 'projects/project_questions_value_errors.html' %}
diff --git a/rdmo/projects/templates/projects/project_questions_form_group_checkbox.html b/rdmo/projects/templates/projects/project_questions_form_group_checkbox.html
index 48ed47be19..bb344cd818 100644
--- a/rdmo/projects/templates/projects/project_questions_form_group_checkbox.html
+++ b/rdmo/projects/templates/projects/project_questions_form_group_checkbox.html
@@ -24,6 +24,9 @@
ng-disabled="service.project.read_only"
ng-change="service.changed(service.values[question.attribute][valueset.set_prefix][valueset.set_index][$index])" />
+
+ {% include 'projects/project_questions_value_errors.html' %}
+
diff --git a/rdmo/projects/templates/projects/project_questions_form_group_date.html b/rdmo/projects/templates/projects/project_questions_form_group_date.html
index 8994983bd5..36e60cf423 100644
--- a/rdmo/projects/templates/projects/project_questions_form_group_date.html
+++ b/rdmo/projects/templates/projects/project_questions_form_group_date.html
@@ -24,6 +24,8 @@
ng-change="service.changed(value, true)"
ng-class="{'default-value': service.isDefaultValue(question, value)}"/>
+
+ {% include 'projects/project_questions_value_errors.html' %}
diff --git a/rdmo/projects/templates/projects/project_questions_form_group_file.html b/rdmo/projects/templates/projects/project_questions_form_group_file.html
index 4ddf98dc31..ee0db7fad5 100644
--- a/rdmo/projects/templates/projects/project_questions_form_group_file.html
+++ b/rdmo/projects/templates/projects/project_questions_form_group_file.html
@@ -33,6 +33,8 @@
+
+ {% include 'projects/project_questions_value_errors.html' %}
diff --git a/rdmo/projects/templates/projects/project_questions_form_group_radio.html b/rdmo/projects/templates/projects/project_questions_form_group_radio.html
index d1e8f462f6..29e498e8c7 100644
--- a/rdmo/projects/templates/projects/project_questions_form_group_radio.html
+++ b/rdmo/projects/templates/projects/project_questions_form_group_radio.html
@@ -43,6 +43,8 @@
+
+ {% include 'projects/project_questions_value_errors.html' %}
diff --git a/rdmo/projects/templates/projects/project_questions_form_group_range.html b/rdmo/projects/templates/projects/project_questions_form_group_range.html
index 384f413d59..39fb0552d5 100644
--- a/rdmo/projects/templates/projects/project_questions_form_group_range.html
+++ b/rdmo/projects/templates/projects/project_questions_form_group_range.html
@@ -31,6 +31,8 @@
ng-class="{'default-value': service.isDefaultValue(question, value)}"/>
+
+ {% include 'projects/project_questions_value_errors.html' %}
diff --git a/rdmo/projects/templates/projects/project_questions_form_group_select.html b/rdmo/projects/templates/projects/project_questions_form_group_select.html
index d672b98981..4589371cb0 100644
--- a/rdmo/projects/templates/projects/project_questions_form_group_select.html
+++ b/rdmo/projects/templates/projects/project_questions_form_group_select.html
@@ -28,6 +28,8 @@
+
+ {% include 'projects/project_questions_value_errors.html' %}
diff --git a/rdmo/projects/templates/projects/project_questions_form_group_text.html b/rdmo/projects/templates/projects/project_questions_form_group_text.html
index d9284bdf56..e3fad422ec 100644
--- a/rdmo/projects/templates/projects/project_questions_form_group_text.html
+++ b/rdmo/projects/templates/projects/project_questions_form_group_text.html
@@ -21,6 +21,8 @@
ng-disabled="service.project.read_only"
ng-change="service.changed(value)"
ng-class="{'default-value': service.isDefaultValue(question, value)}">
+
+ {% include 'projects/project_questions_value_errors.html' %}
diff --git a/rdmo/projects/templates/projects/project_questions_form_group_textarea.html b/rdmo/projects/templates/projects/project_questions_form_group_textarea.html
index 756e21c96e..26baee92a1 100644
--- a/rdmo/projects/templates/projects/project_questions_form_group_textarea.html
+++ b/rdmo/projects/templates/projects/project_questions_form_group_textarea.html
@@ -23,6 +23,8 @@
ng-class="{'default-value': service.isDefaultValue(question, value)}">
+
+ {% include 'projects/project_questions_value_errors.html' %}
diff --git a/rdmo/projects/templates/projects/project_questions_form_group_yesno.html b/rdmo/projects/templates/projects/project_questions_form_group_yesno.html
index c06ce86618..31dda7fb79 100644
--- a/rdmo/projects/templates/projects/project_questions_form_group_yesno.html
+++ b/rdmo/projects/templates/projects/project_questions_form_group_yesno.html
@@ -41,6 +41,8 @@
+
+ {% include 'projects/project_questions_value_errors.html' %}
diff --git a/rdmo/projects/templates/projects/project_questions_overview.html b/rdmo/projects/templates/projects/project_questions_overview.html
index 1ee6ea347b..6058e465a5 100644
--- a/rdmo/projects/templates/projects/project_questions_overview.html
+++ b/rdmo/projects/templates/projects/project_questions_overview.html
@@ -10,6 +10,9 @@
+ -
+ {% trans 'Reload page' %}
+
-
{% trans 'Back to my projects' %}
diff --git a/rdmo/projects/templates/projects/project_questions_save_error.html b/rdmo/projects/templates/projects/project_questions_save_error.html
index a3d6d90cba..e3a5aba785 100644
--- a/rdmo/projects/templates/projects/project_questions_save_error.html
+++ b/rdmo/projects/templates/projects/project_questions_save_error.html
@@ -1,7 +1,7 @@
{% load i18n %}
{% load core_tags %}
-
+
{% trans 'An error occurred while saving the answer. Please contact support if this problem persists.' %}
diff --git a/rdmo/projects/templates/projects/project_questions_value_errors.html b/rdmo/projects/templates/projects/project_questions_value_errors.html
new file mode 100644
index 0000000000..d8245287ad
--- /dev/null
+++ b/rdmo/projects/templates/projects/project_questions_value_errors.html
@@ -0,0 +1,21 @@
+{% load i18n %}
+
+
+ -
+
+ {% blocktrans trimmed %}
+ This field could not be saved, since somebody else did so while you were editing.
+ You will need to reload the page to make changes, but your input will be overwritten.
+ {% endblocktrans %}
+
+
+ {% blocktrans trimmed %}
+ This field could not be saved, since somebody else removed it while you were editing.
+ You will need to reload the page to proceed, but your input will be lost.
+ {% endblocktrans %}
+
+
+ {% trans 'You reached the file quota for this project.' %}
+
+
+
diff --git a/rdmo/projects/validators.py b/rdmo/projects/validators.py
index 3c60a66a32..bffa0020e8 100644
--- a/rdmo/projects/validators.py
+++ b/rdmo/projects/validators.py
@@ -1,4 +1,6 @@
from django.conf import settings
+from django.core.exceptions import ObjectDoesNotExist
+from django.utils.dateparse import parse_datetime
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
@@ -7,7 +9,35 @@
from rdmo.core.utils import human2bytes
-class ValueValidator:
+class ValueConflictValidator:
+
+ requires_context = True
+
+ def __call__(self, data, serializer):
+ if serializer.instance:
+ # for an update, check if the value was updated in the meantime
+ updated = serializer.context['view'].request.data.get('updated')
+ if parse_datetime(updated) < serializer.instance.updated:
+ raise serializers.ValidationError({
+ 'conflict': [_('A newer version of this value was found.')]
+ })
+ else:
+ # for a new value, check if there is already a value with the same attribute and indexes
+ try:
+ serializer.context['view'].get_queryset().get(
+ attribute=data.get('attribute'),
+ set_prefix=data.get('set_prefix'),
+ set_index=data.get('set_index'),
+ collection_index=data.get('collection_index')
+ )
+ raise serializers.ValidationError({
+ 'conflict': [_('An existing value for this attribute/set_prefix/set_index/collection_index'
+ ' was found.')]
+ })
+ except ObjectDoesNotExist:
+ pass
+
+class ValueQuotaValidator:
requires_context = True
@@ -20,5 +50,5 @@ def __call__(self, data, serializer):
if project.file_size > human2bytes(settings.PROJECT_FILE_QUOTA):
raise serializers.ValidationError({
- 'value': [_('You reached the file quota for this project.')]
+ 'quota': [_('The file quota for this project has been reached.')]
}) from e
From ef5cd99bd5f65ce2bfedc5cca457cf418cc6cae2 Mon Sep 17 00:00:00 2001
From: Jochen Klar
Date: Mon, 30 Oct 2023 18:56:27 +0100
Subject: [PATCH 08/47] Autosave after eraseValue and removeValue
---
.../static/projects/js/project_questions/services.js | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/rdmo/projects/static/projects/js/project_questions/services.js b/rdmo/projects/static/projects/js/project_questions/services.js
index 77fabffcc4..92b09fc744 100644
--- a/rdmo/projects/static/projects/js/project_questions/services.js
+++ b/rdmo/projects/static/projects/js/project_questions/services.js
@@ -1067,11 +1067,19 @@ angular.module('project_questions')
service.values[attribute][set_prefix][set_index][collection_index].autocomplete_text = '';
service.values[attribute][set_prefix][set_index][collection_index].autocomplete_locked = false;
service.values[attribute][set_prefix][set_index][collection_index].changed = true;
+
+ if (service.settings.project_questions_autosave) {
+ service.save(false);
+ }
};
service.removeValue = function(attribute, set_prefix, set_index, collection_index) {
service.values[attribute][set_prefix][set_index][collection_index].removed = true;
service.values[attribute][set_prefix][set_index][collection_index].changed = true;
+
+ if (service.settings.project_questions_autosave) {
+ service.save(false);
+ }
};
service.openValueSetFormModal = function(questionset, set_prefix, set_index) {
From dd7e41506f7c5f1760a01adf9e5a8a34b87a3472 Mon Sep 17 00:00:00 2001
From: Jochen Klar
Date: Tue, 31 Oct 2023 11:55:03 +0100
Subject: [PATCH 09/47] Add tests for ValueConflictValidator
---
.../projects/tests/test_validator_conflict.py | 123 ++++++++++++++++++
rdmo/projects/validators.py | 5 +-
2 files changed, 125 insertions(+), 3 deletions(-)
create mode 100644 rdmo/projects/tests/test_validator_conflict.py
diff --git a/rdmo/projects/tests/test_validator_conflict.py b/rdmo/projects/tests/test_validator_conflict.py
new file mode 100644
index 0000000000..e48191c6ce
--- /dev/null
+++ b/rdmo/projects/tests/test_validator_conflict.py
@@ -0,0 +1,123 @@
+from datetime import timedelta
+
+import pytest
+
+from rest_framework.exceptions import ValidationError as RestFameworkValidationError
+
+from ..models import Project, Value
+from ..serializers.v1 import ValueSerializer
+from ..validators import ValueConflictValidator
+
+project_id = 1
+attribute_path = attribute__path='individual/single/text'
+
+
+def test_serializer_create(db):
+ class MockedView:
+ project = Project.objects.get(id=project_id)
+
+ value = Value.objects.get(project_id=project_id, snapshot=None, attribute__path=attribute_path)
+
+ validator = ValueConflictValidator()
+ serializer = ValueSerializer()
+ serializer.context['view'] = MockedView()
+
+ validator({
+ 'attribute': value.attribute,
+ 'set_prefix': value.set_prefix,
+ 'set_index': value.set_index,
+ 'collection_index': value.collection_index + 1,
+ }, serializer)
+
+
+def test_serializer_create_error(db):
+ class MockedView:
+ project = Project.objects.get(id=project_id)
+
+ value = Value.objects.get(project_id=project_id, snapshot=None, attribute__path=attribute_path)
+
+ validator = ValueConflictValidator()
+ serializer = ValueSerializer()
+ serializer.context['view'] = MockedView()
+
+ with pytest.raises(RestFameworkValidationError):
+ validator({
+ 'attribute': value.attribute,
+ 'set_prefix': value.set_prefix,
+ 'set_index': value.set_index,
+ 'collection_index': value.collection_index,
+ }, serializer)
+
+
+def test_serializer_update(db):
+ value = Value.objects.get(project_id=project_id, snapshot=None, attribute__path=attribute_path)
+
+ class MockedRequest:
+ data = {
+ 'updated': value.updated .isoformat()
+ }
+
+ class MockedView:
+ request = MockedRequest()
+ project = Project.objects.get(id=project_id)
+
+ validator = ValueConflictValidator()
+ serializer = ValueSerializer()
+ serializer.instance = value
+ serializer.context['view'] = MockedView()
+
+ validator({
+ 'attribute': value.attribute,
+ 'set_prefix': value.set_prefix,
+ 'set_index': value.set_index,
+ 'collection_index': value.collection_index,
+ }, serializer)
+
+
+def test_serializer_update_error(db):
+ value = Value.objects.get(project_id=project_id, snapshot=None, attribute__path=attribute_path)
+
+ class MockedRequest:
+ data = {
+ 'updated': (value.updated - timedelta(seconds=1)).isoformat()
+ }
+
+ class MockedView:
+ request = MockedRequest()
+ project = Project.objects.get(id=project_id)
+
+ validator = ValueConflictValidator()
+ serializer = ValueSerializer()
+ serializer.instance = value
+ serializer.context['view'] = MockedView()
+
+ with pytest.raises(RestFameworkValidationError):
+ validator({
+ 'attribute': value.attribute,
+ 'set_prefix': value.set_prefix,
+ 'set_index': value.set_index,
+ 'collection_index': value.collection_index,
+ }, serializer)
+
+
+def test_serializer_update_missing_updated(db):
+ value = Value.objects.get(project_id=project_id, snapshot=None, attribute__path=attribute_path)
+
+ class MockedRequest:
+ data = {}
+
+ class MockedView:
+ request = MockedRequest()
+ project = Project.objects.get(id=project_id)
+
+ validator = ValueConflictValidator()
+ serializer = ValueSerializer()
+ serializer.instance = value
+ serializer.context['view'] = MockedView()
+
+ validator({
+ 'attribute': value.attribute,
+ 'set_prefix': value.set_prefix,
+ 'set_index': value.set_index,
+ 'collection_index': value.collection_index,
+ }, serializer)
diff --git a/rdmo/projects/validators.py b/rdmo/projects/validators.py
index bffa0020e8..086a387f4d 100644
--- a/rdmo/projects/validators.py
+++ b/rdmo/projects/validators.py
@@ -17,14 +17,14 @@ def __call__(self, data, serializer):
if serializer.instance:
# for an update, check if the value was updated in the meantime
updated = serializer.context['view'].request.data.get('updated')
- if parse_datetime(updated) < serializer.instance.updated:
+ if updated is not None and parse_datetime(updated) < serializer.instance.updated:
raise serializers.ValidationError({
'conflict': [_('A newer version of this value was found.')]
})
else:
# for a new value, check if there is already a value with the same attribute and indexes
try:
- serializer.context['view'].get_queryset().get(
+ serializer.context['view'].project.values.filter(snapshot=None).get(
attribute=data.get('attribute'),
set_prefix=data.get('set_prefix'),
set_index=data.get('set_index'),
@@ -47,7 +47,6 @@ def __call__(self, data, serializer):
serializer.context['view'].get_object()
except AssertionError as e:
project = serializer.context['view'].project
-
if project.file_size > human2bytes(settings.PROJECT_FILE_QUOTA):
raise serializers.ValidationError({
'quota': [_('The file quota for this project has been reached.')]
From 6b07b40659171d977ccf42cd55d8dd3fc6ca3fcd Mon Sep 17 00:00:00 2001
From: Jochen Klar
Date: Tue, 31 Oct 2023 11:55:27 +0100
Subject: [PATCH 10/47] Fix human2bytes utils function
---
rdmo/core/tests/test_utils.py | 14 +++++++++++++-
rdmo/core/utils.py | 2 +-
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/rdmo/core/tests/test_utils.py b/rdmo/core/tests/test_utils.py
index f0d9bfb119..87596487fd 100644
--- a/rdmo/core/tests/test_utils.py
+++ b/rdmo/core/tests/test_utils.py
@@ -1,6 +1,6 @@
import pytest
-from rdmo.core.utils import join_url, sanitize_url
+from rdmo.core.utils import human2bytes, join_url, sanitize_url
urls = (
('', ''),
@@ -23,3 +23,15 @@ def test_sanitize_url(url, sanitized_url):
def test_join_url():
assert join_url('https://example.com//', '/terms', 'foo') == 'https://example.com/terms/foo'
+
+
+def test_human2bytes():
+ assert human2bytes('1Gb') == 1e+9
+
+
+def test_human2bytes_none():
+ assert human2bytes(None) == 0
+
+
+def test_human2bytes_zero():
+ assert human2bytes('0') == 0
diff --git a/rdmo/core/utils.py b/rdmo/core/utils.py
index acfaf51fdd..b83c1a945a 100644
--- a/rdmo/core/utils.py
+++ b/rdmo/core/utils.py
@@ -351,7 +351,7 @@ def copy_model(instance, **kwargs):
def human2bytes(string):
- if not string:
+ if not string or string == '0':
return 0
m = re.match(r'([0-9.]+)\s*([A-Za-z]+)', string)
From a5c916fa101496e72d88f5a9f02c99e417260ea1 Mon Sep 17 00:00:00 2001
From: David Wallace
Date: Tue, 31 Oct 2023 17:22:35 +0100
Subject: [PATCH 11/47] chore: refactor human2bytes utils func
---
rdmo/core/constants.py | 18 ++++++++++++++++++
rdmo/core/tests/test_utils.py | 21 +++++++++++----------
rdmo/core/utils.py | 25 +++++--------------------
3 files changed, 34 insertions(+), 30 deletions(-)
diff --git a/rdmo/core/constants.py b/rdmo/core/constants.py
index 38a3ae6a45..e0c6fccd74 100644
--- a/rdmo/core/constants.py
+++ b/rdmo/core/constants.py
@@ -58,3 +58,21 @@
'views.add_view', 'views.change_view', 'views.delete_view'
)
}
+
+HUMAN2BYTES_MAPPER = {
+ "kb": {"base": 1000, "power": 1},
+ "k": {"base": 1000, "power": 1},
+ "mb": {"base": 1000, "power": 2},
+ "m": {"base": 1000, "power": 2},
+ "gb": {"base": 1000, "power": 3},
+ "g": {"base": 1000, "power": 3},
+ "tb": {"base": 1000, "power": 4},
+ "t": {"base": 1000, "power": 4},
+ "p": {"base": 1000, "power": 5},
+ "pb": {"base": 1000, "power": 5},
+ "kib": {"base": 1024, "power": 1},
+ "mib": {"base": 1024, "power": 2},
+ "gib": {"base": 1024, "power": 3},
+ "tib": {"base": 1024, "power": 4},
+ "pib": {"base": 1024, "power": 5},
+}
diff --git a/rdmo/core/tests/test_utils.py b/rdmo/core/tests/test_utils.py
index 87596487fd..5964a56d50 100644
--- a/rdmo/core/tests/test_utils.py
+++ b/rdmo/core/tests/test_utils.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
import pytest
from rdmo.core.utils import human2bytes, join_url, sanitize_url
@@ -15,6 +17,12 @@
(1, ''),
)
+human2bytes_test_values = (
+ ("1Gb", 1e+9),
+ (None, 0),
+ ("0", 0),
+)
+
@pytest.mark.parametrize("url,sanitized_url", urls)
def test_sanitize_url(url, sanitized_url):
@@ -25,13 +33,6 @@ def test_join_url():
assert join_url('https://example.com//', '/terms', 'foo') == 'https://example.com/terms/foo'
-def test_human2bytes():
- assert human2bytes('1Gb') == 1e+9
-
-
-def test_human2bytes_none():
- assert human2bytes(None) == 0
-
-
-def test_human2bytes_zero():
- assert human2bytes('0') == 0
+@pytest.mark.parametrize("human,bytes", human2bytes_test_values)
+def test_human2bytes(human: Optional[str], bytes: float):
+ assert human2bytes(human) == bytes
diff --git a/rdmo/core/utils.py b/rdmo/core/utils.py
index b83c1a945a..c2c944028c 100644
--- a/rdmo/core/utils.py
+++ b/rdmo/core/utils.py
@@ -18,6 +18,8 @@
from defusedcsv import csv
from markdown import markdown
+from .constants import HUMAN2BYTES_MAPPER
+
log = logging.getLogger(__name__)
@@ -357,26 +359,9 @@ def human2bytes(string):
m = re.match(r'([0-9.]+)\s*([A-Za-z]+)', string)
number, unit = float(m.group(1)), m.group(2).strip().lower()
- if unit == 'kb' or unit == 'k':
- return number * 1000
- elif unit == 'mb' or unit == 'm':
- return number * 1000**2
- elif unit == 'gb' or unit == 'g':
- return number * 1000**3
- elif unit == 'tb' or unit == 't':
- return number * 1000**4
- elif unit == 'pb' or unit == 'p':
- return number * 1000**5
- elif unit == 'kib':
- return number * 1024
- elif unit == 'mib':
- return number * 1024**2
- elif unit == 'gib':
- return number * 1024**3
- elif unit == 'tib':
- return number * 1024**4
- elif unit == 'pib':
- return number * 1024**5
+ conversion = HUMAN2BYTES_MAPPER[unit]
+ number = number*conversion['base']**(conversion['power'])
+ return number
def is_truthy(value):
From e31c155b4506359d7924d52eb3d030f59a7ffeee Mon Sep 17 00:00:00 2001
From: Jochen Klar
Date: Tue, 31 Oct 2023 18:37:03 +0100
Subject: [PATCH 12/47] Fix ValueQuotaValidator and add tests
---
rdmo/projects/tests/test_validator_quota.py | 65 +++++++++++++++++++++
rdmo/projects/validators.py | 15 ++---
2 files changed, 71 insertions(+), 9 deletions(-)
create mode 100644 rdmo/projects/tests/test_validator_quota.py
diff --git a/rdmo/projects/tests/test_validator_quota.py b/rdmo/projects/tests/test_validator_quota.py
new file mode 100644
index 0000000000..f700287cac
--- /dev/null
+++ b/rdmo/projects/tests/test_validator_quota.py
@@ -0,0 +1,65 @@
+import pytest
+
+from rest_framework.exceptions import ValidationError as RestFameworkValidationError
+
+from ..serializers.v1 import ValueSerializer
+from ..validators import ValueQuotaValidator
+
+project_id = 1
+attribute_path = attribute__path='individual/single/text'
+
+
+def test_serializer_create_file(db):
+ class MockedProject:
+ file_size = 1
+
+ class MockedView:
+ action = 'create'
+ project = MockedProject()
+
+ validator = ValueQuotaValidator()
+ serializer = ValueSerializer()
+ serializer.context['view'] = MockedView()
+
+ validator({
+ 'value_type': 'file'
+ }, serializer)
+
+
+def test_serializer_create_file_error(db, settings):
+ class MockedProject:
+ file_size = 1
+
+ class MockedView:
+ action = 'create'
+ project = MockedProject()
+
+ settings.PROJECT_FILE_QUOTA = '0'
+
+ validator = ValueQuotaValidator()
+ serializer = ValueSerializer()
+ serializer.context['view'] = MockedView()
+
+ with pytest.raises(RestFameworkValidationError):
+ validator({
+ 'value_type': 'file'
+ }, serializer)
+
+
+def test_serializer_create_text(db, settings):
+ class MockedProject:
+ file_size = 1
+
+ class MockedView:
+ action = 'create'
+ project = MockedProject()
+
+ settings.PROJECT_FILE_QUOTA = '0'
+
+ validator = ValueQuotaValidator()
+ serializer = ValueSerializer()
+ serializer.context['view'] = MockedView()
+
+ validator({
+ 'value_type': 'text'
+ }, serializer)
diff --git a/rdmo/projects/validators.py b/rdmo/projects/validators.py
index 086a387f4d..65f2deebd5 100644
--- a/rdmo/projects/validators.py
+++ b/rdmo/projects/validators.py
@@ -42,12 +42,9 @@ class ValueQuotaValidator:
requires_context = True
def __call__(self, data, serializer):
- if data.get('value_type') == VALUE_TYPE_FILE:
- try:
- serializer.context['view'].get_object()
- except AssertionError as e:
- project = serializer.context['view'].project
- if project.file_size > human2bytes(settings.PROJECT_FILE_QUOTA):
- raise serializers.ValidationError({
- 'quota': [_('The file quota for this project has been reached.')]
- }) from e
+ if serializer.context['view'].action == 'create' and data.get('value_type') == VALUE_TYPE_FILE:
+ project = serializer.context['view'].project
+ if project.file_size > human2bytes(settings.PROJECT_FILE_QUOTA):
+ raise serializers.ValidationError({
+ 'quota': [_('The file quota for this project has been reached.')]
+ })
From 6cc0dbdf6c99f8ce55c90b964888f2e73a4a0da4 Mon Sep 17 00:00:00 2001
From: Jochen Klar
Date: Tue, 31 Oct 2023 18:43:22 +0100
Subject: [PATCH 13/47] Update HUMAN2BYTES_MAPPER and add test
---
rdmo/core/constants.py | 1 +
rdmo/projects/tests/test_validator_quota.py | 23 ++++++++++++++++++++-
2 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/rdmo/core/constants.py b/rdmo/core/constants.py
index e0c6fccd74..1e1cbc3a78 100644
--- a/rdmo/core/constants.py
+++ b/rdmo/core/constants.py
@@ -60,6 +60,7 @@
}
HUMAN2BYTES_MAPPER = {
+ "b": {"base": 1000, "power": 0},
"kb": {"base": 1000, "power": 1},
"k": {"base": 1000, "power": 1},
"mb": {"base": 1000, "power": 2},
diff --git a/rdmo/projects/tests/test_validator_quota.py b/rdmo/projects/tests/test_validator_quota.py
index f700287cac..4eee080e26 100644
--- a/rdmo/projects/tests/test_validator_quota.py
+++ b/rdmo/projects/tests/test_validator_quota.py
@@ -9,7 +9,7 @@
attribute_path = attribute__path='individual/single/text'
-def test_serializer_create_file(db):
+def test_serializer_create_file(db, settings):
class MockedProject:
file_size = 1
@@ -17,6 +17,8 @@ class MockedView:
action = 'create'
project = MockedProject()
+ settings.PROJECT_FILE_QUOTA = '1b'
+
validator = ValueQuotaValidator()
serializer = ValueSerializer()
serializer.context['view'] = MockedView()
@@ -63,3 +65,22 @@ class MockedView:
validator({
'value_type': 'text'
}, serializer)
+
+
+def test_serializer_update(db, settings):
+ class MockedProject:
+ file_size = 1
+
+ class MockedView:
+ action = 'update'
+ project = MockedProject()
+
+ settings.PROJECT_FILE_QUOTA = '0'
+
+ validator = ValueQuotaValidator()
+ serializer = ValueSerializer()
+ serializer.context['view'] = MockedView()
+
+ validator({
+ 'value_type': 'file'
+ }, serializer)
From 41c92fae147c14e863b24e8ee05a0c8930ae6217 Mon Sep 17 00:00:00 2001
From: Jochen Klar
Date: Tue, 22 Aug 2023 17:53:24 +0200
Subject: [PATCH 14/47] Refactor progress bar and take sets and conditions into
account (#596)
---
rdmo/projects/models/project.py | 22 ++------
rdmo/projects/progress.py | 90 +++++++++++++++++++++++++++++++++
2 files changed, 93 insertions(+), 19 deletions(-)
create mode 100644 rdmo/projects/progress.py
diff --git a/rdmo/projects/models/project.py b/rdmo/projects/models/project.py
index eae2128fe2..48d2acc477 100644
--- a/rdmo/projects/models/project.py
+++ b/rdmo/projects/models/project.py
@@ -2,7 +2,6 @@
from django.contrib.sites.models import Site
from django.core.exceptions import ValidationError
from django.db import models
-from django.db.models import Exists, OuterRef
from django.db.models.signals import pre_delete
from django.dispatch import receiver
from django.urls import reverse
@@ -12,12 +11,12 @@
from mptt.models import MPTTModel, TreeForeignKey
from rdmo.core.models import Model
-from rdmo.domain.models import Attribute
-from rdmo.questions.models import Catalog, Question
+from rdmo.questions.models import Catalog
from rdmo.tasks.models import Task
from rdmo.views.models import View
from ..managers import ProjectManager
+from ..progress import get_progress
class Project(MPTTModel, Model):
@@ -88,22 +87,7 @@ def clean(self):
@property
def progress(self):
- # create a queryset for the attributes of the catalog for this project
- # the subquery is used to query only attributes which have a question in the catalog, which is not optional
- questions = Question.objects.filter_by_catalog(self.catalog) \
- .filter(attribute_id=OuterRef('pk')).exclude(is_optional=True)
- attributes = Attribute.objects.annotate(active=Exists(questions)).filter(active=True).distinct()
-
- # query the total number of attributes from the qs above
- total = attributes.count()
-
- # query all current values with attributes from the qs above, but where the text, option, or file field is set,
- # and count only one value per attribute
- values = self.values.filter(snapshot=None) \
- .filter(attribute__in=attributes) \
- .exclude((models.Q(text='') | models.Q(text=None)) & models.Q(option=None) &
- (models.Q(file='') | models.Q(file=None))) \
- .distinct().values('attribute').count()
+ values, total = get_progress(self)
try:
ratio = values / total
diff --git a/rdmo/projects/progress.py b/rdmo/projects/progress.py
new file mode 100644
index 0000000000..b058879a95
--- /dev/null
+++ b/rdmo/projects/progress.py
@@ -0,0 +1,90 @@
+from collections import defaultdict
+
+from django.db.models import Exists, OuterRef, Q
+
+from rdmo.conditions.models import Condition
+from rdmo.questions.models import Catalog, Section, Page, QuestionSet, Question
+
+
+def get_progress(project, snapshot=None):
+ # get all values for this project and snapshot
+ project_values = project.values.filter(snapshot=snapshot).select_related('attribute', 'option')
+
+ # get all conditions for this catalog
+ pages_conditions_subquery = Page.objects.filter_by_catalog(project.catalog).filter(conditions=OuterRef('pk'))
+ questionsets_conditions_subquery = QuestionSet.objects.filter_by_catalog(project.catalog).filter(conditions=OuterRef('pk'))
+ questions_conditions_subquery = Question.objects.filter_by_catalog(project.catalog).filter(conditions=OuterRef('pk'))
+
+ catalog_conditions = Condition.objects.annotate(has_page=Exists(pages_conditions_subquery)) \
+ .annotate(has_questionset=Exists(questionsets_conditions_subquery)) \
+ .annotate(has_question=Exists(questions_conditions_subquery)) \
+ .filter(Q(has_page=True) | Q(has_questionset=True) | Q(has_question=True)) \
+ .distinct().select_related('source', 'target_option')
+
+ # evaluate conditions
+ conditions = set()
+ for condition in catalog_conditions:
+ if condition.resolve(project_values):
+ conditions.add(condition.id)
+
+ # compute sets from values
+ sets = defaultdict(list)
+ for attribute, set_index in project_values.values_list('attribute', 'set_index').distinct():
+ sets[attribute].append(set_index)
+
+ # count the total number of questions, taking sets and conditions into account
+ total_count, attributes = count_questions(project.catalog, sets, conditions)
+
+ # filter the project values for the counted questions and exclude empty values
+ values_count = project_values.filter(attribute__in=attributes) \
+ .exclude((Q(text='') | Q(text=None)) & Q(option=None) &
+ (Q(file='') | Q(file=None))) \
+ .count()
+
+ return values_count, total_count
+
+
+def count_questions(parent_element, sets, conditions):
+ count = 0
+ attributes = []
+
+ for element in parent_element.elements:
+ if isinstance(element, (Catalog, Section)):
+ element_count, element_attributes = count_questions(element, sets, conditions)
+ attributes += element_attributes
+ count += element_count
+ else:
+ element_conditions = set(condition.id for condition in element.conditions.all())
+ if not element_conditions or element_conditions.intersection(conditions):
+ if isinstance(element, Question):
+ attributes.append(element.attribute)
+ count += 1
+ else:
+ if element.attribute:
+ attributes.append(element.attribute)
+
+ element_count, element_attributes = count_questions(element, sets, conditions)
+ set_count = count_sets(element, sets)
+ if set_count > 0:
+ count += element_count * set_count
+ attributes += element_attributes
+
+ return count, attributes
+
+
+def count_sets(parent_element, sets):
+ if parent_element.is_collection:
+ if parent_element.attribute:
+ count = len(sets[parent_element.attribute_id])
+ else:
+ count = 0
+ else:
+ count = 1
+
+ for element in parent_element.elements:
+ if isinstance(element, Question):
+ element_count = len(sets[element.attribute_id])
+ if element_count > count:
+ count = element_count
+
+ return count
From e8bb565b1fd187fe13314eb6bbef2a7480e2157e Mon Sep 17 00:00:00 2001
From: Jochen Klar
Date: Tue, 22 Aug 2023 18:01:13 +0200
Subject: [PATCH 15/47] Check is_optional for progress bar (again)
---
rdmo/projects/progress.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/rdmo/projects/progress.py b/rdmo/projects/progress.py
index b058879a95..8e0693771a 100644
--- a/rdmo/projects/progress.py
+++ b/rdmo/projects/progress.py
@@ -57,8 +57,9 @@ def count_questions(parent_element, sets, conditions):
element_conditions = set(condition.id for condition in element.conditions.all())
if not element_conditions or element_conditions.intersection(conditions):
if isinstance(element, Question):
- attributes.append(element.attribute)
- count += 1
+ if not element.is_optional:
+ attributes.append(element.attribute)
+ count += 1
else:
if element.attribute:
attributes.append(element.attribute)
From 39965a2076adf278131eaa3b483cd808ffc39b54 Mon Sep 17 00:00:00 2001
From: Jochen Klar
Date: Thu, 24 Aug 2023 16:51:07 +0200
Subject: [PATCH 16/47] Add progress_count and progress_total to Project model,
compute on POST when using the API and add progress to the project overview
(#488)
---
.eslintrc.js | 2 +-
rdmo/core/static/core/js/core.js | 7 +++++
.../migrations/0059_project_progress.py | 23 ++++++++++++++++
rdmo/projects/models/project.py | 26 +++++++------------
rdmo/projects/progress.py | 2 +-
.../projects/js/project_questions/services.js | 6 ++---
.../projects/project_questions_progress.html | 4 +--
.../projects/templates/projects/projects.html | 4 +++
.../templates/projects/site_projects.html | 6 ++++-
rdmo/projects/templatetags/projects_tags.py | 13 ++++++++++
rdmo/projects/viewsets.py | 22 +++++++++++++---
11 files changed, 87 insertions(+), 28 deletions(-)
create mode 100644 rdmo/projects/migrations/0059_project_progress.py
diff --git a/.eslintrc.js b/.eslintrc.js
index 1419f87dfe..5d67f9ab04 100644
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -44,5 +44,5 @@ module.exports = {
'react': {
'version': 'detect'
}
- }
+ },
}
diff --git a/rdmo/core/static/core/js/core.js b/rdmo/core/static/core/js/core.js
index 39f154fc2c..ee790f74fa 100644
--- a/rdmo/core/static/core/js/core.js
+++ b/rdmo/core/static/core/js/core.js
@@ -16,6 +16,13 @@ angular.module('core', ['ngResource'])
method: 'PUT',
params: {}
};
+ $resourceProvider.defaults.actions.postAction = {
+ method: 'POST',
+ params: {
+ id: '@id',
+ detail_action: '@detail_action'
+ }
+ };
}])
.filter('capitalize', function() {
diff --git a/rdmo/projects/migrations/0059_project_progress.py b/rdmo/projects/migrations/0059_project_progress.py
new file mode 100644
index 0000000000..abce63ea50
--- /dev/null
+++ b/rdmo/projects/migrations/0059_project_progress.py
@@ -0,0 +1,23 @@
+# Generated by Django 3.2.19 on 2023-08-24 09:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('projects', '0058_meta'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='project',
+ name='progress_count',
+ field=models.IntegerField(help_text='The number of values for the progress bar.', null=True, verbose_name='Progress count'),
+ ),
+ migrations.AddField(
+ model_name='project',
+ name='progress_total',
+ field=models.IntegerField(help_text='The total number of expected values for the progress bar.', null=True, verbose_name='Progress total'),
+ ),
+ ]
diff --git a/rdmo/projects/models/project.py b/rdmo/projects/models/project.py
index 48d2acc477..d13b33a60a 100644
--- a/rdmo/projects/models/project.py
+++ b/rdmo/projects/models/project.py
@@ -16,7 +16,6 @@
from rdmo.views.models import View
from ..managers import ProjectManager
-from ..progress import get_progress
class Project(MPTTModel, Model):
@@ -64,6 +63,16 @@ class Project(MPTTModel, Model):
verbose_name=_('Views'),
help_text=_('The views that will be used for this project.')
)
+ progress_total = models.IntegerField(
+ null=True,
+ verbose_name=_('Progress total'),
+ help_text=_('The total number of expected values for the progress bar.')
+ )
+ progress_count = models.IntegerField(
+ null=True,
+ verbose_name=_('Progress count'),
+ help_text=_('The number of values for the progress bar.')
+ )
class Meta:
ordering = ('tree_id', 'level', 'title')
@@ -85,21 +94,6 @@ def clean(self):
'parent': [_('A project may not be moved to be a child of itself or one of its descendants.')]
})
- @property
- def progress(self):
- values, total = get_progress(self)
-
- try:
- ratio = values / total
- except ZeroDivisionError:
- ratio = 0
-
- return {
- 'total': total,
- 'values': values,
- 'ratio': ratio
- }
-
@property
def catalog_uri(self):
if self.catalog is not None:
diff --git a/rdmo/projects/progress.py b/rdmo/projects/progress.py
index 8e0693771a..6271c11bbd 100644
--- a/rdmo/projects/progress.py
+++ b/rdmo/projects/progress.py
@@ -6,7 +6,7 @@
from rdmo.questions.models import Catalog, Section, Page, QuestionSet, Question
-def get_progress(project, snapshot=None):
+def compute_progress(project, snapshot=None):
# get all values for this project and snapshot
project_values = project.values.filter(snapshot=snapshot).select_related('attribute', 'option')
diff --git a/rdmo/projects/static/projects/js/project_questions/services.js b/rdmo/projects/static/projects/js/project_questions/services.js
index 92b09fc744..9d511bb26b 100644
--- a/rdmo/projects/static/projects/js/project_questions/services.js
+++ b/rdmo/projects/static/projects/js/project_questions/services.js
@@ -955,13 +955,11 @@ angular.module('project_questions')
}
} else {
// update progress
- resources.projects.get({
+ resources.projects.postAction({
id: service.project.id,
detail_action: 'progress'
}, function(response) {
- if (service.progress.values != response.values) {
- service.progress = response
- }
+ service.progress = response
});
// check if we need to refresh the site
diff --git a/rdmo/projects/templates/projects/project_questions_progress.html b/rdmo/projects/templates/projects/project_questions_progress.html
index 9b66e61bb9..1ae56dad41 100644
--- a/rdmo/projects/templates/projects/project_questions_progress.html
+++ b/rdmo/projects/templates/projects/project_questions_progress.html
@@ -3,8 +3,8 @@
- {% blocktrans trimmed with values='{$ service.progress.values $}' total='{$ service.progress.total $}' %}
- {{ values }} of {{ total }}
+ {% blocktrans trimmed with count='{$ service.progress.count $}' total='{$ service.progress.total $}' %}
+ {{ count }} of {{ total }}
{% endblocktrans %}
diff --git a/rdmo/projects/templates/projects/projects.html b/rdmo/projects/templates/projects/projects.html
index 0edfaace36..b2aed864a1 100644
--- a/rdmo/projects/templates/projects/projects.html
+++ b/rdmo/projects/templates/projects/projects.html
@@ -124,6 +124,7 @@
{% trans 'My Projects' %}
{% trans 'Name' %} |
+ {% trans 'Progress' %} |
{% trans 'Role' %} |
{% trans 'Last changed' %} |
|
@@ -138,6 +139,9 @@ {% trans 'My Projects' %}
{{ project.title }}
+
+ {% project_progress project %}
+ |
{{ project.role|projects_role }}
|
diff --git a/rdmo/projects/templates/projects/site_projects.html b/rdmo/projects/templates/projects/site_projects.html
index f2a8f49c23..5caba675ed 100644
--- a/rdmo/projects/templates/projects/site_projects.html
+++ b/rdmo/projects/templates/projects/site_projects.html
@@ -61,7 +61,8 @@ {% blocktrans trimmed with site=request.site %}All projects on {{ site }}{%
- {% trans 'Name' %} |
+ {% trans 'Name' %} |
+ {% trans 'Progress' %} |
{% trans 'Created' %} |
{% trans 'Last changed' %} |
|
@@ -76,6 +77,9 @@