diff --git a/rdmo/core/constants.py b/rdmo/core/constants.py index 7e96075a81..d78c29f4a5 100644 --- a/rdmo/core/constants.py +++ b/rdmo/core/constants.py @@ -5,6 +5,7 @@ VALUE_TYPE_INTEGER = 'integer' VALUE_TYPE_FLOAT = 'float' VALUE_TYPE_BOOLEAN = 'boolean' +VALUE_TYPE_DATE = 'date' VALUE_TYPE_DATETIME = 'datetime' VALUE_TYPE_OPTIONS = 'option' VALUE_TYPE_EMAIL = 'email' @@ -16,6 +17,7 @@ (VALUE_TYPE_INTEGER, _('Integer')), (VALUE_TYPE_FLOAT, _('Float')), (VALUE_TYPE_BOOLEAN, _('Boolean')), + (VALUE_TYPE_DATE, _('Date')), (VALUE_TYPE_DATETIME, _('Datetime')), (VALUE_TYPE_EMAIL, _('E-mail')), (VALUE_TYPE_PHONE, _('Phone')), diff --git a/rdmo/core/settings.py b/rdmo/core/settings.py index bb0014def5..a5c75b9cc8 100644 --- a/rdmo/core/settings.py +++ b/rdmo/core/settings.py @@ -1,3 +1,5 @@ +import re + from django.utils.translation import gettext_lazy as _ SITE_ID = 1 @@ -313,6 +315,56 @@ OPTIONSET_PROVIDERS = [] +PROJECT_VALUES_VALIDATION = False + +PROJECT_VALUES_VALIDATION_URL = True + +PROJECT_VALUES_VALIDATION_INTEGER = True +PROJECT_VALUES_VALIDATION_INTEGER_MESSAGE = _('Enter a valid integer.') +PROJECT_VALUES_VALIDATION_INTEGER_REGEX = re.compile(r'^[+-]?\d+$') + +PROJECT_VALUES_VALIDATION_FLOAT = True +PROJECT_VALUES_VALIDATION_FLOAT_MESSAGE = _('Enter a valid float.') +PROJECT_VALUES_VALIDATION_FLOAT_REGEX = re.compile(r''' + ^[+-]? # Optional sign + ( + \d+ # Digits before the decimal or thousands separator + (,\d{3})* # Optional groups of exactly three digits preceded by a comma (thousands separator) + (\.\d+)? # Optional decimal part, a dot followed by one or more digits + | # OR + \d+ # Digits before the decimal or thousands separator + (\.\d{3})* # Optional groups of exactly three digits preceded by a dot (thousands separator) + (,\d+)? # Optional decimal part, a comma followed by one or more digits + ) + ([eE][+-]?\d+)?$ # Optional exponent part +''', re.VERBOSE) + +PROJECT_VALUES_VALIDATION_BOOLEAN = True +PROJECT_VALUES_VALIDATION_BOOLEAN_MESSAGE = _('Enter a valid boolean (e.g. 0, 1).') +PROJECT_VALUES_VALIDATION_BOOLEAN_REGEX = r'(?i)^(0|1|f|t|false|true)$' + +PROJECT_VALUES_VALIDATION_DATE = True +PROJECT_VALUES_VALIDATION_DATE_MESSAGE = _('Enter a valid date (e.g. "02.03.2024", "03/02/2024", "2024-02-03").') +PROJECT_VALUES_VALIDATION_DATE_REGEX = re.compile(r''' + ^( + \d{1,2}\.\s*\d{1,2}\.\s*\d{2,4} # Format dd.mm.yyyy + | \d{1,2}/\d{1,2}/\d{4} # Format mm/dd/yyyy + | \d{4}-\d{2}-\d{2} # Format yyyy-mm-dd + )$ +''', re.VERBOSE) + +PROJECT_VALUES_VALIDATION_DATETIME = True + +PROJECT_VALUES_VALIDATION_EMAIL = True + +PROJECT_VALUES_VALIDATION_PHONE = True +PROJECT_VALUES_VALIDATION_PHONE_MESSAGE = _('Enter a valid phone number (e.g. "123456" or "+49 (0) 30 123456").') +PROJECT_VALUES_VALIDATION_PHONE_REGEX = re.compile(r''' + ^([+]\d{2,3}\s)? # Optional country code + (\(\d+\)\s)? # Optional area code in parentheses + [\d\s]*$ # Main number with spaces +''', re.VERBOSE) + QUESTIONS_WIDGETS = [ ('text', _('Text'), 'rdmo.projects.widgets.TextWidget'), ('textarea', _('Textarea'), 'rdmo.projects.widgets.TextareaWidget'), diff --git a/rdmo/management/tests/test_import_questions.py b/rdmo/management/tests/test_import_questions.py index 0f2083ad39..6145c3fce4 100644 --- a/rdmo/management/tests/test_import_questions.py +++ b/rdmo/management/tests/test_import_questions.py @@ -283,6 +283,15 @@ def test_update_legacy_questions(db, settings): # check that all elements ended up in the catalog catalog = Catalog.objects.prefetch_elements().first() - descendant_uris = {element.uri for element in catalog.descendants} + descendant_uris = { + element.uri for element in catalog.descendants if any(element.uri.startswith(uri) for uri in [ + 'http://example.com/terms/questions/catalog/individual', + 'http://example.com/terms/questions/catalog/set', + 'http://example.com/terms/questions/catalog/collections', + 'http://example.com/terms/questions/catalog/conditions', + 'http://example.com/terms/questions/catalog/options', + 'http://example.com/terms/questions/catalog/blocks' + ]) + } element_uris = {element['uri'] for _uri, element in elements.items() if element['uri'] != catalog.uri} assert descendant_uris == element_uris diff --git a/rdmo/projects/migrations/0061_alter_value_value_type.py b/rdmo/projects/migrations/0061_alter_value_value_type.py new file mode 100644 index 0000000000..c797b9131a --- /dev/null +++ b/rdmo/projects/migrations/0061_alter_value_value_type.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.8 on 2024-07-18 14:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0060_alter_issue_options'), + ] + + operations = [ + migrations.AlterField( + model_name='value', + name='value_type', + field=models.CharField(choices=[('text', 'Text'), ('url', 'URL'), ('integer', 'Integer'), ('float', 'Float'), ('boolean', 'Boolean'), ('date', 'Date'), ('datetime', 'Datetime'), ('email', 'E-mail'), ('phone', 'Phone'), ('option', 'Option'), ('file', 'File')], default='text', help_text='Type of this value.', max_length=8, verbose_name='Value type'), + ), + ] diff --git a/rdmo/projects/serializers/v1/__init__.py b/rdmo/projects/serializers/v1/__init__.py index 11618b42d9..8816dc9ca7 100644 --- a/rdmo/projects/serializers/v1/__init__.py +++ b/rdmo/projects/serializers/v1/__init__.py @@ -9,7 +9,7 @@ from rdmo.services.validators import ProviderValidator from ...models import Integration, IntegrationOption, Invite, Issue, IssueResource, Membership, Project, Snapshot, Value -from ...validators import ValueConflictValidator, ValueQuotaValidator +from ...validators import ValueConflictValidator, ValueQuotaValidator, ValueTypeValidator class UserSerializer(serializers.ModelSerializer): @@ -267,7 +267,8 @@ class Meta: ) validators = ( ValueConflictValidator(), - ValueQuotaValidator() + ValueQuotaValidator(), + ValueTypeValidator() ) diff --git a/rdmo/projects/static/projects/js/project_questions/services.js b/rdmo/projects/static/projects/js/project_questions/services.js index f7079016d5..1e7a33398e 100644 --- a/rdmo/projects/static/projects/js/project_questions/services.js +++ b/rdmo/projects/static/projects/js/project_questions/services.js @@ -885,7 +885,11 @@ angular.module('project_questions') service.error = response; } else if (response.status == 400) { service.error = true; - value.errors = Object.keys(response.data); + if (angular.isDefined(response.data.text)) { + value.errors = response.data.text + } else { + value.errors = Object.keys(response.data); + } } else if (response.status == 404) { service.error = true; value.errors = ['not_found'] diff --git a/rdmo/projects/templates/projects/project_questions_value_errors.html b/rdmo/projects/templates/projects/project_questions_value_errors.html index d8245287ad..b4224ca869 100644 --- a/rdmo/projects/templates/projects/project_questions_value_errors.html +++ b/rdmo/projects/templates/projects/project_questions_value_errors.html @@ -17,5 +17,8 @@ {% trans 'You reached the file quota for this project.' %} + + {$ error $} + diff --git a/rdmo/projects/tests/test_progress.py b/rdmo/projects/tests/test_progress.py index 31ccbdde36..2b653dc4cd 100644 --- a/rdmo/projects/tests/test_progress.py +++ b/rdmo/projects/tests/test_progress.py @@ -6,8 +6,8 @@ projects = [1, 11] results_map = { - 1: (58, 81), - 11: (0, 29) + 1: (58, 88), + 11: (0, 36) } diff --git a/rdmo/projects/tests/test_validator_value_type.py b/rdmo/projects/tests/test_validator_value_type.py new file mode 100644 index 0000000000..88bf0a46e5 --- /dev/null +++ b/rdmo/projects/tests/test_validator_value_type.py @@ -0,0 +1,113 @@ +import pytest + +from rest_framework.exceptions import ValidationError as RestFameworkValidationError + +from ..validators import ValueTypeValidator + +data = ( + ('url', 'https://example.com'), + ('url', 'http://example.com'), + ('integer', '1'), + ('integer', '-1'), + ('integer', '+1'), + ('integer', '12345'), + ('float', '1'), + ('float', '1.0'), + ('float', '+1.0'), + ('float', '-1.0'), + ('float', '1,000,000.12345'), + ('float', '1,0'), + ('float', '1.000.000,12345'), + ('float', '1.0e20'), + ('float', '1.0E20'), + ('float', '1.0e-20'), + ('float', '1.0e+20'), + ('boolean', '0'), + ('boolean', '1'), + ('boolean', 'f'), + ('boolean', 't'), + ('boolean', 'TrUe'), # spellchecker:disable-line + ('boolean', 'FaLsE'), # spellchecker:disable-line + ('boolean', 'true'), + ('boolean', 'false'), + ('date', '01.02.2024'), + ('date', '1.2.2024'), + ('date', '13.01.1337'), + ('date', '2/1/2024'), + ('date', '2024-01-02'), + ('date', '1. 2. 2024'), + ('datetime', '2024-01-02'), + ('datetime', '2024-01-02T10:00'), + ('datetime', '2024-01-02T10:00:00'), + ('datetime', '2024-01-02T10:00:00.123'), + ('datetime', '2024-01-02T10:00:00+02:00'), + ('email', 'user@example.com'), + ('email', 'user+test@example.com'), + ('email', 'user!test@example.com'), + ('email', 'user.name@example.com'), + ('email', 'user.name+tag@example.com'), + ('phone', '123456'), + ('phone', '123 456'), + ('phone', '362 123456'), + ('phone', '(362) 123456'), + ('phone', '+49 (0) 362123456'), + ('phone', '+49 (0) 362 123456'), +) +data_error = ( + ('url', 'wrong'), + ('url', 'example.com'), + ('integer', 'wrong'), + ('integer', '1.0'), + ('integer', '1b'), + ('float', 'wrong'), + ('float', '1,0000.12456'), + ('float', '1.0000,12456'), + ('float', '1.0a20'), + ('boolean', 'wrong'), + ('boolean', '2'), + ('boolean', '-1'), + ('boolean', 'tr'), + ('boolean', 'truee'), + ('boolean', 'falze'), + ('date', 'wrong'), + ('date', '001.02.2024'), + ('date', '01.02.20240'), + ('date', '1,2.2024'), + ('date', '2-1-2024'), + ('date', '2024-001-02'), + ('date', '20240-01-02'), + ('date', '2024-1-2'), + ('datetime', 'wrong'), + ('datetime', '2024-13-02'), + ('datetime', '2024-13-02Y10:00:00'), + ('datetime', '2024-01-02T10:00:00ZZ+02:00'), + ('datetime', '2024-01-02T25:00'), + ('datetime', '2024-01-02T10:60:00'), + ('email', 'wrong'), + ('email', 'example.com'), + ('email', 'üser@example.com'), + ('email', 'user@test@example.com'), + ('email', 'user@com'), + ('email', 'user@.com'), + ('phone', 'wrong'), + ('phone', '123456a'), + ('phone', '123 456 a'), + ('phone', '362s 123456'), + ('phone', '(3 62) 123456'), + ('phone', '-49 (0) 362123456'), + ('phone', '49 (0) 362 123456'), + ('phone', '1234 (0) 123456'), +) + + +@pytest.mark.parametrize('value_type,text', data) +def test_serializer(db, value_type, text): + validator = ValueTypeValidator() + validator({'value_type': value_type, 'text': text}) + + +@pytest.mark.parametrize('value_type,text', data_error) +def test_serializer_error(db, value_type, text): + validator = ValueTypeValidator() + with pytest.raises(RestFameworkValidationError): + validator({'value_type': value_type, 'text': text}) diff --git a/rdmo/projects/tests/test_viewset_project_progress.py b/rdmo/projects/tests/test_viewset_project_progress.py index 02eaabc6b2..45f900d68f 100644 --- a/rdmo/projects/tests/test_viewset_project_progress.py +++ b/rdmo/projects/tests/test_viewset_project_progress.py @@ -119,7 +119,7 @@ def test_progress_post_unchanged(db, client): project = Project.objects.get(id=1) project.progress_count = progress_count = 58 # the progress in the fixture is not up-to-date - project.progress_total = progress_total = 81 + project.progress_total = progress_total = 88 project.save() project.refresh_from_db() project_updated = project.updated diff --git a/rdmo/projects/tests/test_viewset_project_value.py b/rdmo/projects/tests/test_viewset_project_value.py index 816329b09a..0b5afb6ed0 100644 --- a/rdmo/projects/tests/test_viewset_project_value.py +++ b/rdmo/projects/tests/test_viewset_project_value.py @@ -5,7 +5,7 @@ from django.conf import settings from django.urls import reverse -from rdmo.core.constants import VALUE_TYPE_CHOICES, VALUE_TYPE_FILE, VALUE_TYPE_TEXT +from rdmo.core.constants import VALUE_TYPE_FILE, VALUE_TYPE_TEXT from ..models import Value @@ -56,6 +56,16 @@ ] set_questionsets = [42, 43] +value_texts = ( + ('text', 'Lorem ipsum'), + ('url', 'https://lorem.ipsum'), + ('integer', '1337'), + ('float', '13.37'), + ('boolean', '1'), + ('datetime', '1337-01-13T13:37+13:37'), + ('email', 'user@lorem.ipsum'), + ('phone', '+49 (0) 1337 12345678') +) @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) @@ -101,8 +111,8 @@ def test_detail(db, client, username, password, project_id, value_id): @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) -@pytest.mark.parametrize('value_type,value_type_label', VALUE_TYPE_CHOICES) -def test_create_text(db, client, username, password, project_id, value_type, value_type_label): +@pytest.mark.parametrize('value_type,value_text', value_texts) +def test_create_text(db, client, username, password, project_id, value_type, value_text): client.login(username=username, password=password) url = reverse(urlnames['list'], args=[project_id]) @@ -110,14 +120,14 @@ def test_create_text(db, client, username, password, project_id, value_type, val 'attribute': attribute_id, 'set_index': 0, 'collection_index': 0, - 'text': 'Lorem ipsum', + 'text': value_text, 'value_type': value_type, 'unit': '' } response = client.post(url, data) if project_id in add_value_permission_map.get(username, []): - assert response.status_code == 201 + assert response.status_code == 201, response.content assert isinstance(response.json(), dict) assert response.json().get('id') in Value.objects.filter(project_id=project_id).values_list('id', flat=True) elif project_id in view_value_permission_map.get(username, []): @@ -128,8 +138,8 @@ def test_create_text(db, client, username, password, project_id, value_type, val @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) -@pytest.mark.parametrize('value_type,value_type_label', VALUE_TYPE_CHOICES) -def test_create_option(db, client, username, password, project_id, value_type, value_type_label): +@pytest.mark.parametrize('value_type,value_text', value_texts) +def test_create_option(db, client, username, password, project_id, value_type, value_text): client.login(username=username, password=password) url = reverse(urlnames['list'], args=[project_id]) @@ -137,6 +147,7 @@ def test_create_option(db, client, username, password, project_id, value_type, v 'attribute': attribute_id, 'set_index': 0, 'collection_index': 0, + 'text': value_text, 'option': option_id, 'value_type': value_type, 'unit': '' @@ -155,8 +166,8 @@ def test_create_option(db, client, username, password, project_id, value_type, v @pytest.mark.parametrize('username,password', users) @pytest.mark.parametrize('project_id', projects) -@pytest.mark.parametrize('value_type,value_type_label', VALUE_TYPE_CHOICES) -def test_create_external(db, client, username, password, project_id, value_type, value_type_label): +@pytest.mark.parametrize('value_type,value_text', value_texts) +def test_create_external(db, client, username, password, project_id, value_type, value_text): client.login(username=username, password=password) url = reverse(urlnames['list'], args=[project_id]) @@ -164,7 +175,7 @@ def test_create_external(db, client, username, password, project_id, value_type, 'attribute': attribute_id, 'set_index': 0, 'collection_index': 0, - 'text': 'Lorem ipsum', + 'text': value_text, 'external_id': '1', 'value_type': value_type, 'unit': '' diff --git a/rdmo/projects/validators.py b/rdmo/projects/validators.py index 548c00bf0d..da0ace2423 100644 --- a/rdmo/projects/validators.py +++ b/rdmo/projects/validators.py @@ -1,13 +1,24 @@ -from datetime import timedelta +from datetime import datetime, timedelta from django.conf import settings -from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist +from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist, ValidationError +from django.core.validators import EmailValidator, RegexValidator, URLValidator from django.utils.dateparse import parse_datetime from django.utils.translation import gettext_lazy as _ from rest_framework import serializers -from rdmo.core.constants import VALUE_TYPE_FILE +from rdmo.core.constants import ( + VALUE_TYPE_BOOLEAN, + VALUE_TYPE_DATE, + VALUE_TYPE_DATETIME, + VALUE_TYPE_EMAIL, + VALUE_TYPE_FILE, + VALUE_TYPE_FLOAT, + VALUE_TYPE_INTEGER, + VALUE_TYPE_PHONE, + VALUE_TYPE_URL, +) from rdmo.core.utils import human2bytes @@ -52,6 +63,7 @@ def __call__(self, data, serializer): ' was found.')] }) + class ValueQuotaValidator: requires_context = True @@ -63,3 +75,61 @@ def __call__(self, data, serializer): raise serializers.ValidationError({ 'quota': [_('The file quota for this project has been reached.')] }) + + +class ValueTypeValidator: + + def __call__(self, data): + text = data.get('text') + value_type = data.get('value_type') + + try: + self.validate(text, value_type) + except ValidationError as e: + raise serializers.ValidationError({ + 'text': [e.message] + }) from e + + def validate(self, text, value_type): + if text and settings.PROJECT_VALUES_VALIDATION: + if value_type == VALUE_TYPE_URL and settings.PROJECT_VALUES_VALIDATION_URL: + URLValidator()(text) + + elif value_type == VALUE_TYPE_INTEGER and settings.PROJECT_VALUES_VALIDATION_INTEGER: + RegexValidator( + settings.PROJECT_VALUES_VALIDATION_INTEGER_REGEX, + settings.PROJECT_VALUES_VALIDATION_INTEGER_MESSAGE + )(text) + + elif value_type == VALUE_TYPE_FLOAT and settings.PROJECT_VALUES_VALIDATION_FLOAT: + RegexValidator( + settings.PROJECT_VALUES_VALIDATION_FLOAT_REGEX, + settings.PROJECT_VALUES_VALIDATION_FLOAT_MESSAGE + )(text) + + elif value_type == VALUE_TYPE_BOOLEAN and settings.PROJECT_VALUES_VALIDATION_BOOLEAN: + RegexValidator( + settings.PROJECT_VALUES_VALIDATION_BOOLEAN_REGEX, + settings.PROJECT_VALUES_VALIDATION_BOOLEAN_MESSAGE + )(text) + + elif value_type == VALUE_TYPE_DATE and settings.PROJECT_VALUES_VALIDATION_DATE: + RegexValidator( + settings.PROJECT_VALUES_VALIDATION_DATE_REGEX, + settings.PROJECT_VALUES_VALIDATION_DATE_MESSAGE + )(text) + + elif value_type == VALUE_TYPE_DATETIME and settings.PROJECT_VALUES_VALIDATION_DATETIME: + try: + datetime.fromisoformat(text) + except ValueError as e: + raise ValidationError(_('Enter a valid datetime.')) from e + + elif value_type == VALUE_TYPE_EMAIL and settings.PROJECT_VALUES_VALIDATION_EMAIL: + EmailValidator()(text) + + elif value_type == VALUE_TYPE_PHONE and settings.PROJECT_VALUES_VALIDATION_PHONE: + RegexValidator( + settings.PROJECT_VALUES_VALIDATION_PHONE_REGEX, + settings.PROJECT_VALUES_VALIDATION_PHONE_MESSAGE + )(text) diff --git a/rdmo/questions/migrations/0096_alter_question_value_type.py b/rdmo/questions/migrations/0096_alter_question_value_type.py new file mode 100644 index 0000000000..2a6f2c0ec3 --- /dev/null +++ b/rdmo/questions/migrations/0096_alter_question_value_type.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.8 on 2024-07-18 14:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('questions', '0095_page_short_title'), + ] + + operations = [ + migrations.AlterField( + model_name='question', + name='value_type', + field=models.CharField(choices=[('text', 'Text'), ('url', 'URL'), ('integer', 'Integer'), ('float', 'Float'), ('boolean', 'Boolean'), ('date', 'Date'), ('datetime', 'Datetime'), ('email', 'E-mail'), ('phone', 'Phone'), ('option', 'Option'), ('file', 'File')], help_text='Type of value for this question.', max_length=8, verbose_name='Value type'), + ), + ] diff --git a/rdmo/questions/tests/test_managers.py b/rdmo/questions/tests/test_managers.py index f6238cacf5..8a16ccab77 100644 --- a/rdmo/questions/tests/test_managers.py +++ b/rdmo/questions/tests/test_managers.py @@ -4,4 +4,4 @@ def test_questions_filter_by_catalog(db): catalog = Catalog.objects.prefetch_elements().first() questions = Question.objects.filter_by_catalog(catalog) - assert questions.count() == 89 + assert questions.count() == 97 diff --git a/testing/config/settings/base.py b/testing/config/settings/base.py index 0361001712..52cdc8f2aa 100644 --- a/testing/config/settings/base.py +++ b/testing/config/settings/base.py @@ -87,3 +87,5 @@ ] PROJECT_IMPORTS_LIST = ['url'] + +PROJECT_VALUES_VALIDATION = True diff --git a/testing/export/project.html b/testing/export/project.html index dc61738759..1da6e11571 100644 --- a/testing/export/project.html +++ b/testing/export/project.html @@ -635,5 +635,22 @@
URL
+Integer
+Float
+Bool
+Date
+Datetime
+Phone