Merge pull request rdmorganiser#1066 from rdmorganiser/dev-2.2.0-vali…

…dators

Dev 2.2.0 validators

rdmo/core/constants.py

@@ -5,6 +5,7 @@
 VALUE_TYPE_INTEGER = 'integer'
 VALUE_TYPE_FLOAT = 'float'
 VALUE_TYPE_BOOLEAN = 'boolean'
+VALUE_TYPE_DATE = 'date'
 VALUE_TYPE_DATETIME = 'datetime'
 VALUE_TYPE_OPTIONS = 'option'
 VALUE_TYPE_EMAIL = 'email'
@@ -16,6 +17,7 @@
     (VALUE_TYPE_INTEGER, _('Integer')),
     (VALUE_TYPE_FLOAT, _('Float')),
     (VALUE_TYPE_BOOLEAN, _('Boolean')),
+    (VALUE_TYPE_DATE, _('Date')),
     (VALUE_TYPE_DATETIME, _('Datetime')),
     (VALUE_TYPE_EMAIL, _('E-mail')),
     (VALUE_TYPE_PHONE, _('Phone')),

rdmo/core/settings.py

@@ -1,3 +1,5 @@
+import re
+
 from django.utils.translation import gettext_lazy as _
 
 SITE_ID = 1
@@ -313,6 +315,56 @@
 
 OPTIONSET_PROVIDERS = []
 
+PROJECT_VALUES_VALIDATION = False
+
+PROJECT_VALUES_VALIDATION_URL = True
+
+PROJECT_VALUES_VALIDATION_INTEGER = True
+PROJECT_VALUES_VALIDATION_INTEGER_MESSAGE = _('Enter a valid integer.')
+PROJECT_VALUES_VALIDATION_INTEGER_REGEX = re.compile(r'^[+-]?\d+$')
+
+PROJECT_VALUES_VALIDATION_FLOAT = True
+PROJECT_VALUES_VALIDATION_FLOAT_MESSAGE = _('Enter a valid float.')
+PROJECT_VALUES_VALIDATION_FLOAT_REGEX = re.compile(r'''
+    ^[+-]?            # Optional sign
+    (
+        \d+           # Digits before the decimal or thousands separator
+        (,\d{3})*     # Optional groups of exactly three digits preceded by a comma (thousands separator)
+        (\.\d+)?      # Optional decimal part, a dot followed by one or more digits
+        |             # OR
+        \d+           # Digits before the decimal or thousands separator
+        (\.\d{3})*    # Optional groups of exactly three digits preceded by a dot (thousands separator)
+        (,\d+)?       # Optional decimal part, a comma followed by one or more digits
+    )
+    ([eE][+-]?\d+)?$  # Optional exponent part
+''', re.VERBOSE)
+
+PROJECT_VALUES_VALIDATION_BOOLEAN = True
+PROJECT_VALUES_VALIDATION_BOOLEAN_MESSAGE = _('Enter a valid boolean (e.g. 0, 1).')
+PROJECT_VALUES_VALIDATION_BOOLEAN_REGEX = r'(?i)^(0|1|f|t|false|true)$'
+
+PROJECT_VALUES_VALIDATION_DATE = True
+PROJECT_VALUES_VALIDATION_DATE_MESSAGE = _('Enter a valid date (e.g. "02.03.2024", "03/02/2024", "2024-02-03").')
+PROJECT_VALUES_VALIDATION_DATE_REGEX = re.compile(r'''
+    ^(
+        \d{1,2}\.\s*\d{1,2}\.\s*\d{2,4}  # Format dd.mm.yyyy
+        | \d{1,2}/\d{1,2}/\d{4}          # Format mm/dd/yyyy
+        | \d{4}-\d{2}-\d{2}              # Format yyyy-mm-dd
+    )$
+''', re.VERBOSE)
+
+PROJECT_VALUES_VALIDATION_DATETIME = True
+
+PROJECT_VALUES_VALIDATION_EMAIL = True
+
+PROJECT_VALUES_VALIDATION_PHONE = True
+PROJECT_VALUES_VALIDATION_PHONE_MESSAGE = _('Enter a valid phone number (e.g. "123456" or "+49 (0) 30 123456").')
+PROJECT_VALUES_VALIDATION_PHONE_REGEX = re.compile(r'''
+    ^([+]\d{2,3}\s)?  # Optional country code
+    (\(\d+\)\s)?      # Optional area code in parentheses
+    [\d\s]*$          # Main number with spaces
+''', re.VERBOSE)
+
 QUESTIONS_WIDGETS = [
     ('text', _('Text'), 'rdmo.projects.widgets.TextWidget'),
     ('textarea', _('Textarea'), 'rdmo.projects.widgets.TextareaWidget'),

rdmo/management/tests/test_import_questions.py

@@ -283,6 +283,15 @@ def test_update_legacy_questions(db, settings):
 
     # check that all elements ended up in the catalog
     catalog = Catalog.objects.prefetch_elements().first()
-    descendant_uris = {element.uri for element in catalog.descendants}
+    descendant_uris = {
+        element.uri for element in catalog.descendants if any(element.uri.startswith(uri) for uri in [
+            'http://example.com/terms/questions/catalog/individual',
+            'http://example.com/terms/questions/catalog/set',
+            'http://example.com/terms/questions/catalog/collections',
+            'http://example.com/terms/questions/catalog/conditions',
+            'http://example.com/terms/questions/catalog/options',
+            'http://example.com/terms/questions/catalog/blocks'
+        ])
+    }
     element_uris = {element['uri'] for _uri, element in elements.items() if element['uri'] != catalog.uri}
     assert descendant_uris == element_uris

rdmo/projects/migrations/0061_alter_value_value_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.8 on 2024-07-18 14:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0060_alter_issue_options'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='value',
+            name='value_type',
+            field=models.CharField(choices=[('text', 'Text'), ('url', 'URL'), ('integer', 'Integer'), ('float', 'Float'), ('boolean', 'Boolean'), ('date', 'Date'), ('datetime', 'Datetime'), ('email', 'E-mail'), ('phone', 'Phone'), ('option', 'Option'), ('file', 'File')], default='text', help_text='Type of this value.', max_length=8, verbose_name='Value type'),
+        ),
+    ]

rdmo/projects/serializers/v1/__init__.py

@@ -9,7 +9,7 @@
 from rdmo.services.validators import ProviderValidator
 
 from ...models import Integration, IntegrationOption, Invite, Issue, IssueResource, Membership, Project, Snapshot, Value
-from ...validators import ValueConflictValidator, ValueQuotaValidator
+from ...validators import ValueConflictValidator, ValueQuotaValidator, ValueTypeValidator
 
 
 class UserSerializer(serializers.ModelSerializer):
@@ -267,7 +267,8 @@ class Meta:
         )
         validators = (
             ValueConflictValidator(),
-            ValueQuotaValidator()
+            ValueQuotaValidator(),
+            ValueTypeValidator()
         )
 
 

rdmo/projects/static/projects/js/project_questions/services.js

@@ -885,7 +885,11 @@ angular.module('project_questions')
                     service.error = response;
                 } else if (response.status == 400) {
                     service.error = true;
-                    value.errors = Object.keys(response.data);
+                    if (angular.isDefined(response.data.text)) {
+                        value.errors = response.data.text
+                    } else {
+                        value.errors = Object.keys(response.data);
+                    }
                 } else if (response.status == 404) {
                     service.error = true;
                     value.errors = ['not_found']

rdmo/projects/templates/projects/project_questions_value_errors.html

@@ -17,5 +17,8 @@
             <span ng-show="error == 'quota'">
                 {% trans 'You reached the file quota for this project.' %}
             </span>
+            <span ng-hide="error == 'conflict' || error == 'not_found' || error == 'quota'">
+                {$ error $}
+            </span>
         </li>
     </ul>

rdmo/projects/tests/test_progress.py

@@ -6,8 +6,8 @@
 projects = [1, 11]
 
 results_map = {
-    1: (58, 81),
-    11: (0, 29)
+    1: (58, 88),
+    11: (0, 36)
 }
 
 

rdmo/projects/tests/test_validator_value_type.py

@@ -0,0 +1,113 @@
+import pytest
+
+from rest_framework.exceptions import ValidationError as RestFameworkValidationError
+
+from ..validators import ValueTypeValidator
+
+data = (
+    ('url', 'https://example.com'),
+    ('url', 'http://example.com'),
+    ('integer', '1'),
+    ('integer', '-1'),
+    ('integer', '+1'),
+    ('integer', '12345'),
+    ('float', '1'),
+    ('float', '1.0'),
+    ('float', '+1.0'),
+    ('float', '-1.0'),
+    ('float', '1,000,000.12345'),
+    ('float', '1,0'),
+    ('float', '1.000.000,12345'),
+    ('float', '1.0e20'),
+    ('float', '1.0E20'),
+    ('float', '1.0e-20'),
+    ('float', '1.0e+20'),
+    ('boolean', '0'),
+    ('boolean', '1'),
+    ('boolean', 'f'),
+    ('boolean', 't'),
+    ('boolean', 'TrUe'),   # spellchecker:disable-line
+    ('boolean', 'FaLsE'),  # spellchecker:disable-line
+    ('boolean', 'true'),
+    ('boolean', 'false'),
+    ('date', '01.02.2024'),
+    ('date', '1.2.2024'),
+    ('date', '13.01.1337'),
+    ('date', '2/1/2024'),
+    ('date', '2024-01-02'),
+    ('date', '1. 2. 2024'),
+    ('datetime', '2024-01-02'),
+    ('datetime', '2024-01-02T10:00'),
+    ('datetime', '2024-01-02T10:00:00'),
+    ('datetime', '2024-01-02T10:00:00.123'),
+    ('datetime', '2024-01-02T10:00:00+02:00'),
+    ('email', '[email protected]'),
+    ('email', '[email protected]'),
+    ('email', '[email protected]'),
+    ('email', '[email protected]'),
+    ('email', '[email protected]'),
+    ('phone', '123456'),
+    ('phone', '123 456'),
+    ('phone', '362 123456'),
+    ('phone', '(362) 123456'),
+    ('phone', '+49 (0) 362123456'),
+    ('phone', '+49 (0) 362 123456'),
+)
+data_error = (
+    ('url', 'wrong'),
+    ('url', 'example.com'),
+    ('integer', 'wrong'),
+    ('integer', '1.0'),
+    ('integer', '1b'),
+    ('float', 'wrong'),
+    ('float', '1,0000.12456'),
+    ('float', '1.0000,12456'),
+    ('float', '1.0a20'),
+    ('boolean', 'wrong'),
+    ('boolean', '2'),
+    ('boolean', '-1'),
+    ('boolean', 'tr'),
+    ('boolean', 'truee'),
+    ('boolean', 'falze'),
+    ('date', 'wrong'),
+    ('date', '001.02.2024'),
+    ('date', '01.02.20240'),
+    ('date', '1,2.2024'),
+    ('date', '2-1-2024'),
+    ('date', '2024-001-02'),
+    ('date', '20240-01-02'),
+    ('date', '2024-1-2'),
+    ('datetime', 'wrong'),
+    ('datetime', '2024-13-02'),
+    ('datetime', '2024-13-02Y10:00:00'),
+    ('datetime', '2024-01-02T10:00:00ZZ+02:00'),
+    ('datetime', '2024-01-02T25:00'),
+    ('datetime', '2024-01-02T10:60:00'),
+    ('email', 'wrong'),
+    ('email', 'example.com'),
+    ('email', 'ü[email protected]'),
+    ('email', 'user@[email protected]'),
+    ('email', 'user@com'),
+    ('email', '[email protected]'),
+    ('phone', 'wrong'),
+    ('phone', '123456a'),
+    ('phone', '123 456 a'),
+    ('phone', '362s 123456'),
+    ('phone', '(3 62) 123456'),
+    ('phone', '-49 (0) 362123456'),
+    ('phone', '49 (0) 362 123456'),
+    ('phone', '1234 (0) 123456'),
+)
+
+
+@pytest.mark.parametrize('value_type,text', data)
+def test_serializer(db, value_type, text):
+    validator = ValueTypeValidator()
+    validator({'value_type': value_type, 'text': text})
+
+
+@pytest.mark.parametrize('value_type,text', data_error)
+def test_serializer_error(db, value_type, text):
+    validator = ValueTypeValidator()
+    with pytest.raises(RestFameworkValidationError):
+        validator({'value_type': value_type, 'text': text})

rdmo/projects/tests/test_viewset_project_progress.py

@@ -119,7 +119,7 @@ def test_progress_post_unchanged(db, client):
 
     project = Project.objects.get(id=1)
     project.progress_count = progress_count = 58  # the progress in the fixture is not up-to-date
-    project.progress_total = progress_total = 81
+    project.progress_total = progress_total = 88
     project.save()
     project.refresh_from_db()
     project_updated = project.updated

rdmo/projects/tests/test_viewset_project_value.py

@@ -5,7 +5,7 @@
 from django.conf import settings
 from django.urls import reverse
 
-from rdmo.core.constants import VALUE_TYPE_CHOICES, VALUE_TYPE_FILE, VALUE_TYPE_TEXT
+from rdmo.core.constants import VALUE_TYPE_FILE, VALUE_TYPE_TEXT
 
 from ..models import Value
 
@@ -56,6 +56,16 @@
 ]
 set_questionsets = [42, 43]
 
+value_texts = (
+    ('text', 'Lorem ipsum'),
+    ('url', 'https://lorem.ipsum'),
+    ('integer', '1337'),
+    ('float', '13.37'),
+    ('boolean', '1'),
+    ('datetime', '1337-01-13T13:37+13:37'),
+    ('email', '[email protected]'),
+    ('phone', '+49 (0) 1337 12345678')
+)
 
 @pytest.mark.parametrize('username,password', users)
 @pytest.mark.parametrize('project_id', projects)
@@ -101,23 +111,23 @@ def test_detail(db, client, username, password, project_id, value_id):
 
 @pytest.mark.parametrize('username,password', users)
 @pytest.mark.parametrize('project_id', projects)
-@pytest.mark.parametrize('value_type,value_type_label', VALUE_TYPE_CHOICES)
-def test_create_text(db, client, username, password, project_id, value_type, value_type_label):
+@pytest.mark.parametrize('value_type,value_text', value_texts)
+def test_create_text(db, client, username, password, project_id, value_type, value_text):
     client.login(username=username, password=password)
 
     url = reverse(urlnames['list'], args=[project_id])
     data = {
         'attribute': attribute_id,
         'set_index': 0,
         'collection_index': 0,
-        'text': 'Lorem ipsum',
+        'text': value_text,
         'value_type': value_type,
         'unit': ''
     }
     response = client.post(url, data)
 
     if project_id in add_value_permission_map.get(username, []):
-        assert response.status_code == 201
+        assert response.status_code == 201, response.content
         assert isinstance(response.json(), dict)
         assert response.json().get('id') in Value.objects.filter(project_id=project_id).values_list('id', flat=True)
     elif project_id in view_value_permission_map.get(username, []):
@@ -128,15 +138,16 @@ def test_create_text(db, client, username, password, project_id, value_type, val
 
 @pytest.mark.parametrize('username,password', users)
 @pytest.mark.parametrize('project_id', projects)
-@pytest.mark.parametrize('value_type,value_type_label', VALUE_TYPE_CHOICES)
-def test_create_option(db, client, username, password, project_id, value_type, value_type_label):
+@pytest.mark.parametrize('value_type,value_text', value_texts)
+def test_create_option(db, client, username, password, project_id, value_type, value_text):
     client.login(username=username, password=password)
 
     url = reverse(urlnames['list'], args=[project_id])
     data = {
         'attribute': attribute_id,
         'set_index': 0,
         'collection_index': 0,
+        'text': value_text,
         'option': option_id,
         'value_type': value_type,
         'unit': ''
@@ -155,16 +166,16 @@ def test_create_option(db, client, username, password, project_id, value_type, v
 
 @pytest.mark.parametrize('username,password', users)
 @pytest.mark.parametrize('project_id', projects)
-@pytest.mark.parametrize('value_type,value_type_label', VALUE_TYPE_CHOICES)
-def test_create_external(db, client, username, password, project_id, value_type, value_type_label):
+@pytest.mark.parametrize('value_type,value_text', value_texts)
+def test_create_external(db, client, username, password, project_id, value_type, value_text):
     client.login(username=username, password=password)
 
     url = reverse(urlnames['list'], args=[project_id])
     data = {
         'attribute': attribute_id,
         'set_index': 0,
         'collection_index': 0,
-        'text': 'Lorem ipsum',
+        'text': value_text,
         'external_id': '1',
         'value_type': value_type,
         'unit': ''