Merge pull request #679 from rdmorganiser/refactor_progressbar

Refactor progress bar and overview

.eslintrc.js

@@ -44,5 +44,5 @@ module.exports = {
     'react': {
       'version': 'detect'
     }
-  }
+  },
 }

rdmo/core/static/core/js/core.js

@@ -16,6 +16,13 @@ angular.module('core', ['ngResource'])
         method: 'PUT',
         params: {}
     };
+    $resourceProvider.defaults.actions.postAction = {
+        method: 'POST',
+        params: {
+            id: '@id',
+            detail_action: '@detail_action'
+        }
+    };
 }])
 
 .filter('capitalize', function() {

rdmo/projects/managers.py

@@ -1,5 +1,6 @@
 from django.conf import settings
 from django.db import models
+from django.db.models import Q
 
 from mptt.models import TreeManager
 from mptt.querysets import TreeQuerySet
@@ -141,6 +142,12 @@ def filter_user(self, user):
         else:
             return self.none()
 
+    def exclude_empty(self):
+        return self.exclude((Q(text='') | Q(text=None)) & Q(option=None) & (Q(file='') | Q(file=None)))
+
+    def distinct_list(self):
+        return self.order_by('attribute').values_list('attribute', 'set_prefix', 'set_index').distinct()
+
 
 class ProjectManager(CurrentSiteManagerMixin, TreeManager):
 

rdmo/projects/migrations/0059_project_progress.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.2.19 on 2023-08-24 09:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('projects', '0058_meta'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='progress_count',
+            field=models.IntegerField(help_text='The number of values for the progress bar.', null=True, verbose_name='Progress count'),
+        ),
+        migrations.AddField(
+            model_name='project',
+            name='progress_total',
+            field=models.IntegerField(help_text='The total number of expected values for the progress bar.', null=True, verbose_name='Progress total'),
+        ),
+    ]

rdmo/projects/models/project.py

@@ -2,7 +2,6 @@
 from django.contrib.sites.models import Site
 from django.core.exceptions import ValidationError
 from django.db import models
-from django.db.models import Exists, OuterRef
 from django.db.models.signals import pre_delete
 from django.dispatch import receiver
 from django.urls import reverse
@@ -12,8 +11,7 @@
 from mptt.models import MPTTModel, TreeForeignKey
 
 from rdmo.core.models import Model
-from rdmo.domain.models import Attribute
-from rdmo.questions.models import Catalog, Question
+from rdmo.questions.models import Catalog
 from rdmo.tasks.models import Task
 from rdmo.views.models import View
 
@@ -65,6 +63,16 @@ class Project(MPTTModel, Model):
         verbose_name=_('Views'),
         help_text=_('The views that will be used for this project.')
     )
+    progress_total = models.IntegerField(
+        null=True,
+        verbose_name=_('Progress total'),
+        help_text=_('The total number of expected values for the progress bar.')
+    )
+    progress_count = models.IntegerField(
+        null=True,
+        verbose_name=_('Progress count'),
+        help_text=_('The number of values for the progress bar.')
+    )
 
     class Meta:
         ordering = ('tree_id', 'level', 'title')
@@ -86,36 +94,6 @@ def clean(self):
                 'parent': [_('A project may not be moved to be a child of itself or one of its descendants.')]
             })
 
-    @property
-    def progress(self):
-        # create a queryset for the attributes of the catalog for this project
-        # the subquery is used to query only attributes which have a question in the catalog, which is not optional
-        questions = Question.objects.filter_by_catalog(self.catalog) \
-                                    .filter(attribute_id=OuterRef('pk')).exclude(is_optional=True)
-        attributes = Attribute.objects.annotate(active=Exists(questions)).filter(active=True).distinct()
-
-        # query the total number of attributes from the qs above
-        total = attributes.count()
-
-        # query all current values with attributes from the qs above, but where the text, option, or file field is set,
-        # and count only one value per attribute
-        values = self.values.filter(snapshot=None) \
-                            .filter(attribute__in=attributes) \
-                            .exclude((models.Q(text='') | models.Q(text=None)) & models.Q(option=None) &
-                                     (models.Q(file='') | models.Q(file=None))) \
-                            .distinct().values('attribute').count()
-
-        try:
-            ratio = values / total
-        except ZeroDivisionError:
-            ratio = 0
-
-        return {
-            'total': total,
-            'values': values,
-            'ratio': ratio
-        }
-
     @property
     def catalog_uri(self):
         if self.catalog is not None:

rdmo/projects/permissions.py

@@ -62,3 +62,14 @@ class HasProjectPagePermission(HasProjectPermission):
 
     def get_required_object_permissions(self, method, model_cls):
         return ('projects.view_page_object', )
+
+
+class HasProjectProgressPermission(HasProjectPermission):
+
+    def get_required_object_permissions(self, method, model_cls):
+        if method == 'GET':
+            return ('projects.view_project_object', )
+        elif method == 'POST':
+            return ('projects.change_project_progress_object', )
+        else:
+            raise RuntimeError('Unsupported method for HasProjectProgressPermission')

rdmo/projects/progress.py

@@ -0,0 +1,148 @@
+from collections import defaultdict
+
+from django.db.models import Exists, OuterRef, Q
+
+from rdmo.conditions.models import Condition
+from rdmo.questions.models import Page, Question, QuestionSet
+
+
+def resolve_conditions(project, values):
+    # get all conditions for this catalog
+    pages_conditions_subquery = Page.objects.filter_by_catalog(project.catalog) \
+                                            .filter(conditions=OuterRef('pk'))
+    questionsets_conditions_subquery = QuestionSet.objects.filter_by_catalog(project.catalog) \
+                                                          .filter(conditions=OuterRef('pk'))
+    questions_conditions_subquery = Question.objects.filter_by_catalog(project.catalog) \
+                                                    .filter(conditions=OuterRef('pk'))
+
+    catalog_conditions = Condition.objects.annotate(has_page=Exists(pages_conditions_subquery)) \
+                                          .annotate(has_questionset=Exists(questionsets_conditions_subquery)) \
+                                          .annotate(has_question=Exists(questions_conditions_subquery)) \
+                                          .filter(Q(has_page=True) | Q(has_questionset=True) | Q(has_question=True)) \
+                                          .distinct().select_related('source', 'target_option')
+
+    # evaluate conditions
+    conditions = set()
+    for condition in catalog_conditions:
+        if condition.resolve(values):
+            conditions.add(condition.id)
+
+    # return all true conditions for this project
+    return conditions
+
+
+def compute_navigation(section, project, snapshot=None):
+    # get all values for this project and snapshot
+    values = project.values.filter(snapshot=snapshot).select_related('attribute', 'option')
+
+    # get true conditions
+    conditions = resolve_conditions(project, values)
+
+    # compute sets from values (including empty values)
+    sets = defaultdict(lambda: defaultdict(list))
+    for attribute, set_prefix, set_index in values.distinct_list():
+        sets[attribute][set_prefix].append(set_index)
+
+    # query distinct, non empty set values
+    values_list = values.exclude_empty().distinct_list()
+
+    navigation = []
+    for catalog_section in project.catalog.elements:
+        navigation_section = {
+            'id': catalog_section.id,
+            'uri': catalog_section.uri,
+            'title': catalog_section.title,
+            'first': catalog_section.elements[0].id if section.elements else None
+        }
+        if catalog_section.id == section.id:
+            navigation_section['pages'] = []
+            for page in catalog_section.elements:
+                pages_conditions = {page.id for page in page.conditions.all()}
+                show = bool(not pages_conditions or pages_conditions.intersection(conditions))
+
+                # count the total number of questions, taking sets and conditions into account
+                counts = count_questions(page, sets, conditions)
+
+                # filter the values_list for the attributes, and compute the total sum of counts
+                count = len(tuple(filter(lambda value: value[0] in counts.keys(), values_list)))
+                total = sum(counts.values())
+
+                navigation_section['pages'].append({
+                    'id': page.id,
+                    'uri': page.uri,
+                    'title': page.title,
+                    'show': show,
+                    'count': count,
+                    'total': total
+                })
+
+        navigation.append(navigation_section)
+
+    return navigation
+
+
+def compute_progress(project, snapshot=None):
+    # get all values for this project and snapshot
+    values = project.values.filter(snapshot=snapshot).select_related('attribute', 'option')
+
+    # get true conditions
+    conditions = resolve_conditions(project, values)
+
+    # compute sets from values (including empty values)
+    sets = defaultdict(lambda: defaultdict(list))
+    for attribute, set_prefix, set_index in values.distinct_list():
+        sets[attribute][set_prefix].append(set_index)
+
+    # query distinct, non empty set values
+    values_list = values.exclude_empty().distinct_list()
+
+
+    # count the total number of questions, taking sets and conditions into account
+    counts = count_questions(project.catalog, sets, conditions)
+
+    # filter the values_list for the attributes, and compute the total sum of counts
+    count = len(tuple(filter(lambda value: value[0] in counts.keys(), values_list)))
+    total = sum(counts.values())
+
+    return count, total
+
+
+def count_questions(element, sets, conditions):
+    counts = defaultdict(int)
+
+    # obtain the maximum number of set-distinct values the questions in this element
+    # this number is how often each question is displayed and we will use this number
+    # to determine how often a question needs to be counted
+    if isinstance(element, (Page, QuestionSet)) and element.is_collection:
+        set_count = 0
+
+        if element.attribute is not None:
+            child_count = sum(len(set_indexes) for set_indexes in sets[element.attribute.id].values())
+            set_count = max(set_count, child_count)
+
+        for child in element.elements:
+            if isinstance(child, Question):
+                child_count = sum(len(set_indexes) for set_indexes in sets[child.attribute.id].values())
+                set_count = max(set_count, child_count)
+    else:
+        set_count = 1
+
+    # loop over all children of this element
+    for child in element.elements:
+        # look for the elements conditions
+        if isinstance(child, (Page, QuestionSet, Question)):
+            child_conditions = {condition.id for condition in child.conditions.all()}
+        else:
+            child_conditions = []
+
+        if not child_conditions or child_conditions.intersection(conditions):
+            if isinstance(child, Question):
+                # for questions add the set_count to the counts dict
+                # use the max function, since the same attribute could apear twice in the tree
+                if child.attribute is not None:
+                    counts[child.attribute.id] = max(counts[child.attribute.id], set_count)
+            else:
+                # for everthing else, call this function recursively
+                counts.update(count_questions(child, sets, conditions))
+
+    return counts

rdmo/projects/rules.py

@@ -57,6 +57,7 @@ def is_site_manager_for_current_site(user, request):
 
 rules.add_perm('projects.view_project_object', is_project_member | is_site_manager)
 rules.add_perm('projects.change_project_object', is_project_manager | is_project_owner | is_site_manager)
+rules.add_perm('projects.change_project_progress_object', is_project_author | is_project_manager | is_project_owner | is_site_manager)  # noqa: E501
 rules.add_perm('projects.delete_project_object', is_project_owner | is_site_manager)
 rules.add_perm('projects.leave_project_object', is_current_project_member)
 rules.add_perm('projects.export_project_object', is_project_owner | is_project_manager | is_site_manager)

rdmo/projects/serializers/v1/overview.py

@@ -1,51 +1,18 @@
 from rest_framework import serializers
 
 from rdmo.projects.models import Project
-from rdmo.questions.models import Catalog, Page, Section
-
-
-class PageSerializer(serializers.ModelSerializer):
-
-    class Meta:
-        model = Page
-        fields = (
-            'id',
-            'title',
-            'has_conditions'
-        )
-
-
-class SectionSerializer(serializers.ModelSerializer):
-
-    pages = serializers.SerializerMethodField()
-
-    class Meta:
-        model = Section
-        fields = (
-            'id',
-            'title',
-            'pages'
-        )
-
-    def get_pages(self, obj):
-        return PageSerializer(obj.elements, many=True, read_only=True).data
+from rdmo.questions.models import Catalog
 
 
 class CatalogSerializer(serializers.ModelSerializer):
 
-    sections = serializers.SerializerMethodField()
-
     class Meta:
         model = Catalog
         fields = (
             'id',
-            'title',
-            'sections'
+            'title'
         )
 
-    def get_sections(self, obj):
-        return SectionSerializer(obj.elements, many=True, read_only=True).data
-
 
 class ProjectOverviewSerializer(serializers.ModelSerializer):
 

rdmo/projects/serializers/v1/page.py

@@ -181,6 +181,7 @@ def get_section(self, obj):
         return {
            'id': section.id,
            'title': section.title,
+           'first': section.elements[0].id if section.elements else None
         } if section else {}
 
     def get_prev_page(self, obj):

rdmo/projects/static/projects/css/project_questions.scss

@@ -302,3 +302,9 @@
 .project-progress {
     margin-bottom: 10px;
 }
+.project-progress-count {
+    position: absolute;
+    left: 0;
+    right: 0;
+    text-align: center;
+}