From a901f7bcf603dec781988b5342ed54146a6937f3 Mon Sep 17 00:00:00 2001 From: David Wallace Date: Wed, 7 Aug 2024 09:50:56 +0200 Subject: [PATCH] fix(progress): check for resolved conditions in dataset count Signed-off-by: David Wallace --- rdmo/projects/progress.py | 74 +++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/rdmo/projects/progress.py b/rdmo/projects/progress.py index 39c4cd8a06..bdaac97847 100644 --- a/rdmo/projects/progress.py +++ b/rdmo/projects/progress.py @@ -8,13 +8,13 @@ from rdmo.questions.models import Page, Question, QuestionSet -def resolve_conditions(project, values): +def get_conditions_from_catalog(catalog): # get all conditions for this catalog - pages_conditions_subquery = Page.objects.filter_by_catalog(project.catalog) \ + pages_conditions_subquery = Page.objects.filter_by_catalog(catalog) \ .filter(conditions=OuterRef('pk')) - questionsets_conditions_subquery = QuestionSet.objects.filter_by_catalog(project.catalog) \ + questionsets_conditions_subquery = QuestionSet.objects.filter_by_catalog(catalog) \ .filter(conditions=OuterRef('pk')) - questions_conditions_subquery = Question.objects.filter_by_catalog(project.catalog) \ + questions_conditions_subquery = Question.objects.filter_by_catalog(catalog) \ .filter(conditions=OuterRef('pk')) catalog_conditions = Condition.objects.annotate(has_page=Exists(pages_conditions_subquery)) \ @@ -22,28 +22,40 @@ def resolve_conditions(project, values): .annotate(has_question=Exists(questions_conditions_subquery)) \ .filter(Q(has_page=True) | Q(has_questionset=True) | Q(has_question=True)) \ .distinct().select_related('source', 'target_option') + return catalog_conditions - # evaluate conditions - conditions = set() - for condition in catalog_conditions: - if condition.resolve(values): - conditions.add(condition.id) +def evaluate_conditions(conditions, values): + # evaluate conditions + condition_true_ids = defaultdict(list) + for condition in conditions: + values_that_resolve_condition = [] + for value in values: + if condition.resolve([value]): + values_that_resolve_condition.append(value.id) + condition_true_ids[condition.id] = values.filter(id__in=values_that_resolve_condition) # return all true conditions for this project - return conditions + return condition_true_ids + + +def compute_sets_from_values(values): + # compute sets from values (including empty values) + sets = defaultdict(lambda: defaultdict(list)) + for attribute, set_prefix, set_index in values.distinct_list(): + sets[attribute][set_prefix].append(set_index) + return sets def compute_navigation(section, project, snapshot=None): # get all values for this project and snapshot values = project.values.filter(snapshot=snapshot).select_related('attribute', 'option') + # get conditions from project.catalog + conditions = get_conditions_from_catalog(project.catalog) # get true conditions - conditions = resolve_conditions(project, values) - + condition_true_ids = evaluate_conditions(conditions, values) # compute sets from values (including empty values) - sets = defaultdict(lambda: defaultdict(list)) - for attribute, set_prefix, set_index in values.distinct_list(): - sets[attribute][set_prefix].append(set_index) + sets = compute_sets_from_values(values) # query distinct, non empty set values values_list = values.exclude_empty().distinct_list() @@ -63,10 +75,10 @@ def compute_navigation(section, project, snapshot=None): for page in catalog_section.elements: pages_conditions = {page.id for page in page.conditions.all()} - show = bool(not pages_conditions or pages_conditions.intersection(conditions)) + show = bool(not pages_conditions or pages_conditions.intersection(condition_true_ids)) # count the total number of questions, taking sets and conditions into account - counts = count_questions(page, sets, conditions) + counts = count_questions(page, sets, condition_true_ids) # filter the values_list for the attributes, and compute the total sum of counts count = len(tuple(filter(lambda value: value[0] in counts.keys(), values_list))) @@ -94,20 +106,18 @@ def compute_progress(project, snapshot=None): # get all values for this project and snapshot values = project.values.filter(snapshot=snapshot).select_related('attribute', 'option') + # get conditions for project and catalog + conditions = get_conditions_from_catalog(project.catalog) # get true conditions - conditions = resolve_conditions(project, values) - + condition_true_ids = evaluate_conditions(conditions, values) # compute sets from values (including empty values) - sets = defaultdict(lambda: defaultdict(list)) - for attribute, set_prefix, set_index in values.distinct_list(): - sets[attribute][set_prefix].append(set_index) + sets = compute_sets_from_values(values) # query distinct, non empty set values values_list = values.exclude_empty().distinct_list() - # count the total number of questions, taking sets and conditions into account - counts = count_questions(project.catalog, sets, conditions) + counts = count_questions(project.catalog, sets, condition_true_ids) # filter the values_list for the attributes, and compute the total sum of counts count = len(tuple(filter(lambda value: value[0] in counts.keys(), values_list))) @@ -141,6 +151,7 @@ def count_questions(element, sets, conditions): set_count = len(counted_sets) else: + counted_sets = set() set_count = 1 # loop over all children of this element @@ -149,9 +160,9 @@ def count_questions(element, sets, conditions): if isinstance(child, (Page, QuestionSet, Question)): child_conditions = {condition.id for condition in child.conditions.all()} else: - child_conditions = [] + child_conditions = set() - if not child_conditions or child_conditions.intersection(conditions): + if child_conditions.intersection(conditions): if isinstance(child, Question): # for regular questions add the set_count to the counts dict, since the # question should be answered in every set @@ -159,7 +170,16 @@ def count_questions(element, sets, conditions): # only answered questions count for the progress/navigation # use the max function, since the same attribute could appear twice in the tree if child.attribute is not None: - if child.is_optional: + resolved_condition_sets = set() + condition_intersection = list(child_conditions.intersection(conditions)) + for child_condition in condition_intersection: + triggered_values_sets = compute_sets_from_values(conditions[child_condition]) + resolving_sets = {b for i in triggered_values_sets.values() for a in i.values() for b in a} + resolved_condition_sets.update(resolving_sets) + if condition_intersection: + set_count = len(counted_sets - resolved_condition_sets) + if (child.is_optional): + # or (child.is_collection and set_count > 1)): child_count = sum(len(set_indexes) for set_indexes in sets[child.attribute.id].values()) counts[child.attribute.id] = max(counts[child.attribute.id], child_count) else: