feat(qual): new QA verification field DEV-734

src/formpack/schema/fields.py

@@ -1,17 +1,17 @@
 # coding: utf-8
 import math
-from collections import defaultdict, OrderedDict
-from dateutil import parser
+import statistics
+from collections import OrderedDict, defaultdict
 from functools import partial
 from operator import itemgetter
 
-import statistics
+from dateutil import parser
 
-from .datadef import FormDataDef, FormChoice
 from ..constants import UNSPECIFIED_TRANSLATION
 from ..utils import singlemode
 from ..utils.ordered_collection import OrderedDefaultdict
 from ..utils.string import list_to_csv
+from .datadef import FormChoice, FormDataDef
 
 
 class FormField(FormDataDef):
@@ -262,6 +262,8 @@ def from_json_definition(
             'qualText': QualField,
             'transcript': QualTranscriptField,
             'translation': QualTranslationField,
+            'qualVerification': QualVerificationField,
+            'qualSource': QualSourceField,
         }
 
         args = {
@@ -520,6 +522,8 @@ def format(
 
 
 class QualField(TextField):
+    SUPPLEMENTAL_DETAILS_FIELD = '_supplementalDetails'
+
     def _get_label(self, *args, **kwargs):
         source_label = self.source_field._get_label(*args, **kwargs)
         # hard-coded first label because qualitative analysis does not yet
@@ -540,7 +544,7 @@ def get_value_from_entry(self, entry):
 
         try:
             # all responses nested within `qual`
-            responses = entry['_supplementalDetails'][source]['qual']
+            responses = entry[self.SUPPLEMENTAL_DETAILS_FIELD][source]['qual']
         except KeyError:
             return ''
 
@@ -673,6 +677,42 @@ def _get_label(self, *args, **kwargs):
         return f'{source_label} - translation ({self.language})'
 
 
+class QualMetadataField(QualField):
+    @property
+    def value_field(self):
+        raise NotImplementedError()
+
+    def _get_label(self, *args, **kwargs):
+        source_label = self.source_field._get_label(*args, **kwargs)
+        return f'{source_label} - {self.value_field}'
+
+    def get_value_from_entry(self, entry):
+        name_parts = self.name.split('/')
+        # source question path, analysis question uuid, value field
+        assert len(name_parts) >= 3
+        analysis_question_uuid = name_parts[-2]
+        survey_question_path = '/'.join(name_parts[:-2])
+        try:
+            response = entry[self.SUPPLEMENTAL_DETAILS_FIELD][
+                survey_question_path
+            ]['qual'][analysis_question_uuid]
+        except KeyError:
+            return ''
+        return response.get(self.value_field, False)
+
+
+class QualVerificationField(QualMetadataField):
+    @property
+    def value_field(self):
+        return 'verified'
+
+
+class QualSourceField(QualMetadataField):
+    @property
+    def value_field(self):
+        return 'source'
+
+
 class MediaField(TextField):
     def get_labels(self, include_media_url=False, *args, **kwargs):
         label = self._get_label(*args, **kwargs)

src/formpack/utils/dft.py

@@ -0,0 +1,32 @@
+from typing import Any, Callable
+
+from formpack.schema import FormField
+
+
+# Basic recursive depth-first traversal
+def dft_recurse(
+    root: FormField,
+    tree: dict[str, list[FormField]],
+    process_field: Callable[[FormField], Any],
+):
+    seen = set()
+    result = [root]
+    seen.add(root.path)
+    for child in tree[root.path]:
+        dft_recurse_inner(child, tree, process_field, result, seen)
+    return result
+
+
+def dft_recurse_inner(
+    root: FormField,
+    tree: dict[str, list[FormField]],
+    process_field: Callable[[FormField], Any],
+    result: list[Any],
+    seen: set[str],
+):
+    if root.path in seen:
+        return
+    result.append(process_field(root))
+    seen.add(root.path)
+    for child in tree[root.path]:
+        dft_recurse_inner(child, tree, process_field, result, seen)

src/formpack/version.py

@@ -1,26 +1,23 @@
 # coding: utf-8
 from collections import OrderedDict, defaultdict
-from typing import (
-    Dict,
-    List,
-    Union,
-)
+from typing import Dict, List, Union
 
 from pyxform import aliases as pyxform_aliases
 
 from .constants import UNTRANSLATED
-from .errors import SchemaError
-from .errors import TranslationError
-from .schema import FormField, FormGroup, FormSection, FormChoice
+from .errors import SchemaError, TranslationError
+from .schema import FormChoice, FormField, FormGroup, FormSection
 from .submission import FormSubmission
-from .utils import parse_xml_to_xmljson, normalize_data_type
-from .utils.xlsform_parameters import parameters_string_to_dict, parameters_dict_to_string
-
+from .utils import normalize_data_type, parse_xml_to_xmljson
+from .utils.dft import dft_recurse
 from .utils.flatten_content import flatten_content
 from .utils.xform_tools import formversion_pyxform
+from .utils.xlsform_parameters import (
+    parameters_dict_to_string,
+    parameters_string_to_dict,
+)
 from .validators import validate_content
 
-
 YES_NO = pyxform_aliases.yes_no
 
 
@@ -123,24 +120,37 @@ def _get_fields_by_source(self) -> Dict[str, List[FormField]]:
             fields_by_source[field.source].append(field)
         return fields_by_source
 
-    def _map_sections_to_analysis_fields(
-        self, survey_field: FormField
-    ) -> List[FormField]:
-        _fields = []
-        for analysis_field in self.fields_by_source[survey_field.path]:
-            analysis_field.section = survey_field.section
-            analysis_field.source_field = survey_field
-            _fields.append(analysis_field)
-        return _fields
+    def _map_sections_to_analysis_field(
+        self,
+        survey_fields_by_path: dict[str, FormField],
+        analysis_field: FormField,
+    ) -> FormField:
+        if (
+            not hasattr(analysis_field, 'source')
+            or analysis_field.source not in survey_fields_by_path
+        ):
+            return analysis_field
+        survey_field = survey_fields_by_path[analysis_field.source]
+        analysis_field.section = survey_field.section
+        analysis_field.source_field = survey_field
+        return analysis_field
 
     def insert_analysis_fields(
-        self, fields: List[FormField]
+        self, survey_fields: List[FormField]
     ) -> List[FormField]:
+        all_fields = [*survey_fields, *self.fields]
+        survey_fields_by_path = {field.path: field for field in all_fields}
         _fields = []
-        for field in fields:
-            _fields.append(field)
-            if field.path in self.fields_by_source:
-                _fields += self._map_sections_to_analysis_fields(field)
+        for field in survey_fields:
+            _fields.extend(
+                dft_recurse(
+                    root=field,
+                    tree=self.fields_by_source,
+                    process_field=lambda node: self._map_sections_to_analysis_field(
+                        survey_fields_by_path, node
+                    ),
+                )
+            )
         return _fields
 
 

tests/fixtures/analysis_form_advanced/analysis_form.json

@@ -41,6 +41,12 @@
       }
     ]
   },
+  {
+    "type": "qualVerification",
+    "name": "clerk_interactions/record_a_note/uuid_for_tone_of_voice/verified",
+    "label": "verified",
+    "source": "clerk_interactions/record_a_note/uuid_for_tone_of_voice"
+  },
   {
     "type": "qualText",
     "name": "clerk_interactions/goods_sold/uuid_for_comment",

tests/fixtures/analysis_form_advanced/v1.json

@@ -85,6 +85,7 @@
           },
           "qual": {
             "uuid_for_tone_of_voice": {
+              "verified": true,
               "value": [
                 {
                   "uuid": "uuid_for_excited",

tests/test_additional_field_exports.py

@@ -171,6 +171,7 @@ def test_additional_field_exports_advanced():
         'record_a_note',
         'record_a_note - transcript (en)',
         "record_a_note - How was the tone of the clerk's voice?",
+        "record_a_note - How was the tone of the clerk's voice? - verified",
         'goods_sold',
         'goods_sold/chocolate',
         'goods_sold/fruit',
@@ -183,6 +184,7 @@ def test_additional_field_exports_advanced():
             'clerk_interaction_1.mp3',
             'Hello how may I help you?',
             'Excited,Confused',
+            True,
             'chocolate',
             '1',
             '0',
@@ -194,6 +196,7 @@ def test_additional_field_exports_advanced():
             'clerk_interaction_2.mp3',
             'Thank you for your business',
             'Anxious,Excited',
+            False,
             'chocolate fruit pasta',
             '1',
             '1',
@@ -205,6 +208,7 @@ def test_additional_field_exports_advanced():
             'clerk_interaction_3.mp3',
             '',
             '',
+            '',
             'pasta',
             '0',
             '0',
@@ -223,6 +227,7 @@ def test_additional_field_exports_advanced():
         'record_a_note',
         'record_a_note - transcript (en)',
         "record_a_note - How was the tone of the clerk's voice?",
+        "record_a_note - How was the tone of the clerk's voice? - verified",
         'goods_sold/chocolate',
         'goods_sold/fruit',
         'goods_sold/pasta',
@@ -234,6 +239,7 @@ def test_additional_field_exports_advanced():
             'clerk_interaction_1.mp3',
             'Hello how may I help you?',
             'Excited,Confused',
+            True,
             '1',
             '0',
             '0',
@@ -244,6 +250,7 @@ def test_additional_field_exports_advanced():
             'clerk_interaction_2.mp3',
             'Thank you for your business',
             'Anxious,Excited',
+            False,
             '1',
             '1',
             '1',
@@ -254,6 +261,7 @@ def test_additional_field_exports_advanced():
             'clerk_interaction_3.mp3',
             '',
             '',
+            '',
             '0',
             '0',
             '1',
@@ -271,6 +279,7 @@ def test_additional_field_exports_advanced():
         'record_a_note',
         'record_a_note - transcript (en)',
         "record_a_note - How was the tone of the clerk's voice?",
+        "record_a_note - How was the tone of the clerk's voice? - verified",
         'goods_sold',
         'goods_sold - Comment on the goods sold at the store',
         'goods_sold - Rate the quality of the goods sold at the store',
@@ -280,6 +289,7 @@ def test_additional_field_exports_advanced():
             'clerk_interaction_1.mp3',
             'Hello how may I help you?',
             'Excited,Confused',
+            True,
             'chocolate',
             'Not much diversity',
             'High quality',
@@ -288,6 +298,7 @@ def test_additional_field_exports_advanced():
             'clerk_interaction_2.mp3',
             'Thank you for your business',
             'Anxious,Excited',
+            False,
             'chocolate fruit pasta',
             '',
             'Average quality',
@@ -296,6 +307,7 @@ def test_additional_field_exports_advanced():
             'clerk_interaction_3.mp3',
             '',
             '',
+            '',
             'pasta',
             '',
             'High quality',
@@ -507,3 +519,4 @@ def test_simple_report_with_analysis_form():
         "What is the shop's name?",
         "What is the clerk's name?",
     ]
+