From 791272b417c0bbc74e26576d5f9c2b10822ebbff Mon Sep 17 00:00:00 2001 From: rgraber Date: Tue, 17 Feb 2026 09:48:58 -0500 Subject: [PATCH 1/8] feat(qual): new QA verification field --- src/formpack/schema/fields.py | 30 ++++++++++++++++--- src/formpack/version.py | 13 +++++++- .../analysis_form_advanced/analysis_form.json | 6 ++++ tests/test_additional_field_exports.py | 12 ++++++++ 4 files changed, 56 insertions(+), 5 deletions(-) diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index d8229d8..2028d58 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -1,17 +1,18 @@ # coding: utf-8 +import logging import math -from collections import defaultdict, OrderedDict -from dateutil import parser +import statistics +from collections import OrderedDict, defaultdict from functools import partial from operator import itemgetter -import statistics +from dateutil import parser -from .datadef import FormDataDef, FormChoice from ..constants import UNSPECIFIED_TRANSLATION from ..utils import singlemode from ..utils.ordered_collection import OrderedDefaultdict from ..utils.string import list_to_csv +from .datadef import FormChoice, FormDataDef class FormField(FormDataDef): @@ -262,6 +263,7 @@ def from_json_definition( 'qualText': QualField, 'transcript': QualTranscriptField, 'translation': QualTranslationField, + 'qualVerification': QualVerificationField, } args = { @@ -673,6 +675,26 @@ def _get_label(self, *args, **kwargs): return f'{source_label} - translation ({self.language})' +class QualVerificationField(QualField): + def _get_label(self, *args, **kwargs): + source_label = self.source_field._get_label(*args, **kwargs) + return f'{source_label} - verified' + + def get_value_from_entry(self, entry): + name_parts = self.name.split('/') + # source question path, analysis question uuid, "verified" + assert len(name_parts) >= 3 + analysis_question_uuid = name_parts[-2] + survey_question_path = '/'.join(name_parts[:-2]) + try: + response = entry['_supplementalDetails'][survey_question_path][ + 'qual' + ][analysis_question_uuid] + except KeyError: + return '' + return response.get('verified', False) + + class MediaField(TextField): def get_labels(self, include_media_url=False, *args, **kwargs): label = self._get_label(*args, **kwargs) diff --git a/src/formpack/version.py b/src/formpack/version.py index f4b2e9b..65d0d91 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -137,10 +137,21 @@ def insert_analysis_fields( self, fields: List[FormField] ) -> List[FormField]: _fields = [] + i = 0 for field in fields: _fields.append(field) + seen = set() + # DFS-ish traversal + while i < len(_fields): + # add any children of _fields[i] + field = _fields[i] + if field.path in seen: + continue if field.path in self.fields_by_source: - _fields += self._map_sections_to_analysis_fields(field) + # insert any child fields just after their source fields + _fields[i+1:i+1] = self._map_sections_to_analysis_fields(field) + i += 1 + seen.add(field.path) return _fields diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json index 51eac80..0fc1aad 100644 --- a/tests/fixtures/analysis_form_advanced/analysis_form.json +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -41,6 +41,12 @@ } ] }, + { + "type": "qualVerification", + "name": "clerk_interactions/record_a_note/uuid_for_tone_of_voice/verified", + "label": "verified", + "source": "clerk_interactions/record_a_note/uuid_for_tone_of_voice" + }, { "type": "qualText", "name": "clerk_interactions/goods_sold/uuid_for_comment", diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 2fe8650..8d191ae 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -171,6 +171,7 @@ def test_additional_field_exports_advanced(): 'record_a_note', 'record_a_note - transcript (en)', "record_a_note - How was the tone of the clerk's voice?", + "record_a_note - How was the tone of the clerk's voice? - verified", 'goods_sold', 'goods_sold/chocolate', 'goods_sold/fruit', @@ -183,6 +184,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_1.mp3', 'Hello how may I help you?', 'Excited,Confused', + False, 'chocolate', '1', '0', @@ -194,6 +196,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_2.mp3', 'Thank you for your business', 'Anxious,Excited', + False, 'chocolate fruit pasta', '1', '1', @@ -205,6 +208,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_3.mp3', '', '', + '', 'pasta', '0', '0', @@ -223,6 +227,7 @@ def test_additional_field_exports_advanced(): 'record_a_note', 'record_a_note - transcript (en)', "record_a_note - How was the tone of the clerk's voice?", + "record_a_note - How was the tone of the clerk's voice? - verified", 'goods_sold/chocolate', 'goods_sold/fruit', 'goods_sold/pasta', @@ -234,6 +239,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_1.mp3', 'Hello how may I help you?', 'Excited,Confused', + False, '1', '0', '0', @@ -244,6 +250,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_2.mp3', 'Thank you for your business', 'Anxious,Excited', + False, '1', '1', '1', @@ -254,6 +261,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_3.mp3', '', '', + '', '0', '0', '1', @@ -271,6 +279,7 @@ def test_additional_field_exports_advanced(): 'record_a_note', 'record_a_note - transcript (en)', "record_a_note - How was the tone of the clerk's voice?", + "record_a_note - How was the tone of the clerk's voice? - verified", 'goods_sold', 'goods_sold - Comment on the goods sold at the store', 'goods_sold - Rate the quality of the goods sold at the store', @@ -280,6 +289,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_1.mp3', 'Hello how may I help you?', 'Excited,Confused', + False, 'chocolate', 'Not much diversity', 'High quality', @@ -288,6 +298,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_2.mp3', 'Thank you for your business', 'Anxious,Excited', + False, 'chocolate fruit pasta', '', 'Average quality', @@ -296,6 +307,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_3.mp3', '', '', + '', 'pasta', '', 'High quality', From 9e185b79b31a1be592968daef026654b04f9f6a0 Mon Sep 17 00:00:00 2001 From: rgraber Date: Wed, 18 Feb 2026 08:24:12 -0500 Subject: [PATCH 2/8] fixup!: comment --- src/formpack/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/formpack/version.py b/src/formpack/version.py index 65d0d91..2b9c4ae 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -141,7 +141,7 @@ def insert_analysis_fields( for field in fields: _fields.append(field) seen = set() - # DFS-ish traversal + # modified depth-first traversal while i < len(_fields): # add any children of _fields[i] field = _fields[i] From f6a2dfcad0310953898455bc9a3db55a9906004a Mon Sep 17 00:00:00 2001 From: rgraber Date: Wed, 18 Feb 2026 08:27:42 -0500 Subject: [PATCH 3/8] fixup!: rm unused import --- src/formpack/schema/fields.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 2028d58..1a9d04a 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -1,5 +1,4 @@ # coding: utf-8 -import logging import math import statistics from collections import OrderedDict, defaultdict From 6247563bddeb78582a30fea0457e5247ba9df941 Mon Sep 17 00:00:00 2001 From: rgraber Date: Wed, 18 Feb 2026 13:13:55 -0500 Subject: [PATCH 4/8] fixup!: better dft --- src/formpack/version.py | 73 +++++----- tests/fixtures/analysis_form_advanced/v1.json | 1 + tests/test_additional_field_exports.py | 7 +- tests/test_dft.py | 125 ++++++++++++++++++ 4 files changed, 166 insertions(+), 40 deletions(-) create mode 100644 tests/test_dft.py diff --git a/src/formpack/version.py b/src/formpack/version.py index 2b9c4ae..c4e33aa 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -1,26 +1,23 @@ # coding: utf-8 from collections import OrderedDict, defaultdict -from typing import ( - Dict, - List, - Union, -) +from typing import Dict, List, Union from pyxform import aliases as pyxform_aliases from .constants import UNTRANSLATED -from .errors import SchemaError -from .errors import TranslationError -from .schema import FormField, FormGroup, FormSection, FormChoice +from .errors import SchemaError, TranslationError +from .schema import FormChoice, FormField, FormGroup, FormSection from .submission import FormSubmission -from .utils import parse_xml_to_xmljson, normalize_data_type -from .utils.xlsform_parameters import parameters_string_to_dict, parameters_dict_to_string - +from .utils import normalize_data_type, parse_xml_to_xmljson +from .utils.dft import dft_recurse from .utils.flatten_content import flatten_content from .utils.xform_tools import formversion_pyxform +from .utils.xlsform_parameters import ( + parameters_dict_to_string, + parameters_string_to_dict, +) from .validators import validate_content - YES_NO = pyxform_aliases.yes_no @@ -123,35 +120,37 @@ def _get_fields_by_source(self) -> Dict[str, List[FormField]]: fields_by_source[field.source].append(field) return fields_by_source - def _map_sections_to_analysis_fields( - self, survey_field: FormField - ) -> List[FormField]: - _fields = [] - for analysis_field in self.fields_by_source[survey_field.path]: - analysis_field.section = survey_field.section - analysis_field.source_field = survey_field - _fields.append(analysis_field) - return _fields + def _map_sections_to_analysis_field( + self, + survey_fields_by_path: dict[str, FormField], + analysis_field: FormField, + ) -> FormField: + if ( + not hasattr(analysis_field, 'source') + or analysis_field.source not in survey_fields_by_path + ): + return analysis_field + survey_field = survey_fields_by_path[analysis_field.source] + analysis_field.section = survey_field.section + analysis_field.source_field = survey_field + return analysis_field def insert_analysis_fields( - self, fields: List[FormField] + self, survey_fields: List[FormField] ) -> List[FormField]: + all_fields = [*survey_fields, *self.fields] + survey_fields_by_path = {field.path: field for field in all_fields} _fields = [] - i = 0 - for field in fields: - _fields.append(field) - seen = set() - # modified depth-first traversal - while i < len(_fields): - # add any children of _fields[i] - field = _fields[i] - if field.path in seen: - continue - if field.path in self.fields_by_source: - # insert any child fields just after their source fields - _fields[i+1:i+1] = self._map_sections_to_analysis_fields(field) - i += 1 - seen.add(field.path) + for field in survey_fields: + _fields.extend( + dft_recurse( + root=field, + tree=self.fields_by_source, + process_field=lambda node: self._map_sections_to_analysis_field( + survey_fields_by_path, node + ), + ) + ) return _fields diff --git a/tests/fixtures/analysis_form_advanced/v1.json b/tests/fixtures/analysis_form_advanced/v1.json index aab8766..f594af4 100644 --- a/tests/fixtures/analysis_form_advanced/v1.json +++ b/tests/fixtures/analysis_form_advanced/v1.json @@ -85,6 +85,7 @@ }, "qual": { "uuid_for_tone_of_voice": { + "verified": true, "value": [ { "uuid": "uuid_for_excited", diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 8d191ae..ac3373c 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -184,7 +184,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_1.mp3', 'Hello how may I help you?', 'Excited,Confused', - False, + True, 'chocolate', '1', '0', @@ -239,7 +239,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_1.mp3', 'Hello how may I help you?', 'Excited,Confused', - False, + True, '1', '0', '0', @@ -289,7 +289,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_1.mp3', 'Hello how may I help you?', 'Excited,Confused', - False, + True, 'chocolate', 'Not much diversity', 'High quality', @@ -519,3 +519,4 @@ def test_simple_report_with_analysis_form(): "What is the shop's name?", "What is the clerk's name?", ] + diff --git a/tests/test_dft.py b/tests/test_dft.py new file mode 100644 index 0000000..3905f46 --- /dev/null +++ b/tests/test_dft.py @@ -0,0 +1,125 @@ +from collections import defaultdict + +from formpack.schema import FormField +from formpack.utils.dft import dft_recurse + + +def test_depth_first_traversal(): + uuids = ['uuid-analysis-q1', 'uuid-analysis-q2'] + survey_field = FormField.from_json_definition( + definition={ + 'type': 'audio', + '$kuid': 'pq4yg66', + 'label': ['q1'], + '$xpath': 'q1', + 'required': False, + 'name': 'q1', + }, + translations=[None], + ) + analysis_fields = [] + # add QA questions + for i in range(2): + q_uuid = uuids[i] + analysis_fields.append( + FormField.from_json_definition( + definition={ + 'label': f'Analysis question {i}?', + 'source': 'q1', + 'name': f'q1/{q_uuid}', + 'type': 'qualInteger', + 'dtpath': f'q1/{q_uuid}', + }, + translations=[None], + ) + ) + # add verification fields + for i in range(2): + q_uuid = uuids[i] + analysis_fields.append( + FormField.from_json_definition( + definition={ + 'label': f'Analysis question {i} verification', + 'source': f'q1/{q_uuid}', + 'name': f'q1/{q_uuid}/verification', + 'type': 'qualVerification', + 'dtpath': f'q1/{q_uuid}/verified', + }, + translations=[None], + ) + ) + tree = defaultdict(list) + for field in analysis_fields: + tree[field.source].append(field) + all_nodes = dft_recurse( + root=survey_field, tree=tree, process_field=lambda x: x + ) + all_nodes = [node.name for node in all_nodes] + assert all_nodes == [ + 'q1', + 'q1/uuid-analysis-q1', + 'q1/uuid-analysis-q1/verification', + 'q1/uuid-analysis-q2', + 'q1/uuid-analysis-q2/verification', + ] + +def test_depth_first_traversal_handles_cycles(): + uuids = ['uuid-analysis-q1', 'uuid-analysis-q2'] + survey_field = FormField.from_json_definition( + definition={ + 'type': 'audio', + '$kuid': 'pq4yg66', + 'label': ['q1'], + '$xpath': 'q1', + 'required': False, + 'name': 'q1', + }, + translations=[None], + ) + analysis_fields = [] + # add QA questions + for i in range(2): + q_uuid = uuids[i] + analysis_fields.append( + FormField.from_json_definition( + definition={ + 'label': f'Analysis question {i}?', + 'source': 'q1', + 'name': f'q1/{q_uuid}', + 'type': 'qualInteger', + 'dtpath': f'q1/{q_uuid}', + }, + translations=[None], + ) + ) + # add verification fields + for i in range(2): + q_uuid = uuids[i] + analysis_fields.append( + FormField.from_json_definition( + definition={ + 'label': f'Analysis question {i} verification', + 'source': f'q1/{q_uuid}', + 'name': f'q1/{q_uuid}/verification', + 'type': 'qualVerification', + 'dtpath': f'q1/{q_uuid}/verified', + }, + translations=[None], + ) + ) + tree = defaultdict(list) + for field in analysis_fields: + tree[field.source].append(field) + # force a circular path + tree['q1/uuid-analysis-q1/verification'] = [survey_field] + all_nodes = dft_recurse( + root=survey_field, tree=tree, process_field=lambda x: x + ) + all_nodes = [node.name for node in all_nodes] + assert all_nodes == [ + 'q1', + 'q1/uuid-analysis-q1', + 'q1/uuid-analysis-q1/verification', + 'q1/uuid-analysis-q2', + 'q1/uuid-analysis-q2/verification', + ] From e639012a02a4911ebc23b2059fa96f99cfc514ae Mon Sep 17 00:00:00 2001 From: rgraber Date: Wed, 18 Feb 2026 13:15:28 -0500 Subject: [PATCH 5/8] fixup!: add file --- src/formpack/utils/dft.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 src/formpack/utils/dft.py diff --git a/src/formpack/utils/dft.py b/src/formpack/utils/dft.py new file mode 100644 index 0000000..20eeb31 --- /dev/null +++ b/src/formpack/utils/dft.py @@ -0,0 +1,20 @@ + +from formpack.schema import FormField + +# Basic recursive depth-first traversal +def dft_recurse(root:FormField, tree: dict[str, list[FormField]], process_field): + seen = set() + result = [root] + seen.add(root.path) + for child in tree[root.path]: + dft_recurse_inner(child, tree, process_field, result, seen) + return result + +def dft_recurse_inner(root:FormField, tree: dict[str, list[FormField]], process_field, result, seen): + if root.path in seen: + return + result.append(process_field(root)) + seen.add(root.path) + for child in tree[root.path]: + dft_recurse_inner(child, tree, process_field, result, seen) + From 4d98b93514c5d2491df2c921a7fa5565facbbd14 Mon Sep 17 00:00:00 2001 From: rgraber Date: Thu, 19 Feb 2026 08:07:46 -0500 Subject: [PATCH 6/8] fixup!: changes from review --- src/formpack/schema/fields.py | 6 ++++-- src/formpack/utils/dft.py | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 1a9d04a..5f591f0 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -521,6 +521,8 @@ def format( class QualField(TextField): + SUPPLEMENTAL_DETAILS_FIELD = '_supplementalDetails' + def _get_label(self, *args, **kwargs): source_label = self.source_field._get_label(*args, **kwargs) # hard-coded first label because qualitative analysis does not yet @@ -541,7 +543,7 @@ def get_value_from_entry(self, entry): try: # all responses nested within `qual` - responses = entry['_supplementalDetails'][source]['qual'] + responses = entry[self.SUPPLEMENTAL_DETAILS_FIELD][source]['qual'] except KeyError: return '' @@ -686,7 +688,7 @@ def get_value_from_entry(self, entry): analysis_question_uuid = name_parts[-2] survey_question_path = '/'.join(name_parts[:-2]) try: - response = entry['_supplementalDetails'][survey_question_path][ + response = entry[self.SUPPLEMENTAL_DETAILS_FIELD][survey_question_path][ 'qual' ][analysis_question_uuid] except KeyError: diff --git a/src/formpack/utils/dft.py b/src/formpack/utils/dft.py index 20eeb31..a1bdeee 100644 --- a/src/formpack/utils/dft.py +++ b/src/formpack/utils/dft.py @@ -1,8 +1,13 @@ +from typing import Callable, Any from formpack.schema import FormField # Basic recursive depth-first traversal -def dft_recurse(root:FormField, tree: dict[str, list[FormField]], process_field): +def dft_recurse( + root: FormField, + tree: dict[str, list[FormField]], + process_field: Callable[[FormField], Any] +): seen = set() result = [root] seen.add(root.path) @@ -10,11 +15,16 @@ def dft_recurse(root:FormField, tree: dict[str, list[FormField]], process_field) dft_recurse_inner(child, tree, process_field, result, seen) return result -def dft_recurse_inner(root:FormField, tree: dict[str, list[FormField]], process_field, result, seen): +def dft_recurse_inner( + root: FormField, + tree: dict[str, list[FormField]], + process_field: Callable[[FormField], Any], + result: list[Any], + seen: set[str] +): if root.path in seen: return result.append(process_field(root)) seen.add(root.path) for child in tree[root.path]: dft_recurse_inner(child, tree, process_field, result, seen) - From 5ac7547122db79dd080df162db8add2472f101bc Mon Sep 17 00:00:00 2001 From: rgraber Date: Thu, 19 Feb 2026 08:10:16 -0500 Subject: [PATCH 7/8] fixup!: darker --- src/formpack/schema/fields.py | 6 +++--- src/formpack/utils/dft.py | 8 +++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index 5f591f0..cb02087 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -688,9 +688,9 @@ def get_value_from_entry(self, entry): analysis_question_uuid = name_parts[-2] survey_question_path = '/'.join(name_parts[:-2]) try: - response = entry[self.SUPPLEMENTAL_DETAILS_FIELD][survey_question_path][ - 'qual' - ][analysis_question_uuid] + response = entry[self.SUPPLEMENTAL_DETAILS_FIELD][ + survey_question_path + ]['qual'][analysis_question_uuid] except KeyError: return '' return response.get('verified', False) diff --git a/src/formpack/utils/dft.py b/src/formpack/utils/dft.py index a1bdeee..8c76d57 100644 --- a/src/formpack/utils/dft.py +++ b/src/formpack/utils/dft.py @@ -1,12 +1,13 @@ -from typing import Callable, Any +from typing import Any, Callable from formpack.schema import FormField + # Basic recursive depth-first traversal def dft_recurse( root: FormField, tree: dict[str, list[FormField]], - process_field: Callable[[FormField], Any] + process_field: Callable[[FormField], Any], ): seen = set() result = [root] @@ -15,12 +16,13 @@ def dft_recurse( dft_recurse_inner(child, tree, process_field, result, seen) return result + def dft_recurse_inner( root: FormField, tree: dict[str, list[FormField]], process_field: Callable[[FormField], Any], result: list[Any], - seen: set[str] + seen: set[str], ): if root.path in seen: return From 20a0a30f085f5e3639b4d3b67a4d2b25d144d5ae Mon Sep 17 00:00:00 2001 From: rgraber Date: Thu, 19 Feb 2026 09:47:41 -0500 Subject: [PATCH 8/8] fixup!: source field --- src/formpack/schema/fields.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index cb02087..c6e9f8c 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -263,6 +263,7 @@ def from_json_definition( 'transcript': QualTranscriptField, 'translation': QualTranslationField, 'qualVerification': QualVerificationField, + 'qualSource': QualSourceField, } args = { @@ -676,14 +677,18 @@ def _get_label(self, *args, **kwargs): return f'{source_label} - translation ({self.language})' -class QualVerificationField(QualField): +class QualMetadataField(QualField): + @property + def value_field(self): + raise NotImplementedError() + def _get_label(self, *args, **kwargs): source_label = self.source_field._get_label(*args, **kwargs) - return f'{source_label} - verified' + return f'{source_label} - {self.value_field}' def get_value_from_entry(self, entry): name_parts = self.name.split('/') - # source question path, analysis question uuid, "verified" + # source question path, analysis question uuid, value field assert len(name_parts) >= 3 analysis_question_uuid = name_parts[-2] survey_question_path = '/'.join(name_parts[:-2]) @@ -693,7 +698,19 @@ def get_value_from_entry(self, entry): ]['qual'][analysis_question_uuid] except KeyError: return '' - return response.get('verified', False) + return response.get(self.value_field, False) + + +class QualVerificationField(QualMetadataField): + @property + def value_field(self): + return 'verified' + + +class QualSourceField(QualMetadataField): + @property + def value_field(self): + return 'source' class MediaField(TextField):