diff --git a/src/formpack/schema/fields.py b/src/formpack/schema/fields.py index d8229d8..c6e9f8c 100644 --- a/src/formpack/schema/fields.py +++ b/src/formpack/schema/fields.py @@ -1,17 +1,17 @@ # coding: utf-8 import math -from collections import defaultdict, OrderedDict -from dateutil import parser +import statistics +from collections import OrderedDict, defaultdict from functools import partial from operator import itemgetter -import statistics +from dateutil import parser -from .datadef import FormDataDef, FormChoice from ..constants import UNSPECIFIED_TRANSLATION from ..utils import singlemode from ..utils.ordered_collection import OrderedDefaultdict from ..utils.string import list_to_csv +from .datadef import FormChoice, FormDataDef class FormField(FormDataDef): @@ -262,6 +262,8 @@ def from_json_definition( 'qualText': QualField, 'transcript': QualTranscriptField, 'translation': QualTranslationField, + 'qualVerification': QualVerificationField, + 'qualSource': QualSourceField, } args = { @@ -520,6 +522,8 @@ def format( class QualField(TextField): + SUPPLEMENTAL_DETAILS_FIELD = '_supplementalDetails' + def _get_label(self, *args, **kwargs): source_label = self.source_field._get_label(*args, **kwargs) # hard-coded first label because qualitative analysis does not yet @@ -540,7 +544,7 @@ def get_value_from_entry(self, entry): try: # all responses nested within `qual` - responses = entry['_supplementalDetails'][source]['qual'] + responses = entry[self.SUPPLEMENTAL_DETAILS_FIELD][source]['qual'] except KeyError: return '' @@ -673,6 +677,42 @@ def _get_label(self, *args, **kwargs): return f'{source_label} - translation ({self.language})' +class QualMetadataField(QualField): + @property + def value_field(self): + raise NotImplementedError() + + def _get_label(self, *args, **kwargs): + source_label = self.source_field._get_label(*args, **kwargs) + return f'{source_label} - {self.value_field}' + + def get_value_from_entry(self, entry): + name_parts = self.name.split('/') + # source question path, analysis question uuid, value field + assert len(name_parts) >= 3 + analysis_question_uuid = name_parts[-2] + survey_question_path = '/'.join(name_parts[:-2]) + try: + response = entry[self.SUPPLEMENTAL_DETAILS_FIELD][ + survey_question_path + ]['qual'][analysis_question_uuid] + except KeyError: + return '' + return response.get(self.value_field, False) + + +class QualVerificationField(QualMetadataField): + @property + def value_field(self): + return 'verified' + + +class QualSourceField(QualMetadataField): + @property + def value_field(self): + return 'source' + + class MediaField(TextField): def get_labels(self, include_media_url=False, *args, **kwargs): label = self._get_label(*args, **kwargs) diff --git a/src/formpack/utils/dft.py b/src/formpack/utils/dft.py new file mode 100644 index 0000000..8c76d57 --- /dev/null +++ b/src/formpack/utils/dft.py @@ -0,0 +1,32 @@ +from typing import Any, Callable + +from formpack.schema import FormField + + +# Basic recursive depth-first traversal +def dft_recurse( + root: FormField, + tree: dict[str, list[FormField]], + process_field: Callable[[FormField], Any], +): + seen = set() + result = [root] + seen.add(root.path) + for child in tree[root.path]: + dft_recurse_inner(child, tree, process_field, result, seen) + return result + + +def dft_recurse_inner( + root: FormField, + tree: dict[str, list[FormField]], + process_field: Callable[[FormField], Any], + result: list[Any], + seen: set[str], +): + if root.path in seen: + return + result.append(process_field(root)) + seen.add(root.path) + for child in tree[root.path]: + dft_recurse_inner(child, tree, process_field, result, seen) diff --git a/src/formpack/version.py b/src/formpack/version.py index f4b2e9b..c4e33aa 100644 --- a/src/formpack/version.py +++ b/src/formpack/version.py @@ -1,26 +1,23 @@ # coding: utf-8 from collections import OrderedDict, defaultdict -from typing import ( - Dict, - List, - Union, -) +from typing import Dict, List, Union from pyxform import aliases as pyxform_aliases from .constants import UNTRANSLATED -from .errors import SchemaError -from .errors import TranslationError -from .schema import FormField, FormGroup, FormSection, FormChoice +from .errors import SchemaError, TranslationError +from .schema import FormChoice, FormField, FormGroup, FormSection from .submission import FormSubmission -from .utils import parse_xml_to_xmljson, normalize_data_type -from .utils.xlsform_parameters import parameters_string_to_dict, parameters_dict_to_string - +from .utils import normalize_data_type, parse_xml_to_xmljson +from .utils.dft import dft_recurse from .utils.flatten_content import flatten_content from .utils.xform_tools import formversion_pyxform +from .utils.xlsform_parameters import ( + parameters_dict_to_string, + parameters_string_to_dict, +) from .validators import validate_content - YES_NO = pyxform_aliases.yes_no @@ -123,24 +120,37 @@ def _get_fields_by_source(self) -> Dict[str, List[FormField]]: fields_by_source[field.source].append(field) return fields_by_source - def _map_sections_to_analysis_fields( - self, survey_field: FormField - ) -> List[FormField]: - _fields = [] - for analysis_field in self.fields_by_source[survey_field.path]: - analysis_field.section = survey_field.section - analysis_field.source_field = survey_field - _fields.append(analysis_field) - return _fields + def _map_sections_to_analysis_field( + self, + survey_fields_by_path: dict[str, FormField], + analysis_field: FormField, + ) -> FormField: + if ( + not hasattr(analysis_field, 'source') + or analysis_field.source not in survey_fields_by_path + ): + return analysis_field + survey_field = survey_fields_by_path[analysis_field.source] + analysis_field.section = survey_field.section + analysis_field.source_field = survey_field + return analysis_field def insert_analysis_fields( - self, fields: List[FormField] + self, survey_fields: List[FormField] ) -> List[FormField]: + all_fields = [*survey_fields, *self.fields] + survey_fields_by_path = {field.path: field for field in all_fields} _fields = [] - for field in fields: - _fields.append(field) - if field.path in self.fields_by_source: - _fields += self._map_sections_to_analysis_fields(field) + for field in survey_fields: + _fields.extend( + dft_recurse( + root=field, + tree=self.fields_by_source, + process_field=lambda node: self._map_sections_to_analysis_field( + survey_fields_by_path, node + ), + ) + ) return _fields diff --git a/tests/fixtures/analysis_form_advanced/analysis_form.json b/tests/fixtures/analysis_form_advanced/analysis_form.json index 51eac80..0fc1aad 100644 --- a/tests/fixtures/analysis_form_advanced/analysis_form.json +++ b/tests/fixtures/analysis_form_advanced/analysis_form.json @@ -41,6 +41,12 @@ } ] }, + { + "type": "qualVerification", + "name": "clerk_interactions/record_a_note/uuid_for_tone_of_voice/verified", + "label": "verified", + "source": "clerk_interactions/record_a_note/uuid_for_tone_of_voice" + }, { "type": "qualText", "name": "clerk_interactions/goods_sold/uuid_for_comment", diff --git a/tests/fixtures/analysis_form_advanced/v1.json b/tests/fixtures/analysis_form_advanced/v1.json index aab8766..f594af4 100644 --- a/tests/fixtures/analysis_form_advanced/v1.json +++ b/tests/fixtures/analysis_form_advanced/v1.json @@ -85,6 +85,7 @@ }, "qual": { "uuid_for_tone_of_voice": { + "verified": true, "value": [ { "uuid": "uuid_for_excited", diff --git a/tests/test_additional_field_exports.py b/tests/test_additional_field_exports.py index 2fe8650..ac3373c 100644 --- a/tests/test_additional_field_exports.py +++ b/tests/test_additional_field_exports.py @@ -171,6 +171,7 @@ def test_additional_field_exports_advanced(): 'record_a_note', 'record_a_note - transcript (en)', "record_a_note - How was the tone of the clerk's voice?", + "record_a_note - How was the tone of the clerk's voice? - verified", 'goods_sold', 'goods_sold/chocolate', 'goods_sold/fruit', @@ -183,6 +184,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_1.mp3', 'Hello how may I help you?', 'Excited,Confused', + True, 'chocolate', '1', '0', @@ -194,6 +196,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_2.mp3', 'Thank you for your business', 'Anxious,Excited', + False, 'chocolate fruit pasta', '1', '1', @@ -205,6 +208,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_3.mp3', '', '', + '', 'pasta', '0', '0', @@ -223,6 +227,7 @@ def test_additional_field_exports_advanced(): 'record_a_note', 'record_a_note - transcript (en)', "record_a_note - How was the tone of the clerk's voice?", + "record_a_note - How was the tone of the clerk's voice? - verified", 'goods_sold/chocolate', 'goods_sold/fruit', 'goods_sold/pasta', @@ -234,6 +239,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_1.mp3', 'Hello how may I help you?', 'Excited,Confused', + True, '1', '0', '0', @@ -244,6 +250,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_2.mp3', 'Thank you for your business', 'Anxious,Excited', + False, '1', '1', '1', @@ -254,6 +261,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_3.mp3', '', '', + '', '0', '0', '1', @@ -271,6 +279,7 @@ def test_additional_field_exports_advanced(): 'record_a_note', 'record_a_note - transcript (en)', "record_a_note - How was the tone of the clerk's voice?", + "record_a_note - How was the tone of the clerk's voice? - verified", 'goods_sold', 'goods_sold - Comment on the goods sold at the store', 'goods_sold - Rate the quality of the goods sold at the store', @@ -280,6 +289,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_1.mp3', 'Hello how may I help you?', 'Excited,Confused', + True, 'chocolate', 'Not much diversity', 'High quality', @@ -288,6 +298,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_2.mp3', 'Thank you for your business', 'Anxious,Excited', + False, 'chocolate fruit pasta', '', 'Average quality', @@ -296,6 +307,7 @@ def test_additional_field_exports_advanced(): 'clerk_interaction_3.mp3', '', '', + '', 'pasta', '', 'High quality', @@ -507,3 +519,4 @@ def test_simple_report_with_analysis_form(): "What is the shop's name?", "What is the clerk's name?", ] + diff --git a/tests/test_dft.py b/tests/test_dft.py new file mode 100644 index 0000000..3905f46 --- /dev/null +++ b/tests/test_dft.py @@ -0,0 +1,125 @@ +from collections import defaultdict + +from formpack.schema import FormField +from formpack.utils.dft import dft_recurse + + +def test_depth_first_traversal(): + uuids = ['uuid-analysis-q1', 'uuid-analysis-q2'] + survey_field = FormField.from_json_definition( + definition={ + 'type': 'audio', + '$kuid': 'pq4yg66', + 'label': ['q1'], + '$xpath': 'q1', + 'required': False, + 'name': 'q1', + }, + translations=[None], + ) + analysis_fields = [] + # add QA questions + for i in range(2): + q_uuid = uuids[i] + analysis_fields.append( + FormField.from_json_definition( + definition={ + 'label': f'Analysis question {i}?', + 'source': 'q1', + 'name': f'q1/{q_uuid}', + 'type': 'qualInteger', + 'dtpath': f'q1/{q_uuid}', + }, + translations=[None], + ) + ) + # add verification fields + for i in range(2): + q_uuid = uuids[i] + analysis_fields.append( + FormField.from_json_definition( + definition={ + 'label': f'Analysis question {i} verification', + 'source': f'q1/{q_uuid}', + 'name': f'q1/{q_uuid}/verification', + 'type': 'qualVerification', + 'dtpath': f'q1/{q_uuid}/verified', + }, + translations=[None], + ) + ) + tree = defaultdict(list) + for field in analysis_fields: + tree[field.source].append(field) + all_nodes = dft_recurse( + root=survey_field, tree=tree, process_field=lambda x: x + ) + all_nodes = [node.name for node in all_nodes] + assert all_nodes == [ + 'q1', + 'q1/uuid-analysis-q1', + 'q1/uuid-analysis-q1/verification', + 'q1/uuid-analysis-q2', + 'q1/uuid-analysis-q2/verification', + ] + +def test_depth_first_traversal_handles_cycles(): + uuids = ['uuid-analysis-q1', 'uuid-analysis-q2'] + survey_field = FormField.from_json_definition( + definition={ + 'type': 'audio', + '$kuid': 'pq4yg66', + 'label': ['q1'], + '$xpath': 'q1', + 'required': False, + 'name': 'q1', + }, + translations=[None], + ) + analysis_fields = [] + # add QA questions + for i in range(2): + q_uuid = uuids[i] + analysis_fields.append( + FormField.from_json_definition( + definition={ + 'label': f'Analysis question {i}?', + 'source': 'q1', + 'name': f'q1/{q_uuid}', + 'type': 'qualInteger', + 'dtpath': f'q1/{q_uuid}', + }, + translations=[None], + ) + ) + # add verification fields + for i in range(2): + q_uuid = uuids[i] + analysis_fields.append( + FormField.from_json_definition( + definition={ + 'label': f'Analysis question {i} verification', + 'source': f'q1/{q_uuid}', + 'name': f'q1/{q_uuid}/verification', + 'type': 'qualVerification', + 'dtpath': f'q1/{q_uuid}/verified', + }, + translations=[None], + ) + ) + tree = defaultdict(list) + for field in analysis_fields: + tree[field.source].append(field) + # force a circular path + tree['q1/uuid-analysis-q1/verification'] = [survey_field] + all_nodes = dft_recurse( + root=survey_field, tree=tree, process_field=lambda x: x + ) + all_nodes = [node.name for node in all_nodes] + assert all_nodes == [ + 'q1', + 'q1/uuid-analysis-q1', + 'q1/uuid-analysis-q1/verification', + 'q1/uuid-analysis-q2', + 'q1/uuid-analysis-q2/verification', + ]