From cccbcc177577b4b24cd956eddf2fa32d281bd0dd Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 9 Jul 2025 14:38:30 -0600 Subject: [PATCH 01/66] retrieve anonymized data for registry --- climmob/processes/db/results.py | 49 +++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 846faf8b..5c9a2abe 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -275,17 +275,22 @@ def getData(userOwner, projectCod, registry, assessments, request): assessmentKey = data.question_code fields = [] + + reg_alias = "reg" + for field in registry["fields"]: - fields.append( - userOwner - + "_" - + projectCod - + ".REG_geninfo." - + field["name"] - + " AS " - + "REG_" - + field["name"] - ) + if field["is_sensitive"]: + fields.append( + f"COALESCE(MAX(" + f"CASE WHEN da.col_name = '{field['name']}' AND da.form_id='-'" + f"THEN da.value END)," + f"{reg_alias}.{field['name']}) " + f"AS REG_{field['name']}" + ) + else: + fields.append( + reg_alias + "." + field["name"] + " AS " + "REG_" + field["name"] + ) for assessment in assessments: for field in assessment["fields"]: fields.append( @@ -311,7 +316,9 @@ def getData(userOwner, projectCod, registry, assessments, request): + "_" + projectCod + ".REG_geninfo " + + reg_alias ) + for assessment in assessments: sql = ( sql @@ -325,10 +332,8 @@ def getData(userOwner, projectCod, registry, assessments, request): ) sql = ( sql - + userOwner - + "_" - + projectCod - + ".REG_geninfo." + + reg_alias + + "." + registryKey + " = " + userOwner @@ -339,13 +344,21 @@ def getData(userOwner, projectCod, registry, assessments, request): + "_geninfo." + assessmentKey ) + sql = ( sql - + " ORDER BY cast(" + + " LEFT JOIN " + userOwner + "_" + projectCod - + ".REG_geninfo." + + ".anony da" + + f" ON da.reg_id = {reg_alias}.qst162 " + + f" GROUP BY {reg_alias}.qst162" + ) + sql = ( + sql + + " ORDER BY cast(" + + f"{reg_alias}." + registryKey + " AS unsigned)" ) @@ -569,7 +582,7 @@ def getJSONResult( if includeRegistry: registryXML = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "db", "reg", "create.xml"] + *[userOwner, projectCod, "db", "reg", "create.xml"], ) if os.path.exists(registryXML): data["registry"] = { @@ -605,7 +618,7 @@ def getJSONResult( "ass", assessment.ass_cod, "create.xml", - ] + ], ) if os.path.exists(assessmentXML): data["assessments"].append( From d2b3315cf4064d17a87b91551bc2a2deafa6c395 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 10 Jul 2025 10:15:56 -0600 Subject: [PATCH 02/66] retrieve anonymized data for assessments --- climmob/processes/db/results.py | 58 +++++++++++++++++---------------- 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 5c9a2abe..5df5ef3c 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -292,21 +292,28 @@ def getData(userOwner, projectCod, registry, assessments, request): reg_alias + "." + field["name"] + " AS " + "REG_" + field["name"] ) for assessment in assessments: + assessment_alias = "assess_" + assessment["code"] for field in assessment["fields"]: - fields.append( - userOwner - + "_" - + projectCod - + ".ASS" - + assessment["code"] - + "_geninfo." - + field["name"] - + " AS " - + "ASS" - + assessment["code"] - + "_" - + field["name"] - ) + if field["is_sensitive"]: + fields.append( + f"COALESCE(MAX(" + f"CASE WHEN da.col_name = '{field['name']}' AND da.form_id='{assessment['code']}'" + f"THEN da.value END)," + f"{assessment_alias}.{field['name']}) " + f"AS " + "ASS" + assessment["code"] + "_" + f"{field['name']}" + ) + else: + fields.append( + assessment_alias + + "." + + field["name"] + + " AS " + + "ASS" + + assessment["code"] + + "_" + + field["name"] + ) sql = ( "SELECT " @@ -320,6 +327,7 @@ def getData(userOwner, projectCod, registry, assessments, request): ) for assessment in assessments: + assessment_alias = "assess_" + assessment["code"] sql = ( sql + " LEFT JOIN " @@ -328,7 +336,9 @@ def getData(userOwner, projectCod, registry, assessments, request): + projectCod + ".ASS" + assessment["code"] - + "_geninfo ON " + + "_geninfo " + + assessment_alias + + " ON " ) sql = ( sql @@ -336,12 +346,8 @@ def getData(userOwner, projectCod, registry, assessments, request): + "." + registryKey + " = " - + userOwner - + "_" - + projectCod - + ".ASS" - + assessment["code"] - + "_geninfo." + + assessment_alias + + "." + assessmentKey ) @@ -355,13 +361,9 @@ def getData(userOwner, projectCod, registry, assessments, request): + f" ON da.reg_id = {reg_alias}.qst162 " + f" GROUP BY {reg_alias}.qst162" ) - sql = ( - sql - + " ORDER BY cast(" - + f"{reg_alias}." - + registryKey - + " AS unsigned)" - ) + sql = sql + f" ORDER BY cast({reg_alias}.{registryKey} AS unsigned)" + + print(sql) data = sql_fetch_all(sql) From 257b4459c77efb41c1af6134ff73513da8f3cdf9 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Tue, 15 Jul 2025 09:50:35 -0600 Subject: [PATCH 03/66] get sensitivity from the database --- climmob/processes/db/question.py | 15 +++++++++++++++ climmob/processes/db/results.py | 32 +++++++++++++++++++++++++------- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index 20aa0655..e5c94e81 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -40,6 +40,7 @@ "getDefaultQuestionLanguage", "getQuestionOwner", "knowIfUserHasCreatedTranslations", + "get_question_sensitivity_by_project_id", ] log = logging.getLogger(__name__) @@ -514,3 +515,17 @@ def knowIfUserHasCreatedTranslations(request, userId): return True return False + + +def get_question_sensitivity_by_project_id(project_id, request): + query = ( + request.dbsession.query(Question.question_code, Question.question_sensitive) + .join(Registry, Registry.question_id == Question.question_id) + .filter(Registry.project_id == project_id) + .union( + request.dbsession.query(Question.question_code, Question.question_sensitive) + .join(AssDetail, AssDetail.question_id == Question.question_id) + .filter(AssDetail.project_id == project_id) + ) + ) + return query.all() diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 5df5ef3c..b4432991 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -1,12 +1,13 @@ import datetime import decimal import os +import re from lxml import etree -from climmob.models import Assessment, Question, Project, mapFromSchema +from climmob.models import Assessment, Question, Project, mapFromSchema, Registry from climmob.models.repository import sql_fetch_all, sql_fetch_one -from climmob.processes import getCombinations +from climmob.processes import getCombinations, get_question_sensitivity_by_project_id __all__ = ["getJSONResult", "getCombinationsData"] @@ -264,7 +265,22 @@ def getPackageData(userOwner, projectId, projectCod, request): return packages -def getData(userOwner, projectCod, registry, assessments, request): +def is_field_sensitive(field, questions): + for q in questions: + if q.question_sensitive == 0: + continue + patterns = [ + rf"^{q.question_code}(_[abc])?(_oth)?$", + rf"^perf_{q.question_code}_[123]$", + rf"^char_{q.question_code}_(pos|neg)$", + ] + for pattern in patterns: + if re.fullmatch(pattern, field["name"]): + return True + return False + + +def getData(userOwner, project_id, projectCod, registry, assessments, request): data = ( request.dbsession.query(Question).filter(Question.question_regkey == 1).first() ) @@ -274,12 +290,14 @@ def getData(userOwner, projectCod, registry, assessments, request): ) assessmentKey = data.question_code + questions = get_question_sensitivity_by_project_id(project_id, request) + fields = [] reg_alias = "reg" for field in registry["fields"]: - if field["is_sensitive"]: + if is_field_sensitive(field, questions): fields.append( f"COALESCE(MAX(" f"CASE WHEN da.col_name = '{field['name']}' AND da.form_id='-'" @@ -294,7 +312,7 @@ def getData(userOwner, projectCod, registry, assessments, request): for assessment in assessments: assessment_alias = "assess_" + assessment["code"] for field in assessment["fields"]: - if field["is_sensitive"]: + if is_field_sensitive(field, questions): fields.append( f"COALESCE(MAX(" f"CASE WHEN da.col_name = '{field['name']}' AND da.form_id='{assessment['code']}'" @@ -363,8 +381,6 @@ def getData(userOwner, projectCod, registry, assessments, request): ) sql = sql + f" ORDER BY cast({reg_alias}.{registryKey} AS unsigned)" - print(sql) - data = sql_fetch_all(sql) result = [] @@ -653,6 +669,7 @@ def getJSONResult( ) data["data"] = getData( userOwner, + projectId, projectCod, data["registry"], data["assessments"], @@ -664,6 +681,7 @@ def getJSONResult( data["specialfields"] = [] data["data"] = getData( userOwner, + projectId, projectCod, data["registry"], data["assessments"], From 096767f713bfd358f19636d324ba8979a970d679 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 16 Jul 2025 14:50:34 -0600 Subject: [PATCH 04/66] add flag to toggle data anonymization --- climmob/processes/db/results.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index b4432991..35867af0 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -280,7 +280,9 @@ def is_field_sensitive(field, questions): return False -def getData(userOwner, project_id, projectCod, registry, assessments, request): +def getData( + userOwner, project_id, projectCod, registry, assessments, request, anonymize=False +): data = ( request.dbsession.query(Question).filter(Question.question_regkey == 1).first() ) @@ -297,7 +299,7 @@ def getData(userOwner, project_id, projectCod, registry, assessments, request): reg_alias = "reg" for field in registry["fields"]: - if is_field_sensitive(field, questions): + if anonymize and is_field_sensitive(field, questions): fields.append( f"COALESCE(MAX(" f"CASE WHEN da.col_name = '{field['name']}' AND da.form_id='-'" @@ -312,7 +314,7 @@ def getData(userOwner, project_id, projectCod, registry, assessments, request): for assessment in assessments: assessment_alias = "assess_" + assessment["code"] for field in assessment["fields"]: - if is_field_sensitive(field, questions): + if anonymize and is_field_sensitive(field, questions): fields.append( f"COALESCE(MAX(" f"CASE WHEN da.col_name = '{field['name']}' AND da.form_id='{assessment['code']}'" @@ -575,6 +577,7 @@ def getJSONResult( includeRegistry=True, includeAssessment=True, assessmentCode="", + anonymize=False, ): data = {} res = ( @@ -686,6 +689,7 @@ def getJSONResult( data["registry"], data["assessments"], request, + anonymize=anonymize, ) data["importantfields"] = [] From 3e74f3f92c7feda023fe0596307f27acb7652bb3 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 16 Jul 2025 15:42:04 -0600 Subject: [PATCH 05/66] add buttons to generate and download anonymized registry and assessments --- climmob/processes/db/results.py | 1 + climmob/templates/dashboard/dashboard.jinja2 | 4 ++++ .../project/productsList/productsList.jinja2 | 8 +++++++ climmob/views/editData.py | 7 ++++++- climmob/views/productsList.py | 21 ++++++++++++------- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 35867af0..30eb88a2 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -677,6 +677,7 @@ def getJSONResult( data["registry"], data["assessments"], request, + anonymize=anonymize, ) data["importantfields"] = getImportantFields(projectId, request) diff --git a/climmob/templates/dashboard/dashboard.jinja2 b/climmob/templates/dashboard/dashboard.jinja2 index d70caf03..b48c10fd 100755 --- a/climmob/templates/dashboard/dashboard.jinja2 +++ b/climmob/templates/dashboard/dashboard.jinja2 @@ -566,7 +566,9 @@ {{ _("View and edit data") }} {{ _("Download data in .CSV format") }} + {{ _("Download anonymized data in .CSV format") }} {{ _("Download data in .XLSX format") }} + {{ _("Download anonymized data in .XLSX format") }} {% block dataprivacy_download_data_registry %} @@ -703,7 +705,9 @@ {% if assessment.asstotal > 0 %} {{ _("View and edit data") }} {{ _("Download data in .CSV format") }} + {{ _("Download anonymized data in .CSV format") }} {{ _("Download data in .XLSX format") }} + {{ _("Download anonymized data in .XLSX format") }} {% block dataprivacy_download_data_assessment scoped%} diff --git a/climmob/templates/snippets/project/productsList/productsList.jinja2 b/climmob/templates/snippets/project/productsList/productsList.jinja2 index 4037e63d..8df6985c 100644 --- a/climmob/templates/snippets/project/productsList/productsList.jinja2 +++ b/climmob/templates/snippets/project/productsList/productsList.jinja2 @@ -72,6 +72,10 @@ {% else %} {% if product.process_name == "create_data_Registration_" %} {{ _("Information collected in the participant registration form in .CSV format") }} + {% elif product.process_name == "create_data_Registration_anonymized_" %} + {{ _("Information collected in the participant registration form in .CSV format (anonymized)") }} + {% elif "anonymized" in product.process_name %} + {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .CSV format (anonymized)") }} {% else %} {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .CSV format") }} {% endif %} @@ -137,6 +141,10 @@ {% else %} {% if product.process_name == "create_data_xlsx_Registration_" %} {{ _("Information collected in the participant registration form in .XLSX format") }} + {% elif product.process_name == "create_data_xlsx_Registration_anonymized_" %} + {{ _("Information collected in the participant registration form in .XLSX format (anonymized)") }} + {% elif "anonymized" in product.process_name %} + {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .XLSX format (anonymized)") }} {% else %} {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .XLSX format") }} {% endif %} diff --git a/climmob/views/editData.py b/climmob/views/editData.py index d990ccea..71c8cc42 100755 --- a/climmob/views/editData.py +++ b/climmob/views/editData.py @@ -30,6 +30,7 @@ def processView(self): activeProjectCod = self.request.matchdict["project"] formId = self.request.matchdict["formid"] formatId = self.request.matchdict["formatid"] + anonymize = bool(self.request.params.get("anonymize")) includeRegistry = True includeAssessment = True code = "" @@ -56,6 +57,9 @@ def processView(self): else: raise HTTPNotFound() + if anonymize: + formId += "_anonymized" + info = getJSONResult( activeProjectUser, activeProjectId, @@ -64,6 +68,7 @@ def processView(self): includeRegistry, includeAssessment, code, + anonymize=anonymize, ) if formatId not in ["csv", "xlsx"]: @@ -216,7 +221,7 @@ def processView(self): path = os.path.join( self.request.registry.settings["user.repository"], - *[activeProjectUser, activeProjectCod] + *[activeProjectUser, activeProjectCod], ) if code == "": paths = ["db", formId, "create.xml"] diff --git a/climmob/views/productsList.py b/climmob/views/productsList.py index ca3ee7e2..002c0d31 100644 --- a/climmob/views/productsList.py +++ b/climmob/views/productsList.py @@ -1,4 +1,5 @@ import os +import re from pyramid.httpexceptions import HTTPFound from pyramid.httpexceptions import HTTPNotFound @@ -114,15 +115,19 @@ def processView(self): "observationcards", "climmobexplanationkit", ]: - assessId = product["process_name"].split("_")[3] - if product["product_id"] == "dataxlsx": - assessId = product["process_name"].split("_")[4] - - product["extraInformation"] = get_project_assessment_info( - activeProjectData["project_id"], - assessId, - self.request, + product["extraInformation"] = None + pattern = re.compile( + r".+?(?:(?:Assessment))_(?:anonymized_)?" # not captured + r"([a-f0-9]{12})" # captured (group 1) ) + match = pattern.fullmatch(product["process_name"]) + if match: + assess_id = match.group(1) + product["extraInformation"] = get_project_assessment_info( + activeProjectData["project_id"], + assess_id, + self.request, + ) productsAvailable.append(product) From 450cb2a84f0045733328d54497f3337eb069af48 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 18 Jul 2025 13:44:13 -0600 Subject: [PATCH 06/66] combine csv and xlsx product creation --- climmob/products/analysisdata/analysisdata.py | 37 ++-- climmob/products/analysisdata/celerytasks.py | 70 ++++--- climmob/products/climmob_products.py | 20 ++ climmob/products/dataxlsx/__init__.py | 1 - climmob/products/dataxlsx/celerytasks.py | 187 ------------------ climmob/products/dataxlsx/dataxlsx.py | 71 ------- climmob/views/editData.py | 40 ++-- climmob/views/productsList.py | 31 ++- climmob/views/project_analysis.py | 4 +- 9 files changed, 116 insertions(+), 345 deletions(-) delete mode 100644 climmob/products/dataxlsx/__init__.py delete mode 100644 climmob/products/dataxlsx/celerytasks.py delete mode 100644 climmob/products/dataxlsx/dataxlsx.py diff --git a/climmob/products/analysisdata/analysisdata.py b/climmob/products/analysisdata/analysisdata.py index 9dc66067..684a2a59 100644 --- a/climmob/products/analysisdata/analysisdata.py +++ b/climmob/products/analysisdata/analysisdata.py @@ -3,34 +3,45 @@ registryHaveQuestionOfMultimediaType, assessmentHaveQuestionOfMultimediaType, ) -from climmob.products.analysisdata.celerytasks import create_CSV +from climmob.products.analysisdata.celerytasks import create_raw_data_file from climmob.products.climmob_products import ( createProductDirectory, registerProductInstance, ) -def create_datacsv(userOwner, projectId, projectCod, info, request, form, code): +def create_raw_data(userOwner, projectId, projectCod, info, request, form, code, file_type="csv", anonymized=False): # We create the plugin directory if it does not exists and return it - # The path user.repository in development.ini/user/project/products/product and - # user.repository in development.ini/user/project/products/product/outputs - path = createProductDirectory(request, userOwner, projectCod, "datacsv") + extra = "-anonymized" if anonymized else "" + + name_output = form + f"_data{extra}" + if code != "": + name_output += "_" + code + + name_output += "_" + projectCod + + path = createProductDirectory(request, userOwner, projectCod, f"data{file_type}{extra}") # We call the Celery task that will generate the output packages.pdf - task = create_CSV.apply_async((path, info, projectCod, form, code), queue="ClimMob") + task = create_raw_data_file.apply_async((path, info, name_output, file_type), queue="ClimMob") # We register the instance of the output with the task ID of celery # This will go to the products table that then you can monitor and use # in the nice product interface # u.registerProductInstance(user, project, 'cards', 'cards.pdf', task.id, request) - nameOutput = form + "_data" - if code != "": - nameOutput += "_" + code + + mimetypes = { + "csv": "text/csv", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + } + mimetype = mimetypes.get(file_type) + + process_name = f"create_data{extra}_{'xlsx_' if file_type == 'xlsx' else ''}" + form + "_" + code registerProductInstance( projectId, - "datacsv", - nameOutput + "_" + projectCod + ".csv", - "text/csv", - "create_data_" + form + "_" + code, + f"data{file_type}{extra}", + name_output + f".{file_type}", + mimetype, + process_name, task.id, request, ) diff --git a/climmob/products/analysisdata/celerytasks.py b/climmob/products/analysisdata/celerytasks.py index 3d7a45d9..df0ef2f0 100644 --- a/climmob/products/analysisdata/celerytasks.py +++ b/climmob/products/analysisdata/celerytasks.py @@ -1,6 +1,6 @@ -import json import os -import shutil as sh + +import pandas as pd from climmob.config.celery_app import celeryApp from climmob.plugins.utilities import climmobCeleryTask @@ -8,37 +8,51 @@ @celeryApp.task(base=climmobCeleryTask) -def create_CSV(path, info, projectCod, form, code): - - # if os.path.exists(path): - # sh.rmtree(path) +def create_raw_data_file(path, info, name_output, file_type): - nameOutput = form + "_data" - if code != "": - nameOutput += "_" + code - - pathout = os.path.join(path, "outputs") + path_out = os.path.join(path, "outputs") if not os.path.exists(path): os.makedirs(path) - os.makedirs(pathout) + os.makedirs(path_out) - if os.path.exists(pathout + "/" + nameOutput + "_" + projectCod + ".csv"): - os.remove(pathout + "/" + nameOutput + "_" + projectCod + ".csv") + # TODO replace labels - pathInputFiles = os.path.join(path, "inputFile") - os.makedirs(pathInputFiles) + df = pd.DataFrame(info) + if file_type == "xlsx": + df.to_excel(os.path.join(path_out, name_output) + f".{file_type}", index=False) + elif file_type == "csv": + df.to_csv(os.path.join(path_out, name_output) + f".{file_type}", index=False) - with open(pathInputFiles + "/info.json", "w") as outfile: - jsonString = json.dumps(info, indent=4, ensure_ascii=False) - outfile.write(jsonString) - if os.path.exists(pathInputFiles + "/info.json"): - try: - createCSV( - pathout + "/" + nameOutput + "_" + projectCod + ".csv", - pathInputFiles + "/info.json", - ) - except Exception as e: - print("We can't create the CSV." + str(e)) + # if os.path.exists(path): + # sh.rmtree(path) - sh.rmtree(pathInputFiles) + # nameOutput = form + "_data" + # if code != "": + # nameOutput += "_" + code + # + # pathout = os.path.join(path, "outputs") + # if not os.path.exists(path): + # os.makedirs(path) + # os.makedirs(pathout) + # + # if os.path.exists(pathout + "/" + nameOutput + "_" + projectCod + ".csv"): + # os.remove(pathout + "/" + nameOutput + "_" + projectCod + ".csv") + # + # pathInputFiles = os.path.join(path, "inputFile") + # os.makedirs(pathInputFiles) + # + # with open(pathInputFiles + "/info.json", "w") as outfile: + # jsonString = json.dumps(info, indent=4, ensure_ascii=False) + # outfile.write(jsonString) + # + # if os.path.exists(pathInputFiles + "/info.json"): + # try: + # createCSV( + # pathout + "/" + nameOutput + "_" + projectCod + ".csv", + # pathInputFiles + "/info.json", + # ) + # except Exception as e: + # print("We can't create the CSV." + str(e)) + # + # sh.rmtree(pathInputFiles) diff --git a/climmob/products/climmob_products.py b/climmob/products/climmob_products.py index 1940b9f6..2d608088 100644 --- a/climmob/products/climmob_products.py +++ b/climmob/products/climmob_products.py @@ -228,6 +228,16 @@ def register_products(config): ) products.append(datacsv) + datacsv_anonymized = addProduct("datacsv-anonymized", "Information collected in the project anonymized.") + addMetadataToProduct(datacsv_anonymized, "author", "Johann Ávalos") + addMetadataToProduct(datacsv_anonymized, "version", "1.0") + addMetadataToProduct( + datacsv_anonymized, + "Licence", + "Copyright 2025, MrBot Software Solutions", + ) + products.append(datacsv_anonymized) + # FORM documentform = addProduct( "documentform", "Create a document pdf to collect information." @@ -362,6 +372,16 @@ def register_products(config): ) products.append(dataxlsx) + dataxlsx_anonymized = addProduct("dataxlsx-anonymized", "Information collected in the project anonymized in XLSX format.") + addMetadataToProduct(dataxlsx_anonymized, "author", "Johann Ávalos") + addMetadataToProduct(dataxlsx_anonymized, "version", "1.0") + addMetadataToProduct( + dataxlsx_anonymized, + "Licence", + "Copyright 2025, MrBot Software Solutions", + ) + products.append(dataxlsx_anonymized) + # INPUT FILES datajson = addProduct( "datajson", "data.json file used as input for report generation." diff --git a/climmob/products/dataxlsx/__init__.py b/climmob/products/dataxlsx/__init__.py deleted file mode 100644 index 95ed645a..00000000 --- a/climmob/products/dataxlsx/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from climmob.products.dataxlsx.celerytasks import * diff --git a/climmob/products/dataxlsx/celerytasks.py b/climmob/products/dataxlsx/celerytasks.py deleted file mode 100644 index 17909ba2..00000000 --- a/climmob/products/dataxlsx/celerytasks.py +++ /dev/null @@ -1,187 +0,0 @@ -import os -import shutil as sh -import uuid -from climmob.config.celery_app import celeryApp -from climmob.plugins.utilities import climmobCeleryTask -from subprocess import Popen, PIPE -from climmob.models import get_engine -import pandas as pd -import multiprocessing - - -@celeryApp.task(base=climmobCeleryTask) -def create_XLSX( - settings, - path, - userOwner, - projectCod, - projectId, - form, - code, - finalName, - sensitive=False, -): - - num_workers = ( - multiprocessing.cpu_count() - int(settings.get("server:threads", "1")) - 1 - ) - - if num_workers <= 0: - num_workers = 1 - - pathout = os.path.join(path, "outputs") - if not os.path.exists(pathout): - os.makedirs(pathout) - - pathOfTheUser = os.path.join(settings["user.repository"], *[userOwner, projectCod]) - - UuidForTempDirectory = str(uuid.uuid4()) - - pathtmp = os.path.join(path, "tmp_" + UuidForTempDirectory) - if not os.path.exists(pathtmp): - os.makedirs(pathtmp) - - pathtmpout = os.path.join(path, "tmp_out_" + UuidForTempDirectory) - if not os.path.exists(pathtmpout): - os.makedirs(pathtmpout) - - listOfRequiredInformation = [] - listOfGeneratedXLSX = [] - - listOfRequiredInformation.append({"form": "Registration", "code": ""}) - - if form == "Report": - try: - engine = get_engine(settings) - sql = "SELECT * FROM assessment WHERE project_id='{}' AND ass_status > 0 ORDER BY ass_days".format( - projectId - ) - listOfAssessments = engine.execute(sql).fetchall() - - for assessment in listOfAssessments: - listOfRequiredInformation.append( - {"form": "Assessment", "code": assessment[0]} - ) - engine.dispose() - except Exception as e: - print("Error in the query for get the assessments") - else: - if form == "Assessment": - listOfRequiredInformation.append({"form": form, "code": code}) - - for requiredInformation in listOfRequiredInformation: - - nameOutput = requiredInformation["form"] + "_data" - if requiredInformation["code"] != "": - nameOutput += "_" + requiredInformation["code"] - - xlsx_file = os.path.join(pathtmpout, *[nameOutput + "_" + projectCod + ".xlsx"]) - - if requiredInformation["code"] != "": - pathOfTheForm = os.path.join( - pathOfTheUser, *["db", "ass", requiredInformation["code"]] - ) - mainTable = "ASS" + requiredInformation["code"] + "_geninfo" - else: - pathOfTheForm = os.path.join(pathOfTheUser, *["db", "reg"]) - mainTable = "REG_geninfo" - - create_xml = os.path.join(pathOfTheForm, *["create.xml"]) - - mysql_user = settings["odktools.mysql.user"] - mysql_password = settings["odktools.mysql.password"] - mysql_host = settings["odktools.mysql.host"] - mysql_port = settings["odktools.mysql.port"] - odk_tools_dir = settings["odktools.path"] - - paths = ["utilities", "MySQLToXLSX", "mysqltoxlsx"] - mysql_to_xlsx = os.path.join(odk_tools_dir, *paths) - - args = [ - mysql_to_xlsx, - "-H " + mysql_host, - "-P " + mysql_port, - "-u " + mysql_user, - "-p '{}'".format(mysql_password), - "-s " + userOwner + "_" + projectCod, - "-x " + create_xml, - "-o " + xlsx_file, - "-T " + pathtmp, - "-w {}".format(num_workers), - "-r {}".format(3), - ] - if sensitive: - args.append("-c") - - commandToExec = " ".join(map(str, args)) - - # p = Popen(args, stdout=PIPE, stderr=PIPE, shell=True) - p = Popen(commandToExec, stdout=PIPE, stderr=PIPE, shell=True) - stdout, stderr = p.communicate() - if p.returncode == 0: - # os.system("mv " + xlsx_file + " " + pathout) - listOfGeneratedXLSX.append(xlsx_file) - else: - print( - "MySQLToXLSX Error: " - + stderr.decode() - + "-" - + stdout.decode() - + ". Args: " - + " ".join(args) - ) - error = stdout.decode() + stderr.decode() - if error.find("Worksheet name is already in use") >= 0: - print( - "A worksheet name has been repeated. Excel only allow 30 characters in the worksheet name. " - "You can fix this by editing the dictionary and change the description of the tables " - "to a maximum of " - "30 characters." - ) - else: - print( - "Unknown error while creating the XLSX. Sorry about this. " - "Please report this error as an issue on https://github.com/mrbotcr/py3climmob" - ) - - if len(listOfGeneratedXLSX) > 0: - mergeXLSXForms(listOfGeneratedXLSX, pathout, finalName) - - sh.rmtree(pathtmpout) - if os.path.exists(pathtmp): - sh.rmtree(pathtmp) - - -def mergeXLSXForms(listOfGeneratedXLSX, pathout, finalName): - - merged_inner = pd.DataFrame() - - for index, XLSX in enumerate(listOfGeneratedXLSX): - - if index == 0: - registryDF = pd.read_excel( - XLSX, - sheet_name=0, - ) - registryDF = registryDF.add_suffix("_reg") - - merged_inner = registryDF - else: - - assessmentDF = pd.read_excel( - XLSX, - sheet_name=0, - ) - assessmentDF = assessmentDF.add_suffix("_ass_" + str(index)) - if not assessmentDF.empty: - merged_inner = pd.merge( - left=merged_inner, - right=assessmentDF, - left_on="qst162_reg", - right_on="qst163_ass_" + str(index), - how="left", - ) - - merged_inner.to_excel(finalName, sheet_name="Sheet1", index=False) - - os.system("mv " + finalName + " " + pathout) diff --git a/climmob/products/dataxlsx/dataxlsx.py b/climmob/products/dataxlsx/dataxlsx.py deleted file mode 100644 index c05ffe7f..00000000 --- a/climmob/products/dataxlsx/dataxlsx.py +++ /dev/null @@ -1,71 +0,0 @@ -import climmob.plugins as p -from climmob.processes import ( - registryHaveQuestionOfMultimediaType, - assessmentHaveQuestionOfMultimediaType, -) -from climmob.products.dataxlsx.celerytasks import create_XLSX -from climmob.products.climmob_products import ( - createProductDirectory, - registerProductInstance, -) - - -def create_XLSXToDownload(userOwner, projectId, projectCod, request, form, code): - # We create the plugin directory if it does not exists and return it - # The path user.repository in development.ini/user/project/products/product and - # user.repository in development.ini/user/project/products/product/outputs - settings = {} - for key, value in request.registry.settings.items(): - if isinstance(value, str): - settings[key] = value - - path = createProductDirectory(request, userOwner, projectCod, "dataxlsx") - # We call the Celery task that will generate the output packages.pdf - nameOutput = form + "_data" - if code != "": - nameOutput += "_" + code - - task = create_XLSX.apply_async( - ( - settings, - path, - userOwner, - projectCod, - projectId, - form, - code, - nameOutput + "_" + projectCod + ".xlsx", - ), - queue="ClimMob", - ) - # We register the instance of the output with the task ID of celery - # This will go to the products table that then you can monitor and use - # in the nice product interface - # u.registerProductInstance(user, project, 'cards', 'cards.pdf', task.id, request) - - registerProductInstance( - projectId, - "dataxlsx", - nameOutput + "_" + projectCod + ".xlsx", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "create_data_xlsx_" + form + "_" + code, - task.id, - request, - ) - - for plugin in p.PluginImplementations(p.IMultimedia): - thereAreMultimedia = False - if form == "Registration": - thereAreMultimedia = registryHaveQuestionOfMultimediaType( - request, projectId - ) - - if form == "Assessment": - thereAreMultimedia = assessmentHaveQuestionOfMultimediaType( - request, projectId, code - ) - - if thereAreMultimedia: - plugin.start_multimedia_download( - request, userOwner, projectId, projectCod, form, code - ) diff --git a/climmob/views/editData.py b/climmob/views/editData.py index 71c8cc42..ed7ee8b1 100755 --- a/climmob/views/editData.py +++ b/climmob/views/editData.py @@ -12,8 +12,7 @@ projectExists, getJSONResult, ) -from climmob.products.analysisdata.analysisdata import create_datacsv -from climmob.products.dataxlsx.dataxlsx import create_XLSXToDownload +from climmob.products.analysisdata.analysisdata import create_raw_data from climmob.products.errorLogDocument.errorLogDocument import create_error_log_document from climmob.views.classes import privateView from climmob.views.editDataDB import ( @@ -57,9 +56,6 @@ def processView(self): else: raise HTTPNotFound() - if anonymize: - formId += "_anonymized" - info = getJSONResult( activeProjectUser, activeProjectId, @@ -74,31 +70,23 @@ def processView(self): if formatId not in ["csv", "xlsx"]: raise HTTPNotFound() - if formatId == "csv": - create_datacsv( - activeProjectUser, - activeProjectId, - activeProjectCod, - info, - self.request, - formId, - code, - ) + create_raw_data( + activeProjectUser, + activeProjectId, + activeProjectCod, + info["data"], + self.request, + formId, + code, + file_type=formatId, + anonymized=anonymize + ) - if formatId == "xlsx": - formatExtra = formatId + "_" - create_XLSXToDownload( - activeProjectUser, - activeProjectId, - activeProjectCod, - self.request, - formId, - code, - ) + extra = "-anonymized" if anonymize else "" url = self.request.route_url( "productList", - _query={"product1": "create_data_" + formatExtra + formId + "_" + code}, + _query={"product1": f"create_data{extra}_{'xlsx_' if formatId == 'xlsx' else ''}" + formatExtra + formId + "_" + code}, ) self.returnRawViewResult = True return HTTPFound(location=url) diff --git a/climmob/views/productsList.py b/climmob/views/productsList.py index 002c0d31..b3a4434a 100644 --- a/climmob/views/productsList.py +++ b/climmob/views/productsList.py @@ -31,8 +31,7 @@ getPrjLangDefaultInProject, ) from climmob.products import product_found -from climmob.products.analysisdata.analysisdata import create_datacsv -from climmob.products.dataxlsx.dataxlsx import create_XLSXToDownload +from climmob.products.analysisdata.analysisdata import create_raw_data from climmob.products.colors.colors import create_colors_cards from climmob.products.errorLogDocument.errorLogDocument import create_error_log_document from climmob.products.fieldagents.fieldagents import create_fieldagents_report @@ -282,9 +281,13 @@ def processView(self): listOfLabels, ) - if productid == "datacsv": - locale = self.request.locale_name + if productid in ["datacsv", "datacsv-anonymized", "dataxlsx", "dataxlsx-anonymized"]: + anonymized = productid in ["datacsv-anonymized", "dataxlsx-anonymized"] + file_type = "csv" if "csv" in productid else "xlsx" infoProduct = processname.split("_") + if file_type == "xlsx": + infoProduct[2] = infoProduct[3] + infoProduct[3] = infoProduct[4] if infoProduct[2] == "Registration": info = getJSONResult( activeProjectData["owner"]["user_name"], @@ -292,6 +295,7 @@ def processView(self): activeProjectData["project_cod"], self.request, includeAssessment=False, + anonymize=anonymized ) else: if infoProduct[2] == "Assessment": @@ -301,6 +305,7 @@ def processView(self): activeProjectData["project_cod"], self.request, assessmentCode=infoProduct[3], + anonymize=anonymized ) else: info = getJSONResult( @@ -308,27 +313,19 @@ def processView(self): activeProjectData["project_id"], activeProjectData["project_cod"], self.request, + anonymize=anonymized ) - create_datacsv( + create_raw_data( activeProjectData["owner"]["user_name"], activeProjectData["project_id"], activeProjectData["project_cod"], - info, + info["data"], self.request, infoProduct[2], infoProduct[3], - ) - - if productid == "dataxlsx": - infoProduct = processname.split("_") - create_XLSXToDownload( - activeProjectData["owner"]["user_name"], - activeProjectData["project_id"], - activeProjectData["project_cod"], - self.request, - infoProduct[3], - infoProduct[4], + file_type=file_type, + anonymized=anonymized ) if productid == "documentform": diff --git a/climmob/views/project_analysis.py b/climmob/views/project_analysis.py index dfd2034f..522c2edd 100644 --- a/climmob/views/project_analysis.py +++ b/climmob/views/project_analysis.py @@ -8,7 +8,7 @@ getProjectProgress, ) from climmob.products.analysis.analysis import create_analysis -from climmob.products.analysisdata.analysisdata import create_datacsv +from climmob.products.analysisdata.analysisdata import create_raw_data from climmob.views.classes import privateView @@ -155,7 +155,7 @@ def processToGenerateTheReport( combinationRerence, ) - create_datacsv( + create_raw_data( activeProjectData["owner"]["user_name"], activeProjectData["project_id"], activeProjectData["project_cod"], From e8a2acd88220e316ba80a2eab222dcfbafdafb9e Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 18 Jul 2025 16:46:02 -0600 Subject: [PATCH 07/66] add anonymized Report buttons --- climmob/products/analysisdata/analysisdata.py | 45 ++++++---- climmob/products/analysisdata/celerytasks.py | 1 - climmob/products/climmob_products.py | 9 +- .../project/productsList/productsList.jinja2 | 84 +++++++++++++++---- climmob/views/editData.py | 10 ++- climmob/views/productsList.py | 21 +++-- 6 files changed, 126 insertions(+), 44 deletions(-) diff --git a/climmob/products/analysisdata/analysisdata.py b/climmob/products/analysisdata/analysisdata.py index 684a2a59..5ef460fd 100644 --- a/climmob/products/analysisdata/analysisdata.py +++ b/climmob/products/analysisdata/analysisdata.py @@ -10,7 +10,17 @@ ) -def create_raw_data(userOwner, projectId, projectCod, info, request, form, code, file_type="csv", anonymized=False): +def create_raw_data( + user_owner, + project_id, + project_cod, + info, + request, + form, + code, + file_type="csv", + anonymized=False, +): # We create the plugin directory if it does not exists and return it extra = "-anonymized" if anonymized else "" @@ -18,11 +28,15 @@ def create_raw_data(userOwner, projectId, projectCod, info, request, form, code, if code != "": name_output += "_" + code - name_output += "_" + projectCod + name_output += "_" + project_cod - path = createProductDirectory(request, userOwner, projectCod, f"data{file_type}{extra}") + path = createProductDirectory( + request, user_owner, project_cod, f"data{file_type}{extra}" + ) # We call the Celery task that will generate the output packages.pdf - task = create_raw_data_file.apply_async((path, info, name_output, file_type), queue="ClimMob") + task = create_raw_data_file.apply_async( + (path, info, name_output, file_type), queue="ClimMob" + ) # We register the instance of the output with the task ID of celery # This will go to the products table that then you can monitor and use # in the nice product interface @@ -30,14 +44,17 @@ def create_raw_data(userOwner, projectId, projectCod, info, request, form, code, mimetypes = { "csv": "text/csv", - "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", } mimetype = mimetypes.get(file_type) - process_name = f"create_data{extra}_{'xlsx_' if file_type == 'xlsx' else ''}" + form + "_" + code + process_name = ( + f"create_data{extra}_" + f"{'xlsx_' if file_type == 'xlsx' else ''}" + form + "_" + code + ) registerProductInstance( - projectId, + project_id, f"data{file_type}{extra}", name_output + f".{file_type}", mimetype, @@ -47,18 +64,18 @@ def create_raw_data(userOwner, projectId, projectCod, info, request, form, code, ) for plugin in p.PluginImplementations(p.IMultimedia): - thereAreMultimedia = False + there_are_multimedia = False if form == "Registration": - thereAreMultimedia = registryHaveQuestionOfMultimediaType( - request, projectId + there_are_multimedia = registryHaveQuestionOfMultimediaType( + request, project_id ) if form == "Assessment": - thereAreMultimedia = assessmentHaveQuestionOfMultimediaType( - request, projectId, code + there_are_multimedia = assessmentHaveQuestionOfMultimediaType( + request, project_id, code ) - if thereAreMultimedia: + if there_are_multimedia: plugin.start_multimedia_download( - request, userOwner, projectId, projectCod, form, code + request, user_owner, project_id, project_cod, form, code ) diff --git a/climmob/products/analysisdata/celerytasks.py b/climmob/products/analysisdata/celerytasks.py index df0ef2f0..80f99c12 100644 --- a/climmob/products/analysisdata/celerytasks.py +++ b/climmob/products/analysisdata/celerytasks.py @@ -23,7 +23,6 @@ def create_raw_data_file(path, info, name_output, file_type): elif file_type == "csv": df.to_csv(os.path.join(path_out, name_output) + f".{file_type}", index=False) - # if os.path.exists(path): # sh.rmtree(path) diff --git a/climmob/products/climmob_products.py b/climmob/products/climmob_products.py index 2d608088..52c04126 100644 --- a/climmob/products/climmob_products.py +++ b/climmob/products/climmob_products.py @@ -228,7 +228,9 @@ def register_products(config): ) products.append(datacsv) - datacsv_anonymized = addProduct("datacsv-anonymized", "Information collected in the project anonymized.") + datacsv_anonymized = addProduct( + "datacsv-anonymized", "Information collected in the project anonymized." + ) addMetadataToProduct(datacsv_anonymized, "author", "Johann Ávalos") addMetadataToProduct(datacsv_anonymized, "version", "1.0") addMetadataToProduct( @@ -372,7 +374,10 @@ def register_products(config): ) products.append(dataxlsx) - dataxlsx_anonymized = addProduct("dataxlsx-anonymized", "Information collected in the project anonymized in XLSX format.") + dataxlsx_anonymized = addProduct( + "dataxlsx-anonymized", + "Information collected in the project anonymized in XLSX format.", + ) addMetadataToProduct(dataxlsx_anonymized, "author", "Johann Ávalos") addMetadataToProduct(dataxlsx_anonymized, "version", "1.0") addMetadataToProduct( diff --git a/climmob/templates/snippets/project/productsList/productsList.jinja2 b/climmob/templates/snippets/project/productsList/productsList.jinja2 index 8df6985c..991526ff 100644 --- a/climmob/templates/snippets/project/productsList/productsList.jinja2 +++ b/climmob/templates/snippets/project/productsList/productsList.jinja2 @@ -29,11 +29,22 @@ {% set productToFocus = "xx" %} - {% set changes = {'projectSummary': true, 'DataCollectionProgress': false, 'QRPackagesEditable':false,'projectDataCollectedCSV':false,'projectDataCollectedXLSX': false, 'productToFocus':""} %} + {% set changes = { + 'projectSummary': true, + 'DataCollectionProgress': false, + 'QRPackagesEditable':false, + 'projectDataCollectedCSV':false, + 'projectDataCollectedXLSX': false, + 'projectDataCollectedCSV-anonymized':false, + 'projectDataCollectedXLSX-anonymized': false, + 'productToFocus':"" + } %} {% if activeProject.project_regstatus > 0 %} {% if changes.update({'DataCollectionProgress': true }) %} {% endif %} {% if changes.update({'projectDataCollectedCSV': true }) %} {% endif %} {% if changes.update({'projectDataCollectedXLSX': true }) %} {% endif %} + {% if changes.update({'projectDataCollectedCSV-anonymized': true }) %} {% endif %} + {% if changes.update({'projectDataCollectedXLSX-anonymized': true }) %} {% endif %} {% endif %} {% block qr_packages_editable_variable scoped%} @@ -54,6 +65,27 @@ {% if product.product_id == "qrpackage" %} {{ _("List of packages with QR for the participant registration form") }} + + {% elif product.product_id == "datacsv-anonymized" %} + {% if product.process_name == "create_data-anonymized_Report_" %} + {% if changes.update({'projectDataCollectedCSV-anonymized': false }) %}{% endif %} + {{ _("Information collected in all the project in .CSV format (anonymized)") }} + {% elif product.process_name == "create_data-anonymized_Registration_" %} + {{ _("Information collected in the participant registration form in .CSV format (anonymized)") }} + {% else %} + {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .CSV format (anonymized)") }} + {% endif %} + + {% elif product.product_id == "dataxlsx-anonymized" %} + {% if product.process_name == "create_data-anonymized_xlsx_Report_" %} + {% if changes.update({'projectDataCollectedXLSX-anonymized': false }) %}{% endif %} + {{ _("Information collected in all the project in .XLSX format (anonymized)") }} + {% elif product.process_name == "create_data-anonymized_xlsx_Registration_" %} + {{ _("Information collected in the participant registration form in .XLSX format (anonymized)") }} + {% else %} + {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .XLSX format (anonymized)") }} + {% endif %} + {% else %} {% if product.product_id == "packages" %} {{ _("List with randomized trial packages") }} @@ -65,23 +97,15 @@ {{ _("Color cards to explain ClimMob") }} {% else %} {% if product.product_id == "datacsv" %} - {% if product.process_name == "create_data_Report_" %} {% if changes.update({'projectDataCollectedCSV': false }) %}{% endif %} {{ _("Information collected in all the project in .CSV format") }} + {% elif product.process_name == "create_data_Registration_" %} + {{ _("Information collected in the participant registration form in .CSV format") }} {% else %} - {% if product.process_name == "create_data_Registration_" %} - {{ _("Information collected in the participant registration form in .CSV format") }} - {% elif product.process_name == "create_data_Registration_anonymized_" %} - {{ _("Information collected in the participant registration form in .CSV format (anonymized)") }} - {% elif "anonymized" in product.process_name %} - {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .CSV format (anonymized)") }} - {% else %} - {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .CSV format") }} - {% endif %} + {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .CSV format") }} {% endif %} - {% else %} {% if product.product_id == "reports" %} {{ _("Analysis report based on trial data") }} @@ -141,10 +165,6 @@ {% else %} {% if product.process_name == "create_data_xlsx_Registration_" %} {{ _("Information collected in the participant registration form in .XLSX format") }} - {% elif product.process_name == "create_data_xlsx_Registration_anonymized_" %} - {{ _("Information collected in the participant registration form in .XLSX format (anonymized)") }} - {% elif "anonymized" in product.process_name %} - {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .XLSX format (anonymized)") }} {% else %} {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .XLSX format") }} {% endif %} @@ -278,10 +298,18 @@ {{ _("Update") }} {% endif %} + {% if product.product_id == "datacsv-anonymized" and product.process_name == "create_data-anonymized_Report_" %} + {{ _("Update") }} + {% endif %} + {% if product.product_id == "dataxlsx" and product.process_name == "create_data_xlsx_Report_" %} {{ _("Update") }} {% endif %} + {% if product.product_id == "dataxlsx-anonymized" and product.process_name == "create_data-anonymized_xlsx_Report_" %} + {{ _("Update") }} + {% endif %} + {% if product.product_id == "generalreport" %} {{ _("Update") }} {% endif %} @@ -332,6 +360,30 @@ {% endif %} + {% if changes["projectDataCollectedXLSX-anonymized"] %} + + {{ _("Information collected in all the project in .XLSX format (anonymized)") }} + + + + {{ _("Create") }} + + + + {% endif %} + + {% if changes["projectDataCollectedCSV-anonymized"] %} + + {{ _("Information collected in all the project in .CSV format (anonymized)") }} + + + + {{ _("Create") }} + + + + {% endif %} + {% if changes.projectSummary %} {{ _("Project summary") }} diff --git a/climmob/views/editData.py b/climmob/views/editData.py index ed7ee8b1..8399f69f 100755 --- a/climmob/views/editData.py +++ b/climmob/views/editData.py @@ -33,7 +33,6 @@ def processView(self): includeRegistry = True includeAssessment = True code = "" - formatExtra = "" if not projectExists( self.user.login, activeProjectUser, activeProjectCod, self.request @@ -79,14 +78,17 @@ def processView(self): formId, code, file_type=formatId, - anonymized=anonymize + anonymized=anonymize, ) - extra = "-anonymized" if anonymize else "" + format_extra = "xlsx_" if formatId == "xlsx" else "" + product_id_extra = "-anonymized" if anonymize else "" url = self.request.route_url( "productList", - _query={"product1": f"create_data{extra}_{'xlsx_' if formatId == 'xlsx' else ''}" + formatExtra + formId + "_" + code}, + _query={ + "product1": f"create_data{product_id_extra}_{format_extra}{formId}_{code}" + }, ) self.returnRawViewResult = True return HTTPFound(location=url) diff --git a/climmob/views/productsList.py b/climmob/views/productsList.py index b3a4434a..79f80969 100644 --- a/climmob/views/productsList.py +++ b/climmob/views/productsList.py @@ -107,16 +107,18 @@ def processView(self): if product["product_id"] in [ "documentform", "datacsv", + "dataxlsx", + "datacsv-anonymized", + "dataxlsx-anonymized", "errorlogdocument", "multimediadownloads", "uploaddata", - "dataxlsx", "observationcards", "climmobexplanationkit", ]: product["extraInformation"] = None pattern = re.compile( - r".+?(?:(?:Assessment))_(?:anonymized_)?" # not captured + r".+?(?:(?:Assessment))_" # not captured r"([a-f0-9]{12})" # captured (group 1) ) match = pattern.fullmatch(product["process_name"]) @@ -281,7 +283,12 @@ def processView(self): listOfLabels, ) - if productid in ["datacsv", "datacsv-anonymized", "dataxlsx", "dataxlsx-anonymized"]: + if productid in [ + "datacsv", + "datacsv-anonymized", + "dataxlsx", + "dataxlsx-anonymized", + ]: anonymized = productid in ["datacsv-anonymized", "dataxlsx-anonymized"] file_type = "csv" if "csv" in productid else "xlsx" infoProduct = processname.split("_") @@ -295,7 +302,7 @@ def processView(self): activeProjectData["project_cod"], self.request, includeAssessment=False, - anonymize=anonymized + anonymize=anonymized, ) else: if infoProduct[2] == "Assessment": @@ -305,7 +312,7 @@ def processView(self): activeProjectData["project_cod"], self.request, assessmentCode=infoProduct[3], - anonymize=anonymized + anonymize=anonymized, ) else: info = getJSONResult( @@ -313,7 +320,7 @@ def processView(self): activeProjectData["project_id"], activeProjectData["project_cod"], self.request, - anonymize=anonymized + anonymize=anonymized, ) create_raw_data( @@ -325,7 +332,7 @@ def processView(self): infoProduct[2], infoProduct[3], file_type=file_type, - anonymized=anonymized + anonymized=anonymized, ) if productid == "documentform": From 00a5042911c17a5c5c693877d91c92310bceeb8b Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 21 Jul 2025 11:09:25 -0600 Subject: [PATCH 08/66] replace options value with their label --- climmob/products/analysisdata/celerytasks.py | 86 ++++++++++-------- climmob/products/analysisdata/exportToCsv.py | 89 ------------------- .../products/errorLogDocument/celerytasks.py | 4 +- climmob/views/editData.py | 2 +- climmob/views/productsList.py | 2 +- 5 files changed, 55 insertions(+), 128 deletions(-) delete mode 100644 climmob/products/analysisdata/exportToCsv.py diff --git a/climmob/products/analysisdata/celerytasks.py b/climmob/products/analysisdata/celerytasks.py index 80f99c12..dcdd80e6 100644 --- a/climmob/products/analysisdata/celerytasks.py +++ b/climmob/products/analysisdata/celerytasks.py @@ -4,7 +4,6 @@ from climmob.config.celery_app import celeryApp from climmob.plugins.utilities import climmobCeleryTask -from climmob.products.analysisdata.exportToCsv import createCSV @celeryApp.task(base=climmobCeleryTask) @@ -15,43 +14,60 @@ def create_raw_data_file(path, info, name_output, file_type): os.makedirs(path) os.makedirs(path_out) - # TODO replace labels + replace_options_with_labels(info) - df = pd.DataFrame(info) + df = pd.DataFrame(info["data"]) if file_type == "xlsx": df.to_excel(os.path.join(path_out, name_output) + f".{file_type}", index=False) elif file_type == "csv": df.to_csv(os.path.join(path_out, name_output) + f".{file_type}", index=False) - # if os.path.exists(path): - # sh.rmtree(path) - - # nameOutput = form + "_data" - # if code != "": - # nameOutput += "_" + code - # - # pathout = os.path.join(path, "outputs") - # if not os.path.exists(path): - # os.makedirs(path) - # os.makedirs(pathout) - # - # if os.path.exists(pathout + "/" + nameOutput + "_" + projectCod + ".csv"): - # os.remove(pathout + "/" + nameOutput + "_" + projectCod + ".csv") - # - # pathInputFiles = os.path.join(path, "inputFile") - # os.makedirs(pathInputFiles) - # - # with open(pathInputFiles + "/info.json", "w") as outfile: - # jsonString = json.dumps(info, indent=4, ensure_ascii=False) - # outfile.write(jsonString) - # - # if os.path.exists(pathInputFiles + "/info.json"): - # try: - # createCSV( - # pathout + "/" + nameOutput + "_" + projectCod + ".csv", - # pathInputFiles + "/info.json", - # ) - # except Exception as e: - # print("We can't create the CSV." + str(e)) - # - # sh.rmtree(pathInputFiles) + +def replace_options_with_labels(data): + for row in data["data"]: + for field in data["registry"]["fields"]: + if field["rtable"] is not None and row["REG_" + field["name"]] is not None: + result = get_option_label( + data["registry"]["lkptables"], + field["rtable"], + field["rfield"], + row["REG_" + field["name"]], + field["isMultiSelect"], + ) + row["REG_" + field["name"]] = result + + for assessment in data["assessments"]: + for field in assessment["fields"]: + if ( + field["rtable"] is not None + and row["ASS" + assessment["code"] + "_" + field["name"]] + is not None + ): + result = get_option_label( + assessment["lkptables"], + field["rtable"], + field["rfield"], + row["ASS" + assessment["code"] + "_" + field["name"]], + field["isMultiSelect"], + ) + row["ASS" + assessment["code"] + "_" + field["name"]] = result + + +def get_option_label(lkptables, rtable, rfield, value, isMultiSelect): + res = None + for lkp in lkptables: + if lkp["name"] == rtable: + for data in lkp["values"]: + if isMultiSelect == "true": + for valueSplit in value.split(" "): + if str(data[rfield]) == str(valueSplit): + if res == None: + res = data[rfield[:-3] + "des"] + else: + res += " - " + data[rfield[:-3] + "des"] + else: + if data[rfield] == value: + res = data[rfield[:-3] + "des"] + break + + return res diff --git a/climmob/products/analysisdata/exportToCsv.py b/climmob/products/analysisdata/exportToCsv.py deleted file mode 100644 index af7a9d81..00000000 --- a/climmob/products/analysisdata/exportToCsv.py +++ /dev/null @@ -1,89 +0,0 @@ -import csv -import json - - -def getRealData(lkptables, rtable, rfield, value, isMultiSelect): - res = None - for lkp in lkptables: - if lkp["name"] == rtable: - for data in lkp["values"]: - if isMultiSelect == "true": - for valueSplit in value.split(" "): - if str(data[rfield]) == str(valueSplit): - if res == None: - res = data[rfield[:-3] + "des"] - else: - res += " - " + data[rfield[:-3] + "des"] - else: - if data[rfield] == value: - res = data[rfield[:-3] + "des"] - break - - return res - - -def createCSV(outputPath, inputFile): - myFile = open(outputPath, "w") - with myFile: - writer = csv.writer(myFile) - - with open(inputFile) as json_file: - data = json.load(json_file) - # SACO LAS COLUMNAS DEL REGISTRY - columns = [] - for field in data["registry"]["fields"]: - columns.append(field["desc"].replace(",", "")) - - # SACO LAS COLUMNAS DE LOS ASSESSMENTS - for assessment in data["assessments"]: - for field in assessment["fields"]: - columns.append(field["desc"].replace(",", "")) - - # ESCRIBO LAS COLUMNAS - writer.writerow(columns) - - # EMPIEZO A SACAR LOS DATOS - for row in data["data"]: - fieldsDataRow = [] - # DATOS DEL REGISTRO - for field in data["registry"]["fields"]: - # print(field) - if field["rtable"] != None and row["REG_" + field["name"]] != None: - result = getRealData( - data["registry"]["lkptables"], - field["rtable"], - field["rfield"], - row["REG_" + field["name"]], - field["isMultiSelect"], - ) - fieldsDataRow.append(str(result).replace(",", "")) - else: - fieldsDataRow.append( - str(row["REG_" + field["name"]]).replace(",", "") - ) - - for assessment in data["assessments"]: - for field in assessment["fields"]: - if ( - field["rtable"] != None - and row["ASS" + assessment["code"] + "_" + field["name"]] - != None - ): - result = getRealData( - assessment["lkptables"], - field["rtable"], - field["rfield"], - row["ASS" + assessment["code"] + "_" + field["name"]], - field["isMultiSelect"], - ) - fieldsDataRow.append(str(result).replace(",", "")) - else: - fieldsDataRow.append( - str( - row[ - "ASS" + assessment["code"] + "_" + field["name"] - ] - ).replace(",", "") - ) - - writer.writerow(fieldsDataRow) diff --git a/climmob/products/errorLogDocument/celerytasks.py b/climmob/products/errorLogDocument/celerytasks.py index bc701b28..6ed0927d 100644 --- a/climmob/products/errorLogDocument/celerytasks.py +++ b/climmob/products/errorLogDocument/celerytasks.py @@ -6,7 +6,7 @@ from climmob.config.celery_app import celeryApp from climmob.config.celery_class import celeryTask -from climmob.products.analysisdata.exportToCsv import getRealData +from climmob.products.analysisdata.celerytasks import get_option_label @celeryApp.task(base=celeryTask, soft_time_limit=7200, time_limit=7200) @@ -134,7 +134,7 @@ def createErrorLogDocument( and row[firstPart + field["name"]] != None and field["name"] != "qst163" ): - result = getRealData( + result = get_option_label( lkps, field["rtable"], field["rfield"], diff --git a/climmob/views/editData.py b/climmob/views/editData.py index 8399f69f..5f417a41 100755 --- a/climmob/views/editData.py +++ b/climmob/views/editData.py @@ -73,7 +73,7 @@ def processView(self): activeProjectUser, activeProjectId, activeProjectCod, - info["data"], + info, self.request, formId, code, diff --git a/climmob/views/productsList.py b/climmob/views/productsList.py index 79f80969..16fb1a33 100644 --- a/climmob/views/productsList.py +++ b/climmob/views/productsList.py @@ -327,7 +327,7 @@ def processView(self): activeProjectData["owner"]["user_name"], activeProjectData["project_id"], activeProjectData["project_cod"], - info["data"], + info, self.request, infoProduct[2], infoProduct[3], From de456835bf1ec77422edf9c9be089acb7efb5a5f Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Tue, 22 Jul 2025 12:36:10 -0600 Subject: [PATCH 09/66] create anonymized table on registry start --- climmob/processes/db/results.py | 2 +- climmob/processes/odk/generator.py | 159 ++++++++++++++++------------- 2 files changed, 89 insertions(+), 72 deletions(-) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 30eb88a2..8a3e3952 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -377,7 +377,7 @@ def getData( + userOwner + "_" + projectCod - + ".anony da" + + ".anonymized da" + f" ON da.reg_id = {reg_alias}.qst162 " + f" GROUP BY {reg_alias}.qst162" ) diff --git a/climmob/processes/odk/generator.py b/climmob/processes/odk/generator.py index 49de1ac2..a8db29c0 100644 --- a/climmob/processes/odk/generator.py +++ b/climmob/processes/odk/generator.py @@ -40,87 +40,104 @@ ] +def execute_command(args, error_msg): + error = False + try: + check_call(args) + except CalledProcessError as e: + msg = f"{error_msg}\nError:\n{str(e)}\n" + log.error(msg) + print(msg) + error = True + return error + + +def create_schema(schema, cnf_file): + print(f"****buildDatabase**Dropping schema {schema}******") + args = [ + "mysql", + f"--defaults-file={cnf_file}", + f"--execute=DROP SCHEMA IF EXISTS {schema}", + ] + error = execute_command(args, "Error dropping schema") + if error: + return error + + print(f"****buildDatabase**Creating new schema {schema}******") + args = [ + "mysql", + f"--defaults-file={cnf_file}", + f"--execute=CREATE SCHEMA {schema}" + " DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci", + ] + error = execute_command(args, "Error creating schema") + if error: + return error + + args = [ + "mysql", + f"--defaults-file={cnf_file}", + f"--execute=CREATE TABLE IF NOT EXISTS {schema}.anonymized " + "(`form_id` varchar(255) NOT NULL," + "`reg_id` int NOT NULL," + "`col_name` varchar(255) NOT NULL," + "`value` varchar(255) DEFAULT NULL," + "PRIMARY KEY (`form_id`,`reg_id`,`col_name`)" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8;", + ] + + error = execute_command(args, "Error creating anonymized table") + return error + + def buildDatabase( cnfFile, createFile, insertFile, schema, dropSchema, settings, outputDir ): error = False if dropSchema: - print("****buildDatabase**Dropping schema******") - args = [] - args.append("mysql") - args.append("--defaults-file=" + cnfFile) - args.append("--execute=DROP SCHEMA IF EXISTS " + schema) - try: - check_call(args) - except CalledProcessError as e: - msg = "Error dropping schema \n" + error = create_schema(schema, cnfFile) + if error: + return error + + print(f"****buildDatabase**Creating tables {schema}******") + args = ["mysql", f"--defaults-file={cnfFile}", schema] + with open(createFile) as input_file: + proc = Popen(args, stdin=input_file, stderr=PIPE, stdout=PIPE) + output, error = proc.communicate() + # if output != "" or error != "": + if proc.returncode != 0: + # print("3") + msg = "Error creating database \n" + msg = msg + "File: " + createFile + "\n" msg = msg + "Error: \n" - msg = msg + str(e) + "\n" + msg = msg + str(error) + "\n" + msg = msg + "Output: \n" + msg = msg + str(output) + "\n" log.error(msg) - print(msg) error = True + if error: + return error + + print(f"****buildDatabase**Inserting into lookup tables for {schema}******") + with open(insertFile) as input_file: + proc = Popen(args, stdin=input_file, stderr=PIPE, stdout=PIPE) + output, error = proc.communicate() + # if output != "" or error != "": + if proc.returncode != 0: + msg = "Error loading lookup tables \n" + msg = msg + "File: " + createFile + "\n" + msg = msg + "Error: \n" + msg = msg + str(error) + "\n" + msg = msg + "Output: \n" + msg = msg + str(output) + "\n" + log.error(msg) + error = True + if error: + return error - if not error: - print("****buildDatabase**Creating new schema******") - args = [] - args.append("mysql") - args.append("--defaults-file=" + cnfFile) - args.append( - "--execute=CREATE SCHEMA " - + schema - + " DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci" - ) - try: - check_call(args) - except CalledProcessError as e: - msg = "Error dropping schema \n" - msg = msg + "Error: \n" - msg = msg + str(e) + "\n" - log.error(msg) - error = True - - if not error: - print("****buildDatabase**Creating tables******") - args = [] - args.append("mysql") - args.append("--defaults-file=" + cnfFile) - args.append(schema) - - with open(createFile) as input_file: - proc = Popen(args, stdin=input_file, stderr=PIPE, stdout=PIPE) - output, error = proc.communicate() - # if output != "" or error != "": - if proc.returncode != 0: - # print("3") - msg = "Error creating database \n" - msg = msg + "File: " + createFile + "\n" - msg = msg + "Error: \n" - msg = msg + str(error) + "\n" - msg = msg + "Output: \n" - msg = msg + str(output) + "\n" - log.error(msg) - error = True - - if not error: - print("****buildDatabase**Inserting into lookup tables******") - with open(insertFile) as input_file: - proc = Popen(args, stdin=input_file, stderr=PIPE, stdout=PIPE) - output, error = proc.communicate() - # if output != "" or error != "": - if proc.returncode != 0: - msg = "Error loading lookup tables \n" - msg = msg + "File: " + createFile + "\n" - msg = msg + "Error: \n" - msg = msg + str(error) + "\n" - msg = msg + "Output: \n" - msg = msg + str(output) + "\n" - log.error(msg) - error = True - - if not error: - print("****buildDatabase**Creating triggers******") - functionForCreateTheTriggers(schema, settings, outputDir, cnfFile) + print(f"****buildDatabase**Creating triggers for {schema}******") + functionForCreateTheTriggers(schema, settings, outputDir, cnfFile) return error From 6a9db0863875fa6413e9d3dd657cebc6b9f5ded3 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 23 Jul 2025 10:16:30 -0600 Subject: [PATCH 10/66] anonymized farmer name options --- climmob/processes/db/results.py | 10 +++++++--- climmob/views/editData.py | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 8a3e3952..2df643e1 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -56,7 +56,7 @@ def getFields(XMLFile, table): return fields -def getLookups(XMLFile, userOwner, projectCod, request): +def getLookups(XMLFile, userOwner, projectCod, anonymize): lktables = [] tree = etree.parse(XMLFile) root = tree.getroot() @@ -95,6 +95,10 @@ def getLookups(XMLFile, userOwner, projectCod, request): avalue = {} for field in atable["fields"]: avalue[field["name"]] = value[field["name"]] + if anonymize and atable["name"].endswith("lkpqst163_opts"): + avalue[ + "qst163_opts_des" + ] = f'Farmer #{avalue["qst163_opts_cod"]}' atable["values"].append(avalue) lktables.append(atable) return lktables @@ -608,7 +612,7 @@ def getJSONResult( if os.path.exists(registryXML): data["registry"] = { "lkptables": getLookups( - registryXML, userOwner, projectCod, request + registryXML, userOwner, projectCod, anonymize ), "fields": getFields(registryXML, "REG_geninfo"), } @@ -651,7 +655,7 @@ def getJSONResult( assessmentXML, userOwner, projectCod, - request, + anonymize, ), "fields": getFields( assessmentXML, diff --git a/climmob/views/editData.py b/climmob/views/editData.py index 5f417a41..a8a90c5f 100755 --- a/climmob/views/editData.py +++ b/climmob/views/editData.py @@ -29,7 +29,7 @@ def processView(self): activeProjectCod = self.request.matchdict["project"] formId = self.request.matchdict["formid"] formatId = self.request.matchdict["formatid"] - anonymize = bool(self.request.params.get("anonymize")) + anonymize = str(self.request.params.get("anonymize")).lower() == "true" includeRegistry = True includeAssessment = True code = "" From e6eda409e85bca8ef8ec89de40cd10fd5fab322d Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 28 Jul 2025 09:55:55 -0600 Subject: [PATCH 11/66] add api route for getting results --- climmob/config/routes.py | 10 ++++++++++ climmob/views/results.py | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 climmob/views/results.py diff --git a/climmob/config/routes.py b/climmob/config/routes.py index cb06dd1d..5b77f7e2 100644 --- a/climmob/config/routes.py +++ b/climmob/config/routes.py @@ -245,6 +245,7 @@ GetRegistrySectionView, ChangeProjectMainLanguage_view, ) +from climmob.views.results import ResultsView from climmob.views.techaliases import deletealias_view from climmob.views.technologies import ( technologies_view, @@ -2165,6 +2166,15 @@ def loadRoutes(config): ) ) + routes.append( + addRoute( + "results", + "/api/results", + ResultsView, + "json", + ) + ) + # --------------------------------------------------------ClimMob Bot--------------------------------------------------------# # Chat diff --git a/climmob/views/results.py b/climmob/views/results.py new file mode 100644 index 00000000..1d75005c --- /dev/null +++ b/climmob/views/results.py @@ -0,0 +1,17 @@ +from climmob.processes import getJSONResult, getProjectData +from climmob.views.classes import apiView + + +class ResultsView(apiView): + def get(self): + active_project_data = getProjectData( + self.context.active_project_id, self.request + ) + anonymize = str(self.request.params.get("anonymize")).lower() == "true" + return getJSONResult( + self.user.login, + self.context.active_project_id, + active_project_data["project_cod"], + self.request, + anonymize=anonymize, + ) From f7f7e41c9e752a7ab5c2576d5b37907074690cdc Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 28 Jul 2025 11:57:19 -0600 Subject: [PATCH 12/66] add question anonymity and type to database model --- climmob/models/climmobv4.py | 20 +++++++++++++++++++- climmob/utility/question.py | 17 ++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/climmob/models/climmobv4.py b/climmob/models/climmobv4.py index edd4ddba..0662fcc9 100644 --- a/climmob/models/climmobv4.py +++ b/climmob/models/climmobv4.py @@ -787,6 +787,16 @@ class Question_subgroup(Base): parent_id = Column(Unicode(80), primary_key=True, nullable=True) +class QuestionType(Base): + __tablename__ = "question_type" + + id = Column(Integer, primary_key=True, nullable=False) + name = Column(Unicode(64), nullable=False) + anonymity_id = Column( + Integer, ForeignKey("question_anonymity.id"), primary_key=True, nullable=False + ) + + class Question(Base): __tablename__ = "question" __table_args__ = ( @@ -805,7 +815,7 @@ class Question(Base): question_unit = Column(Unicode(120)) question_min = Column(Float, nullable=True) question_max = Column(Float, nullable=True) - question_dtype = Column(Integer) + question_dtype = Column(Integer, ForeignKey("question_type.id")) question_cmp = Column(Unicode(120)) question_reqinreg = Column(Integer, server_default=text("'0'")) question_reqinasses = Column(Integer, server_default=text("'0'")) @@ -835,12 +845,20 @@ class Question(Base): qstgroups_user = Column(Unicode(80), nullable=True) qstgroups_id = Column(Unicode(80), nullable=True) question_sensitive = Column(Integer, server_default=text("'0'")) + question_anonymity = Column(Integer, ForeignKey("question_anonymity.id")) question_lang = Column(ForeignKey("i18n.lang_code"), nullable=True) extra = Column(MEDIUMTEXT(collation="utf8mb4_unicode_ci")) i18n = relationship("I18n") user = relationship("User") +class QuestionAnonymity(Base): + __tablename__ = "question_anonymity" + + id = Column(Integer, primary_key=True, nullable=False) + name = Column(Unicode(64), nullable=False) + + class Registry(Base): __tablename__ = "registry" __table_args__ = ( diff --git a/climmob/utility/question.py b/climmob/utility/question.py index 7e538543..3b2eb655 100644 --- a/climmob/utility/question.py +++ b/climmob/utility/question.py @@ -3,14 +3,15 @@ class QuestionType(Enum): TEXT = 1 - RANKING_OF_OPTIONS = 9 - COMPARISON_WITH_CHECK = 10 - LOCATION = 27 DECIMAL = 2 INTEGER = 3 GEOPOINT = 4 SELECT_ONE = 5 SELECT_MULTIPLE = 6 + PACKAGE_CODE = 7 + FARMER = 8 + RANKING_OF_OPTIONS = 9 + COMPARISON_WITH_CHECK = 10 GEOTRACE = 11 GEOSHAPE = 12 DATE = 13 @@ -20,6 +21,7 @@ class QuestionType(Enum): AUDIO = 17 VIDEO = 18 BARCODE_QR = 19 + LOCATION = 27 def is_type_numerical(q_type) -> bool: @@ -27,3 +29,12 @@ def is_type_numerical(q_type) -> bool: int(q_type) == QuestionType.DECIMAL.value or int(q_type) == QuestionType.INTEGER.value ) + + +class QuestionAnonymity(Enum): + REMOVE = 1 + PSEUDONYM = 2 + RANGE = 3 + NOISE = 4 + MASK = 5 + MONTH_YEAR = 6 From eb3cbfd32c66bae32e07d54b2f6abba60f7fa3a5 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Tue, 29 Jul 2025 23:21:50 -0600 Subject: [PATCH 13/66] refactor field query creation --- climmob/processes/db/question.py | 27 ++++++- climmob/processes/db/results.py | 132 +++++++++++++++++++------------ 2 files changed, 104 insertions(+), 55 deletions(-) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index e5c94e81..24851801 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -40,9 +40,11 @@ "getDefaultQuestionLanguage", "getQuestionOwner", "knowIfUserHasCreatedTranslations", - "get_question_sensitivity_by_project_id", + "get_sensitive_questions_anonymity_by_project_id", ] +from climmob.models.climmobv4 import QuestionType + log = logging.getLogger(__name__) @@ -517,15 +519,32 @@ def knowIfUserHasCreatedTranslations(request, userId): return False -def get_question_sensitivity_by_project_id(project_id, request): +def get_sensitive_questions_anonymity_by_project_id(project_id, request): + """ + Retrieve all questions of a project by its id. Includes the registry and all the assessments. + """ query = ( - request.dbsession.query(Question.question_code, Question.question_sensitive) + request.dbsession.query( + Question.question_code, + func.coalesce(Question.question_anonymity, QuestionType.anonymity_id).label( + "question_anonymity" + ), + ) .join(Registry, Registry.question_id == Question.question_id) + .join(QuestionType, QuestionType.id == Question.question_dtype) .filter(Registry.project_id == project_id) + .filter(Question.question_sensitive == 1) .union( - request.dbsession.query(Question.question_code, Question.question_sensitive) + request.dbsession.query( + Question.question_code, + func.coalesce( + Question.question_anonymity, QuestionType.anonymity_id + ).label("question_anonymity"), + ) .join(AssDetail, AssDetail.question_id == Question.question_id) + .join(QuestionType, QuestionType.id == Question.question_dtype) .filter(AssDetail.project_id == project_id) + .filter(Question.question_sensitive == 1) ) ) return query.all() diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 2df643e1..4f66d30c 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -5,9 +5,12 @@ from lxml import etree -from climmob.models import Assessment, Question, Project, mapFromSchema, Registry +from climmob.models import Assessment, Question, Project, mapFromSchema from climmob.models.repository import sql_fetch_all, sql_fetch_one -from climmob.processes import getCombinations, get_question_sensitivity_by_project_id +from climmob.processes import ( + getCombinations, + get_sensitive_questions_anonymity_by_project_id, +) __all__ = ["getJSONResult", "getCombinationsData"] @@ -269,10 +272,8 @@ def getPackageData(userOwner, projectId, projectCod, request): return packages -def is_field_sensitive(field, questions): +def get_question_anonymity(field, questions) -> int | None: for q in questions: - if q.question_sensitive == 0: - continue patterns = [ rf"^{q.question_code}(_[abc])?(_oth)?$", rf"^perf_{q.question_code}_[123]$", @@ -280,13 +281,54 @@ def is_field_sensitive(field, questions): ] for pattern in patterns: if re.fullmatch(pattern, field["name"]): - return True - return False + return q.question_anonymity + return None + + +class QuestionSelectFieldBuilder: + def __init__(self, anonymize): + self.column = None + self.table = None + self.form_id = None + self.prefix = None + self.sensitive = False + self.anonymize = anonymize + + def set_column(self, column): + self.column = column + + def set_table(self, table): + self.table = table + + def set_form_id(self, form_id): + self.form_id = form_id + + def set_prefix(self, prefix): + self.prefix = prefix + + def set_sensitive(self, sensitive): + self.sensitive = sensitive + + def build(self): + if not self.anonymize or not self.sensitive: + return f"{self.table}.{self.column} AS {self.prefix}_{self.column}" + + query = ( + f"COALESCE(MAX(" + f"CASE WHEN da.col_name = '{self.column}' " + f"AND da.form_id='{self.form_id}'" + f"THEN da.value END)," + f"{self.table}.{self.column}) " + f"AS {self.prefix}_{self.column}" + ) + return query def getData( userOwner, project_id, projectCod, registry, assessments, request, anonymize=False ): + from climmob.utility import QuestionAnonymity + data = ( request.dbsession.query(Question).filter(Question.question_regkey == 1).first() ) @@ -296,48 +338,39 @@ def getData( ) assessmentKey = data.question_code - questions = get_question_sensitivity_by_project_id(project_id, request) + questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) fields = [] reg_alias = "reg" + select_field_builder = QuestionSelectFieldBuilder(anonymize) + select_field_builder.set_table(reg_alias) + select_field_builder.set_prefix("REG") + select_field_builder.set_form_id("-") + for field in registry["fields"]: - if anonymize and is_field_sensitive(field, questions): - fields.append( - f"COALESCE(MAX(" - f"CASE WHEN da.col_name = '{field['name']}' AND da.form_id='-'" - f"THEN da.value END)," - f"{reg_alias}.{field['name']}) " - f"AS REG_{field['name']}" - ) - else: - fields.append( - reg_alias + "." + field["name"] + " AS " + "REG_" + field["name"] - ) + select_field_builder.set_column(field["name"]) + if anonymize: + anonymity = get_question_anonymity(field, questions) + if anonymity == QuestionAnonymity.REMOVE.value: + continue + select_field_builder.set_sensitive(anonymity is not None) + fields.append(select_field_builder.build()) + for assessment in assessments: assessment_alias = "assess_" + assessment["code"] + select_field_builder.set_table(assessment_alias) + select_field_builder.set_prefix(f'ASS{assessment["code"]}') + select_field_builder.set_form_id(f"{assessment['code']}") for field in assessment["fields"]: - if anonymize and is_field_sensitive(field, questions): - fields.append( - f"COALESCE(MAX(" - f"CASE WHEN da.col_name = '{field['name']}' AND da.form_id='{assessment['code']}'" - f"THEN da.value END)," - f"{assessment_alias}.{field['name']}) " - f"AS " + "ASS" + assessment["code"] + "_" - f"{field['name']}" - ) - else: - fields.append( - assessment_alias - + "." - + field["name"] - + " AS " - + "ASS" - + assessment["code"] - + "_" - + field["name"] - ) + select_field_builder.set_column(field["name"]) + if anonymize: + anonymity = get_question_anonymity(field, questions) + if anonymity == QuestionAnonymity.REMOVE.value: + continue + select_field_builder.set_sensitive(anonymity is not None) + fields.append(select_field_builder.build()) sql = ( "SELECT " @@ -374,17 +407,14 @@ def getData( + "." + assessmentKey ) - - sql = ( - sql - + " LEFT JOIN " - + userOwner - + "_" - + projectCod - + ".anonymized da" - + f" ON da.reg_id = {reg_alias}.qst162 " - + f" GROUP BY {reg_alias}.qst162" - ) + if anonymize: + sql = ( + sql + + " LEFT JOIN " + + f"{userOwner}_{projectCod}.anonymized da " + + f"ON da.reg_id = {reg_alias}.qst162 " + + f"GROUP BY {reg_alias}.qst162" + ) sql = sql + f" ORDER BY cast({reg_alias}.{registryKey} AS unsigned)" data = sql_fetch_all(sql) From efe7fa8b9b273b6c8f04872d2e3b9ace808fd0a1 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 30 Jul 2025 17:21:32 -0600 Subject: [PATCH 14/66] move register form validation to view class --- .../tests/test_utils/test_views_base_view.py | 7 +-- climmob/utility/__init__.py | 1 - climmob/utility/validators.py | 47 ------------------ climmob/views/basic_views.py | 48 ++++++++++++++++++- 4 files changed, 50 insertions(+), 53 deletions(-) delete mode 100644 climmob/utility/validators.py diff --git a/climmob/tests/test_utils/test_views_base_view.py b/climmob/tests/test_utils/test_views_base_view.py index 8a7f29fd..b87de949 100644 --- a/climmob/tests/test_utils/test_views_base_view.py +++ b/climmob/tests/test_utils/test_views_base_view.py @@ -803,8 +803,9 @@ def setUp(self): @classmethod def setUpClass(cls): cls.patchers["validate_register_form"] = { - "patch": patch( - "climmob.views.basic_views.validate_register_form", + "patch": patch.object( + RegisterView, + "validate_register_form", ), "return_value": (False, {}), } @@ -838,7 +839,7 @@ def tearDown(self): super().tearDown() if self.get_mock("validate_register_form").called: self.get_mock("validate_register_form").assert_called_once_with( - self.request.POST, self.view.request, self.view._ + self.request.POST ) if self.get_mock("add_user").called: self.get_mock("add_user").assert_called_once_with( diff --git a/climmob/utility/__init__.py b/climmob/utility/__init__.py index e50db0db..906b267f 100644 --- a/climmob/utility/__init__.py +++ b/climmob/utility/__init__.py @@ -1,4 +1,3 @@ from climmob.utility.helpers import * -from climmob.utility.validators import * from climmob.utility.factory import * from climmob.utility.question import * diff --git a/climmob/utility/validators.py b/climmob/utility/validators.py deleted file mode 100644 index 004ffcc5..00000000 --- a/climmob/utility/validators.py +++ /dev/null @@ -1,47 +0,0 @@ -import re - -from climmob.processes import userExists, emailExists - -# Form validation - -__all__ = ["validate_register_form"] - - -def validate_register_form(data, request, _): - error_summary = {} - errors = False - - if data["user_password"] != data["user_password2"]: - error_summary["InvalidPassword"] = _("Invalid password") - errors = True - if userExists(data["user_name"], request): - error_summary["UserExists"] = _("Username already exits") - errors = True - if emailExists(data["user_email"], request): - error_summary["EmailExists"] = _( - "There is already an account using to this email" - ) - errors = True - if data["user_policy"] == "False": - error_summary["CheckPolicy"] = _("You need to accept the terms of service") - errors = True - if data["user_name"] == "": - error_summary["EmptyUser"] = _("User cannot be emtpy") - errors = True - if data["user_password"] == "": - error_summary["EmptyPass"] = _("Password cannot be emtpy") - errors = True - if data["user_fullname"] == "": - error_summary["EmptyName"] = _("Full name cannot be emtpy") - errors = True - if data["user_email"] == "": - error_summary["EmptyEmail"] = _("Email cannot be emtpy") - errors = True - reg = re.compile(r"^[a-z0-9]+$") - if not reg.match(data["user_name"]): - error_summary["Caracters"] = _( - "The username can only use lowercase letters and numbers." - ) - errors = True - - return errors, error_summary diff --git a/climmob/views/basic_views.py b/climmob/views/basic_views.py index 6a371c89..7c2accf3 100644 --- a/climmob/views/basic_views.py +++ b/climmob/views/basic_views.py @@ -1,3 +1,4 @@ +import re from datetime import datetime import json import logging @@ -27,8 +28,9 @@ getSectorList, getUserCount, getProjectCount, + userExists, + emailExists, ) -from climmob.utility import validate_register_form from climmob.utility.email import build_email_message from climmob.utility.helpers import readble_date from climmob.views.classes import publicView @@ -360,7 +362,7 @@ def post(self): "countries": getCountryList(self.request), "sectors": getSectorList(self.request), } - errors, error_summary = validate_register_form(data, self.request, self._) + errors, error_summary = self.validate_register_form(data) if errors: response["error_summary"] = error_summary @@ -403,3 +405,45 @@ def post(self): location=self.request.route_url("dashboard"), headers=headers, ) + + # Create validator if needed by another view + def validate_register_form(self, data): + error_summary = {} + errors = False + + if data["user_password"] != data["user_password2"]: + error_summary["InvalidPassword"] = self._("Invalid password") + errors = True + if userExists(data["user_name"], self.request): + error_summary["UserExists"] = self._("Username already exits") + errors = True + if emailExists(data["user_email"], self.request): + error_summary["EmailExists"] = self._( + "There is already an account using to this email" + ) + errors = True + if data["user_policy"] == "False": + error_summary["CheckPolicy"] = self._( + "You need to accept the terms of service" + ) + errors = True + if data["user_name"] == "": + error_summary["EmptyUser"] = self._("User cannot be emtpy") + errors = True + if data["user_password"] == "": + error_summary["EmptyPass"] = self._("Password cannot be emtpy") + errors = True + if data["user_fullname"] == "": + error_summary["EmptyName"] = self._("Full name cannot be emtpy") + errors = True + if data["user_email"] == "": + error_summary["EmptyEmail"] = self._("Email cannot be emtpy") + errors = True + reg = re.compile(r"^[a-z0-9]+$") + if not reg.match(data["user_name"]): + error_summary["Caracters"] = self._( + "The username can only use lowercase letters and numbers." + ) + errors = True + + return errors, error_summary From ca9839b1d860e9dda64beab8e1e3dfaf59ceb433 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 31 Jul 2025 12:08:13 -0600 Subject: [PATCH 15/66] add order to question types --- climmob/models/climmobv4.py | 1 + climmob/utility/question.py | 61 +++++++++++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/climmob/models/climmobv4.py b/climmob/models/climmobv4.py index 0662fcc9..d3ed9b5c 100644 --- a/climmob/models/climmobv4.py +++ b/climmob/models/climmobv4.py @@ -792,6 +792,7 @@ class QuestionType(Base): id = Column(Integer, primary_key=True, nullable=False) name = Column(Unicode(64), nullable=False) + order = Column(Integer, nullable=False) anonymity_id = Column( Integer, ForeignKey("question_anonymity.id"), primary_key=True, nullable=False ) diff --git a/climmob/utility/question.py b/climmob/utility/question.py index 3b2eb655..7b1f421d 100644 --- a/climmob/utility/question.py +++ b/climmob/utility/question.py @@ -1,19 +1,19 @@ -from enum import Enum +from enum import Enum, IntEnum, auto -class QuestionType(Enum): +class QuestionType(IntEnum): TEXT = 1 DECIMAL = 2 INTEGER = 3 - GEOPOINT = 4 + GEO_POINT = 4 SELECT_ONE = 5 SELECT_MULTIPLE = 6 PACKAGE_CODE = 7 FARMER = 8 RANKING_OF_OPTIONS = 9 COMPARISON_WITH_CHECK = 10 - GEOTRACE = 11 - GEOSHAPE = 12 + GEO_TRACE = 11 + GEO_SHAPE = 12 DATE = 13 TIME = 14 DATETIME = 15 @@ -24,11 +24,54 @@ class QuestionType(Enum): LOCATION = 27 +class QuestionTypeLabel(Enum): + TEXT = "Text" + DECIMAL = "Decimal" + INTEGER = "Integer" + GEO_POINT = "GeoPoint" + SELECT_ONE = "Select one" + SELECT_MULTIPLE = "Select multiple" + PACKAGE_CODE = "Package code" + FARMER = "Farmer" + RANKING_OF_OPTIONS = "Ranking of options" + COMPARISON_WITH_CHECK = "Comparison with check" + GEO_TRACE = "GeoTrace" + GEO_SHAPE = "GeoShape" + DATE = "Date" + TIME = "Time" + DATETIME = "DateTime" + IMAGE = "Image" + AUDIO = "Audio" + VIDEO = "Video" + BARCODE_QR = "Barcode/QR" + LOCATION = "Location" + + +class QuestionTypeOrder(IntEnum): + TEXT = auto() + RANKING_OF_OPTIONS = auto() + COMPARISON_WITH_CHECK = auto() + LOCATION = auto() + DECIMAL = auto() + INTEGER = auto() + GEO_POINT = auto() + SELECT_ONE = auto() + SELECT_MULTIPLE = auto() + GEO_TRACE = auto() + GEO_SHAPE = auto() + DATE = auto() + TIME = auto() + DATETIME = auto() + IMAGE = auto() + AUDIO = auto() + VIDEO = auto() + BARCODE_QR = auto() + PACKAGE_CODE = -1 # Not included as an option + FARMER = -1 # Not included as an option + + def is_type_numerical(q_type) -> bool: - return ( - int(q_type) == QuestionType.DECIMAL.value - or int(q_type) == QuestionType.INTEGER.value - ) + return int(q_type) == QuestionType.DECIMAL or int(q_type) == QuestionType.INTEGER class QuestionAnonymity(Enum): From aeea733ea1a4d454b0870234cd53863ba0f6d2ce Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 31 Jul 2025 12:54:10 -0600 Subject: [PATCH 16/66] add labels for question anonymity --- climmob/utility/question.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/climmob/utility/question.py b/climmob/utility/question.py index 7b1f421d..50c7064f 100644 --- a/climmob/utility/question.py +++ b/climmob/utility/question.py @@ -74,10 +74,19 @@ def is_type_numerical(q_type) -> bool: return int(q_type) == QuestionType.DECIMAL or int(q_type) == QuestionType.INTEGER -class QuestionAnonymity(Enum): +class QuestionAnonymity(IntEnum): REMOVE = 1 PSEUDONYM = 2 RANGE = 3 NOISE = 4 MASK = 5 MONTH_YEAR = 6 + + +class QuestionAnonymityLabel(Enum): + REMOVE = "Remove" + PSEUDONYM = "Pseudonym" + RANGE = "Range" + NOISE = "Noise" + MASK = "Mask" + MONTH_YEAR = "Month-Year" From ef346f7c770f509a58a7e6d700bfe22bfd5acae1 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 31 Jul 2025 13:20:16 -0600 Subject: [PATCH 17/66] get question types from database --- climmob/processes/db/question.py | 44 ++++++++++++++++++- .../snippets/question/question-form.jinja2 | 28 +++--------- climmob/views/question.py | 4 ++ 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index 24851801..a8758ab2 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -41,9 +41,11 @@ "getQuestionOwner", "knowIfUserHasCreatedTranslations", "get_sensitive_questions_anonymity_by_project_id", + "get_question_types_with_anonymity_options", ] -from climmob.models.climmobv4 import QuestionType +from climmob.models.climmobv4 import QuestionType, QuestionAnonymity +import climmob.utility as utils log = logging.getLogger(__name__) @@ -548,3 +550,43 @@ def get_sensitive_questions_anonymity_by_project_id(project_id, request): ) ) return query.all() + + +def get_question_types_with_anonymity_options(request): + query = ( + request.dbsession.query( + QuestionType.id.label("q_type_id"), + QuestionType.name.label("q_type_name"), + QuestionAnonymity.id.label("q_anonymity_id"), + QuestionAnonymity.name.label("q_anonymity_name"), + ) + .join(QuestionAnonymity, QuestionAnonymity.id == QuestionType.anonymity_id) + .filter(QuestionType.order != -1) + .order_by(QuestionType.order) + ) + result = mapFromSchema(query.all()) + + def map_type_options(q_type): + mapped_type = { + "id": q_type["q_type_id"], + "name": utils.QuestionTypeLabel[q_type["q_type_name"]].value, + "anonymity_opts": [ + { + "id": q_type["q_anonymity_id"], + "name": utils.QuestionAnonymityLabel[ + q_type["q_anonymity_name"] + ].value, + } + ], + } + if q_type["q_anonymity_id"] != utils.QuestionAnonymity.REMOVE.value: + mapped_type["anonymity_opts"].append( + { + "id": utils.QuestionAnonymity.REMOVE.value, + "name": utils.QuestionAnonymityLabel.REMOVE.value, + } + ) + return mapped_type + + mapped = list(map(map_type_options, result)) + return mapped diff --git a/climmob/templates/snippets/question/question-form.jinja2 b/climmob/templates/snippets/question/question-form.jinja2 index 18e3d885..351578c8 100644 --- a/climmob/templates/snippets/question/question-form.jinja2 +++ b/climmob/templates/snippets/question/question-form.jinja2 @@ -29,27 +29,9 @@
@@ -553,13 +535,13 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %}
- +
+ data-off-text="{{ _('No') }}">
diff --git a/climmob/views/question.py b/climmob/views/question.py index cde6f9a3..87503f5c 100644 --- a/climmob/views/question.py +++ b/climmob/views/question.py @@ -38,6 +38,7 @@ getQuestionOwner, getPhraseTranslationInLanguage, knowIfUserHasCreatedTranslations, + get_question_types_with_anonymity_options, ) from climmob.views.classes import privateView from climmob.views.validators.question.QuestionMinMaxValidator import ( @@ -785,6 +786,8 @@ def processView(self): nextPage = self.request.params.get("next") + question_types = get_question_types_with_anonymity_options(self.request) + regularDict = { "UserQuestion": UserQuestionMoreBioversity(user_name, self.request), "knowIfUserHasCreatedTranslations": knowIfUserHasCreatedTranslations( @@ -799,6 +802,7 @@ def processView(self): "seeQuestion": seeQuestion, "nextPage": nextPage, "sectionActive": "questions", + "question_types": question_types, } return regularDict From 3ebd71110145dfa1fd09f57623362f5a8c1bead3 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 1 Aug 2025 09:36:05 -0600 Subject: [PATCH 18/66] add select input for anonymity --- climmob/templates/question/library.jinja2 | 49 +++++++++++++++++-- .../snippets/question/question-form.jinja2 | 8 +++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/climmob/templates/question/library.jinja2 b/climmob/templates/question/library.jinja2 index 5bc5394e..c73ab484 100644 --- a/climmob/templates/question/library.jinja2 +++ b/climmob/templates/question/library.jinja2 @@ -211,8 +211,11 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} - var elem_ckb_question_sensitive = document.querySelector('#ckb_question_as_sensitive'); - var ckb_question_sensitive = new Switchery(elem_ckb_question_sensitive, { color: '#1AB394' }); + let elem_ckb_question_sensitive = document.querySelector('#ckb_question_as_sensitive'); + let ckb_question_sensitive = new Switchery(elem_ckb_question_sensitive, { color: '#1AB394' }); + + let anonymity_select = document.getElementById("question_anonymity"); + {% endif %} @@ -240,6 +243,28 @@ $("#question_max").val(""); } + const q_types = {{ question_types | tojson }}; + + function update_anonymity_types(value, current_anonymity_id=-1) { + let option; + anonymity_select.innerHTML = ""; + let anonymity_index = 0; + for (const q_type of q_types) { + if (value === q_type.id) { + q_type.anonymity_opts.forEach((anonymity, j) => { + option = new Option(anonymity.name, anonymity.id); + anonymity_select.appendChild(option); + if (anonymity.id === current_anonymity_id) { + anonymity_index = j; + } + }); + break; + } + } + anonymity_select.selectedIndex = anonymity_index; + $('#question_anonymity').trigger('change.select2'); + } + $(document).ready(function() { tour = new Tour({ @@ -345,6 +370,10 @@ $("#question_dtype").select2(); var type = $('#question_dtype').val(); + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + update_anonymity_types(parseInt(type)); + {% endif %} + var question_requiredvalue = 0; var objQRequired = $("#ckb_required_value"); if (objQRequired.is(':checked')) @@ -370,7 +399,12 @@ { clean_extra_fields(); - var value = $('#question_dtype').val(); + const value = $('#question_dtype').val(); + + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + const current_anonymity_id = $('#question_anonymity').val(); + update_anonymity_types(parseInt(value), parseInt(current_anonymity_id)); + {% endif %} $("#question"+value).css("display",'initial'); @@ -688,6 +722,7 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} "question_sensitive": question_sensitive, + "question_anonymity": $('#question_anonymity').val() {% endif %} @@ -904,13 +939,17 @@ hideNumericalInputs() + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + update_anonymity_types(q_types[0].id) + {% endif %} + loadLanguages(); $(".inputForQuestion").val("") $("#question_group").val(category); $("#question_forms").val("3"); $('#question_group').trigger("change.select2"); - $("#question_dtype").val("1") + $("#question_dtype").val(q_types[0].id) $('#question_dtype').trigger('change.select2'); $("#question1").css("display",'initial'); @@ -1054,6 +1093,8 @@ else setSwitchery(ckb_question_sensitive, false) + update_anonymity_types(dataJson["question_dtype"], dataJson["question_anonymity"]) + {% endif %} if(isQuestionTypeNumerical(dataJson["question_dtype"])) diff --git a/climmob/templates/snippets/question/question-form.jinja2 b/climmob/templates/snippets/question/question-form.jinja2 index 351578c8..b340311a 100644 --- a/climmob/templates/snippets/question/question-form.jinja2 +++ b/climmob/templates/snippets/question/question-form.jinja2 @@ -545,6 +545,14 @@
+
+ + +
+ +
+
{% endif %}
From dbcf308b1a6411ce9b69d4e0f6ebbcffe0146254 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 4 Aug 2025 08:17:32 -0600 Subject: [PATCH 19/66] hide anonymity when not sensitive --- climmob/templates/question/library.jinja2 | 34 ++++++++++++++++--- .../snippets/question/question-form.jinja2 | 3 +- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/climmob/templates/question/library.jinja2 b/climmob/templates/question/library.jinja2 index c73ab484..670acd71 100644 --- a/climmob/templates/question/library.jinja2 +++ b/climmob/templates/question/library.jinja2 @@ -216,6 +216,15 @@ let anonymity_select = document.getElementById("question_anonymity"); + elem_ckb_question_sensitive.addEventListener('change', function() { + anonymity_select.value = saved_anonymity_id + $(anonymity_select).trigger('change.select2'); + if (elem_ckb_question_sensitive.checked) { + $("#div-question-anonymity").css('display', 'block'); + } else { + $("#div-question-anonymity").css('display', 'none'); + } + }); {% endif %} @@ -245,6 +254,10 @@ const q_types = {{ question_types | tojson }}; + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + + let saved_anonymity_id = null; + function update_anonymity_types(value, current_anonymity_id=-1) { let option; anonymity_select.innerHTML = ""; @@ -264,6 +277,7 @@ anonymity_select.selectedIndex = anonymity_index; $('#question_anonymity').trigger('change.select2'); } + {% endif %} $(document).ready(function() { @@ -665,9 +679,12 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} - var question_sensitive = 0; - if ($("#ckb_question_as_sensitive").is(':checked')) + let question_sensitive = 0; + let anonymity_input = {}; + if ($("#ckb_question_as_sensitive").is(':checked')) { question_sensitive = 1; + anonymity_input["question_anonymity"] = anonymity_select.value; + } {% endif %} @@ -722,7 +739,7 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} "question_sensitive": question_sensitive, - "question_anonymity": $('#question_anonymity').val() + ...anonymity_input {% endif %} @@ -977,6 +994,7 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} setSwitchery(ckb_question_sensitive, false); + $("#div-question-anonymity").css('display', 'none'); {% endif %} @@ -1088,13 +1106,19 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} - if (dataJson["question_sensitive"] == 1) + if (dataJson["question_sensitive"] === 1) { setSwitchery(ckb_question_sensitive, true) - else + $("#div-question-anonymity").css('display', 'block'); + } + else { setSwitchery(ckb_question_sensitive, false) + $("#div-question-anonymity").css('display', 'none'); + } update_anonymity_types(dataJson["question_dtype"], dataJson["question_anonymity"]) + saved_anonymity_id = dataJson["question_anonymity"]; + {% endif %} if(isQuestionTypeNumerical(dataJson["question_dtype"])) diff --git a/climmob/templates/snippets/question/question-form.jinja2 b/climmob/templates/snippets/question/question-form.jinja2 index b340311a..1d007221 100644 --- a/climmob/templates/snippets/question/question-form.jinja2 +++ b/climmob/templates/snippets/question/question-form.jinja2 @@ -545,8 +545,7 @@ -
- +
- +
+
+ +
+ +
+
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+
+ +
+ + {{ _("Short template for the pseudonym, e.g., 'Farmer-{}'. The curly braces will be replaced with the package code.") }} +
+
+
@@ -558,4 +591,4 @@
{% block qstform_extra %} -{% endblock qstform_extra %} \ No newline at end of file +{% endblock qstform_extra %} diff --git a/climmob/views/question.py b/climmob/views/question.py index 87503f5c..0c23fbc0 100644 --- a/climmob/views/question.py +++ b/climmob/views/question.py @@ -39,6 +39,7 @@ getPhraseTranslationInLanguage, knowIfUserHasCreatedTranslations, get_question_types_with_anonymity_options, + get_question_anonymity_types_as_dict, ) from climmob.views.classes import privateView from climmob.views.validators.question.QuestionMinMaxValidator import ( @@ -803,6 +804,7 @@ def processView(self): "nextPage": nextPage, "sectionActive": "questions", "question_types": question_types, + "anonymity_types": get_question_anonymity_types_as_dict(self.request), } return regularDict From bb14f5943c32eca4f0878cc91094ebf576db0266 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 7 Aug 2025 09:47:38 -0600 Subject: [PATCH 23/66] add anonymization params validation in interface --- climmob/templates/question/library.jinja2 | 80 ++++++++++++++++++- .../snippets/question/question-form.jinja2 | 8 +- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/climmob/templates/question/library.jinja2 b/climmob/templates/question/library.jinja2 index 5a99a692..b1551991 100644 --- a/climmob/templates/question/library.jinja2 +++ b/climmob/templates/question/library.jinja2 @@ -195,6 +195,7 @@ + {% include 'snippets/question/question_enums.jinja2' %} \ No newline at end of file From 933e9c21d5fb512f5409feddc83796ec2fccd66c Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 8 Aug 2025 09:47:33 -0600 Subject: [PATCH 25/66] retrieve type info from enums --- climmob/processes/db/question.py | 82 +++++-------- climmob/templates/question/library.jinja2 | 23 ++-- .../snippets/question/question_enums.jinja2 | 32 ------ climmob/utility/__init__.py | 7 ++ climmob/utility/question.py | 108 +++++++++++++----- climmob/views/question.py | 13 ++- 6 files changed, 140 insertions(+), 125 deletions(-) delete mode 100644 climmob/templates/snippets/question/question_enums.jinja2 diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index 684a5a6f..ab565111 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -1,5 +1,5 @@ import json -from collections import defaultdict +import re from sqlalchemy import func, or_, and_ @@ -42,20 +42,40 @@ "getQuestionOwner", "knowIfUserHasCreatedTranslations", "get_sensitive_questions_anonymity_by_project_id", - "get_question_types_with_anonymity_options", - "get_question_anonymity_types_as_dict", ] -from climmob.models.climmobv4 import ( - QuestionType, - QuestionAnonymity, - QuestionTypeAnonymity, -) -import climmob.utility as utils +from climmob.models.climmobv4 import AnonymizationParameter log = logging.getLogger(__name__) +def save_anonymization_params(question_id, data, request): + delete_existing_anonymization_params(question_id, request) + + params = [] + for key in data.keys(): + pattern = r"anonym_param_([a-z_]+)" + match = re.match(pattern, key) + if match: + params.append({"name": match.group(1), "value": data[key]}) + + for param in params: + new_param = AnonymizationParameter(**param) + new_param.question_id = question_id + request.dbsession.add(new_param) + request.dbsession.flush() + + +def delete_existing_anonymization_params(question_id, request): + try: + request.dbsession.query(AnonymizationParameter).filter( + AnonymizationParameter.question_id == question_id + ).delete() + return True, "" + except Exception as e: + return False, str(e) + + def addQuestion(data, request): _ = request.translate mappeData = mapToSchema(Question, data) @@ -64,6 +84,7 @@ def addQuestion(data, request): try: request.dbsession.add(newQuestion) request.dbsession.flush() + save_anonymization_params(newQuestion.question_id, data, request) return True, newQuestion.question_id except DatabaseError as e: save_point.rollback() @@ -140,6 +161,7 @@ def updateQuestion(data, request): request.dbsession.query(Question).filter( Question.user_name == data["user_name"] ).filter(Question.question_id == data["question_id"]).update(mappeData) + save_anonymization_params(data["question_id"], data, request) return True, data["question_id"] except DatabaseError as e: log.error("Error creating the question. The question is very long") @@ -550,45 +572,3 @@ def get_sensitive_questions_anonymity_by_project_id(project_id, request): ) ) return query.all() - - -def get_question_types_with_anonymity_options(request): - query = ( - request.dbsession.query( - QuestionType.id.label("q_type_id"), - QuestionType.name.label("q_type_name"), - QuestionAnonymity.id.label("q_anonymity_id"), - QuestionAnonymity.name.label("q_anonymity_name"), - ) - .join(QuestionTypeAnonymity, QuestionTypeAnonymity.type_id == QuestionType.id) - .join( - QuestionAnonymity, - QuestionAnonymity.id == QuestionTypeAnonymity.anonymity_id, - ) - .filter(QuestionType.order != -1) - .order_by(QuestionType.order, QuestionAnonymity.id) - ) - result = mapFromSchema(query.all()) - - grouped = defaultdict(list) - for item in result: - key = (item["q_type_id"], item["q_type_name"]) - grouped[key].append( - {"id": item["q_anonymity_id"], "name": item["q_anonymity_name"]} - ) - - result = [ - {"id": id_, "name": name, "anonymity_opts": opts} - for (id_, name), opts in grouped.items() - ] - - return result - - -def get_question_anonymity_types_as_dict(request): - query = request.dbsession.query(QuestionAnonymity) - anonymity_types = mapFromSchema(query.all()) - result = {} - for anonymity in anonymity_types: - result[anonymity["name"]] = anonymity["id"] - return result diff --git a/climmob/templates/question/library.jinja2 b/climmob/templates/question/library.jinja2 index b1551991..1b1a3daa 100644 --- a/climmob/templates/question/library.jinja2 +++ b/climmob/templates/question/library.jinja2 @@ -195,7 +195,6 @@ - {% include 'snippets/question/question_enums.jinja2' %} \ No newline at end of file diff --git a/climmob/utility/__init__.py b/climmob/utility/__init__.py index 906b267f..b68006cb 100644 --- a/climmob/utility/__init__.py +++ b/climmob/utility/__init__.py @@ -1,3 +1,10 @@ from climmob.utility.helpers import * from climmob.utility.factory import * from climmob.utility.question import * + + +def get_enum_as_dict(enum): + result = {} + for member in enum: + result[member.name] = member.value + return result diff --git a/climmob/utility/question.py b/climmob/utility/question.py index 50c7064f..34108c9a 100644 --- a/climmob/utility/question.py +++ b/climmob/utility/question.py @@ -1,6 +1,10 @@ from enum import Enum, IntEnum, auto +def _(x): + return x + + class QuestionType(IntEnum): TEXT = 1 DECIMAL = 2 @@ -25,26 +29,26 @@ class QuestionType(IntEnum): class QuestionTypeLabel(Enum): - TEXT = "Text" - DECIMAL = "Decimal" - INTEGER = "Integer" - GEO_POINT = "GeoPoint" - SELECT_ONE = "Select one" - SELECT_MULTIPLE = "Select multiple" - PACKAGE_CODE = "Package code" - FARMER = "Farmer" - RANKING_OF_OPTIONS = "Ranking of options" - COMPARISON_WITH_CHECK = "Comparison with check" - GEO_TRACE = "GeoTrace" - GEO_SHAPE = "GeoShape" - DATE = "Date" - TIME = "Time" - DATETIME = "DateTime" - IMAGE = "Image" - AUDIO = "Audio" - VIDEO = "Video" - BARCODE_QR = "Barcode/QR" - LOCATION = "Location" + TEXT = _("Text") + DECIMAL = _("Decimal") + INTEGER = _("Integer") + GEO_POINT = _("GeoPoint") + SELECT_ONE = _("Select one") + SELECT_MULTIPLE = _("Select multiple") + PACKAGE_CODE = _("Package code") + FARMER = _("Farmer") + RANKING_OF_OPTIONS = _("Ranking of options") + COMPARISON_WITH_CHECK = _("Comparison with check") + GEO_TRACE = _("GeoTrace") + GEO_SHAPE = _("GeoShape") + DATE = _("Date") + TIME = _("Time") + DATETIME = _("DateTime") + IMAGE = _("Image") + AUDIO = _("Audio") + VIDEO = _("Video") + BARCODE_QR = _("Barcode/QR") + LOCATION = _("Location") class QuestionTypeOrder(IntEnum): @@ -84,9 +88,61 @@ class QuestionAnonymity(IntEnum): class QuestionAnonymityLabel(Enum): - REMOVE = "Remove" - PSEUDONYM = "Pseudonym" - RANGE = "Range" - NOISE = "Noise" - MASK = "Mask" - MONTH_YEAR = "Month-Year" + REMOVE = _("Remove") + PSEUDONYM = _("Pseudonym") + RANGE = _("Range") + NOISE = _("Noise") + MASK = _("Mask") + MONTH_YEAR = _("Month-Year") + + +QA = QuestionAnonymity + + +class QuestionTypeAnonymity(Enum): + TEXT = [QA.REMOVE, QA.PSEUDONYM] + DECIMAL = [QA.REMOVE, QA.RANGE] + INTEGER = [QA.REMOVE, QA.RANGE] + GEO_POINT = [QA.REMOVE, QA.NOISE] + SELECT_ONE = [QA.REMOVE] + SELECT_MULTIPLE = [QA.REMOVE] + PACKAGE_CODE = [QA.REMOVE] + FARMER = [QA.REMOVE] + RANKING_OF_OPTIONS = [QA.REMOVE] + COMPARISON_WITH_CHECK = [QA.REMOVE] + GEO_TRACE = [QA.REMOVE] + GEO_SHAPE = [QA.REMOVE] + DATE = [QA.REMOVE, QA.MONTH_YEAR] + TIME = [QA.REMOVE] + DATETIME = [QA.REMOVE, QA.MONTH_YEAR] + IMAGE = [QA.REMOVE] + AUDIO = [QA.REMOVE] + VIDEO = [QA.REMOVE] + BARCODE_QR = [QA.REMOVE] + LOCATION = [QA.REMOVE] + + +def get_question_types_with_anonymity_labeled(request): + result = [] + for q_type in QuestionType: + order = QuestionTypeOrder[q_type.name].value + if order == -1: + continue + anonymity_opts = [] + for anonymity in QuestionTypeAnonymity[q_type.name].value: + anonymity_name = QuestionAnonymityLabel[anonymity.name].value + anonymity_name = request.translate(anonymity_name) + anonymity_opts.append({"id": anonymity.value, "name": anonymity_name}) + anonymity_opts = sorted(anonymity_opts, key=lambda x: x["id"]) + q_type_name = QuestionTypeLabel[q_type.name].value + q_type_name = request.translate(q_type_name) + result.append( + { + "id": q_type.value, + "name": q_type_name, + "anonymity_opts": anonymity_opts, + "order": order, + } + ) + result = sorted(result, key=lambda x: x["order"]) + return result diff --git a/climmob/views/question.py b/climmob/views/question.py index 0c23fbc0..940ea580 100644 --- a/climmob/views/question.py +++ b/climmob/views/question.py @@ -38,8 +38,12 @@ getQuestionOwner, getPhraseTranslationInLanguage, knowIfUserHasCreatedTranslations, - get_question_types_with_anonymity_options, - get_question_anonymity_types_as_dict, +) +from climmob.utility import ( + get_enum_as_dict, + QuestionAnonymity, + get_question_types_with_anonymity_labeled, + QuestionType, ) from climmob.views.classes import privateView from climmob.views.validators.question.QuestionMinMaxValidator import ( @@ -787,7 +791,7 @@ def processView(self): nextPage = self.request.params.get("next") - question_types = get_question_types_with_anonymity_options(self.request) + question_types = get_question_types_with_anonymity_labeled(self.request) regularDict = { "UserQuestion": UserQuestionMoreBioversity(user_name, self.request), @@ -804,7 +808,8 @@ def processView(self): "nextPage": nextPage, "sectionActive": "questions", "question_types": question_types, - "anonymity_types": get_question_anonymity_types_as_dict(self.request), + "QuestionAnonymity": get_enum_as_dict(QuestionAnonymity), + "QuestionType": get_enum_as_dict(QuestionType), } return regularDict From 375b280a77a4e99410ebfb6aceda46a2220fb05c Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 8 Aug 2025 14:11:50 -0600 Subject: [PATCH 26/66] add params to GET question details --- climmob/processes/db/question.py | 10 ++++++++++ climmob/templates/question/library.jinja2 | 15 +++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index ab565111..0e35de64 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -361,6 +361,16 @@ def userQuestionDetailsById(userOwner, questionId, request, language="default"): data["num_options"] = len(options) data["question_options"] = options + if data["question_sensitive"]: + params = ( + request.dbsession.query( + AnonymizationParameter.name, AnonymizationParameter.value + ) + .filter(AnonymizationParameter.question_id == data["question_id"]) + .all() + ) + data.update(params) + return data diff --git a/climmob/templates/question/library.jinja2 b/climmob/templates/question/library.jinja2 index 1b1a3daa..f44d8435 100644 --- a/climmob/templates/question/library.jinja2 +++ b/climmob/templates/question/library.jinja2 @@ -309,6 +309,10 @@ } } + function clean_anonymization_section() { + anonymity_inputs_selector.prop('value', ""); + } + {% endif %} $(document).ready(function() { @@ -1125,6 +1129,9 @@ clean_extra_fields(); clean_extra_section(); clean_buttoms_question(); + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + clean_anonymization_section(); + {% endif %} loadLanguages(user_name); @@ -1208,6 +1215,14 @@ saved_anonymity_id = dataJson["question_anonymity"]; update_anonymity_types(dataJson["question_dtype"], saved_anonymity_id) + const regex = /anonym_param_([a-z_]+)/; + anonymity_inputs_selector.each(function(index, element) { + const match = regex.exec($(element).attr('name')); + if (match) { + $(element).prop("value", dataJson[match[1]]) + } + }); + {% endif %} if(isQuestionTypeNumerical(dataJson["question_dtype"])) From 35a9ff8533891ccbd32d8a45e43e95a844d97917 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 8 Aug 2025 22:55:49 -0600 Subject: [PATCH 27/66] add anonymization process --- climmob/processes/db/question.py | 91 ++++++++++++++++++++++++++++++++ climmob/processes/db/results.py | 33 +++++------- climmob/processes/odk/api.py | 9 ++++ climmob/utility/question.py | 14 +++++ 4 files changed, 128 insertions(+), 19 deletions(-) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index 0e35de64..e74ca3fd 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -42,13 +42,33 @@ "getQuestionOwner", "knowIfUserHasCreatedTranslations", "get_sensitive_questions_anonymity_by_project_id", + "anonymize_questions", ] from climmob.models.climmobv4 import AnonymizationParameter +from climmob.models.repository import sql_execute +from climmob.utility import get_question_by_field_name, QuestionAnonymity, QuestionType log = logging.getLogger(__name__) +def get_anonymization_params(question_id, request): + result = mapFromSchema( + request.dbsession.query(AnonymizationParameter) + .filter(AnonymizationParameter.question_id == question_id) + .all() + ) + return result + + +def get_anonymization_params_as_dict(question_id, request): + params = get_anonymization_params(question_id, request) + result = {} + for param in params: + result[param["name"]] = param["value"] + return result + + def save_anonymization_params(question_id, data, request): delete_existing_anonymization_params(question_id, request) @@ -565,6 +585,8 @@ def get_sensitive_questions_anonymity_by_project_id(project_id, request): """ query = ( request.dbsession.query( + Question.question_id, + Question.question_dtype, Question.question_code, Question.question_anonymity, ) @@ -573,6 +595,8 @@ def get_sensitive_questions_anonymity_by_project_id(project_id, request): .filter(Question.question_sensitive == 1) .union( request.dbsession.query( + Question.question_id, + Question.question_dtype, Question.question_code, Question.question_anonymity, ) @@ -582,3 +606,70 @@ def get_sensitive_questions_anonymity_by_project_id(project_id, request): ) ) return query.all() + + +def anonymize_questions(request, form, form_id, project_id, schema): + questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) + + registry_id = ( + form["grp_validation/clc_after"] if form_id == "-" else form["grp_1/QST163"] + ) + + pattern = r"grp_\d+/(.+)" + to_anonymize = [] + + for key in form.keys(): + match = re.fullmatch(pattern, key) + if not match: + continue + question = get_question_by_field_name(match.group(1), questions) + if question and question.question_anonymity != QuestionAnonymity.REMOVE.value: + to_anonymize.append( + {"field_name": match.group(1), "value": form[key], "question": question} + ) + + if not to_anonymize: + return True + + anonymized_values = [] + + for field in to_anonymize: + params = get_anonymization_params_as_dict( + field["question"].question_id, request + ) + if field["question"].question_anonymity == QuestionAnonymity.PSEUDONYM.value: + field["value"] = params["pseudonym"].replace("{}", registry_id) + elif field["question"].question_anonymity == QuestionAnonymity.RANGE.value: + parser = ( + int + if field["question"].question_dtype == QuestionType.INTEGER.value + else float + ) + field["value"] = parser(field["value"]) + params["lower_bound"] = parser(params["lower_bound"]) + params["upper_bound"] = parser(params["upper_bound"]) + params["interval"] = parser(params["interval"]) + + if field["value"] < params["lower_bound"]: + field["value"] = f'<{params["lower_bound"]}' + elif field["value"] > params["upper_bound"]: + field["value"] = f'>{params["upper_bound"]}' + else: + i = params["lower_bound"] + while i < params["upper_bound"]: + if i <= field["value"] < (i + params["interval"]): + field["value"] = f'{i}-{i + params["interval"]}' + i += params["interval"] + + value = ( + f"(" + f"'{form_id}', " + f"'{registry_id}', " + f"'{field['field_name']}', " + f"'{field['value']}'" + f")" + ) + anonymized_values.append(value) + + sql = f"INSERT INTO {schema}.anonymized VALUES {', '.join(anonymized_values)}" + sql_execute(sql) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 4f66d30c..a689e9e9 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -14,6 +14,8 @@ __all__ = ["getJSONResult", "getCombinationsData"] +from climmob.utility import get_question_by_field_name + def getMiltiSelectLookUpTable(XMLFile, multiSelectTable): tree = etree.parse(XMLFile) @@ -272,19 +274,6 @@ def getPackageData(userOwner, projectId, projectCod, request): return packages -def get_question_anonymity(field, questions) -> int | None: - for q in questions: - patterns = [ - rf"^{q.question_code}(_[abc])?(_oth)?$", - rf"^perf_{q.question_code}_[123]$", - rf"^char_{q.question_code}_(pos|neg)$", - ] - for pattern in patterns: - if re.fullmatch(pattern, field["name"]): - return q.question_anonymity - return None - - class QuestionSelectFieldBuilder: def __init__(self, anonymize): self.column = None @@ -352,10 +341,13 @@ def getData( for field in registry["fields"]: select_field_builder.set_column(field["name"]) if anonymize: - anonymity = get_question_anonymity(field, questions) - if anonymity == QuestionAnonymity.REMOVE.value: + question = get_question_by_field_name(field["name"], questions) + if ( + question + and question.question_anonymity == QuestionAnonymity.REMOVE.value + ): continue - select_field_builder.set_sensitive(anonymity is not None) + select_field_builder.set_sensitive(question is not None) fields.append(select_field_builder.build()) for assessment in assessments: @@ -366,10 +358,13 @@ def getData( for field in assessment["fields"]: select_field_builder.set_column(field["name"]) if anonymize: - anonymity = get_question_anonymity(field, questions) - if anonymity == QuestionAnonymity.REMOVE.value: + question = get_question_by_field_name(field["name"], questions) + if ( + question + and question.question_anonymity == QuestionAnonymity.REMOVE.value + ): continue - select_field_builder.set_sensitive(anonymity is not None) + select_field_builder.set_sensitive(question is not None) fields.append(select_field_builder.build()) sql = ( diff --git a/climmob/processes/odk/api.py b/climmob/processes/odk/api.py index 6f32e70a..90db4cc4 100644 --- a/climmob/processes/odk/api.py +++ b/climmob/processes/odk/api.py @@ -25,6 +25,7 @@ getTheProjectIdForOwner, ) from climmob.processes.db.json import addJsonLog +from climmob.processes.db.question import anonymize_questions log = logging.getLogger(__name__) @@ -388,6 +389,14 @@ def storeJSONInMySQL( projectId, ): schema = userOwner + "_" + projectCod + + with open(JSONFile, "r", encoding="utf-8") as f: + data = json.load(f) + form_id = "-" + if type == "ASS": + form_id = assessmentid + anonymize_questions(request, data, form_id, projectId, schema) + if type == "REG": manifestFile = os.path.join( request.registry.settings["user.repository"], diff --git a/climmob/utility/question.py b/climmob/utility/question.py index 34108c9a..c561d011 100644 --- a/climmob/utility/question.py +++ b/climmob/utility/question.py @@ -1,3 +1,4 @@ +import re from enum import Enum, IntEnum, auto @@ -146,3 +147,16 @@ def get_question_types_with_anonymity_labeled(request): ) result = sorted(result, key=lambda x: x["order"]) return result + + +def get_question_by_field_name(field_name, questions): + for q in questions: + patterns = [ + rf"^{q.question_code}(_[abc])?(_oth)?$", + rf"^perf_{q.question_code}_[123]$", + rf"^char_{q.question_code}_(pos|neg)$", + ] + for pattern in patterns: + if re.fullmatch(pattern, field_name): + return q + return None From fef3abb143c276b066c3c6a1e692dfa39a7848cf Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 11 Aug 2025 10:46:56 -0600 Subject: [PATCH 28/66] add anonymization for dates with year-month --- climmob/processes/db/question.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index e74ca3fd..b00ee527 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -1,5 +1,6 @@ import json import re +from datetime import datetime from sqlalchemy import func, or_, and_ @@ -660,7 +661,11 @@ def anonymize_questions(request, form, form_id, project_id, schema): if i <= field["value"] < (i + params["interval"]): field["value"] = f'{i}-{i + params["interval"]}' i += params["interval"] - + elif field["question"].question_anonymity == QuestionAnonymity.MONTH_YEAR.value: + dt = datetime.fromisoformat(field["value"]) + field["value"] = dt.strftime("%Y-%m") + elif field["question"].question_anonymity == QuestionAnonymity.NOISE.value: + pass value = ( f"(" f"'{form_id}', " From 53a38c7d53067dc633ce23754169c53b775d72dc Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 11 Aug 2025 12:59:58 -0600 Subject: [PATCH 29/66] remove anonymized values when form is canceled --- climmob/processes/db/question.py | 6 ++++++ climmob/views/assessment.py | 4 ++++ climmob/views/registry.py | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index b00ee527..b5849445 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -44,6 +44,7 @@ "knowIfUserHasCreatedTranslations", "get_sensitive_questions_anonymity_by_project_id", "anonymize_questions", + "remove_anonymized_values_by_form_id", ] from climmob.models.climmobv4 import AnonymizationParameter @@ -678,3 +679,8 @@ def anonymize_questions(request, form, form_id, project_id, schema): sql = f"INSERT INTO {schema}.anonymized VALUES {', '.join(anonymized_values)}" sql_execute(sql) + + +def remove_anonymized_values_by_form_id(schema, form_id): + sql = f"DELETE FROM {schema}.anonymized where form_id='{form_id}'" + sql_execute(sql) diff --git a/climmob/views/assessment.py b/climmob/views/assessment.py index b9ef5c26..b945e1a8 100644 --- a/climmob/views/assessment.py +++ b/climmob/views/assessment.py @@ -34,6 +34,7 @@ getPhraseTranslationInLanguage, update_project_status, clone_assessment, + remove_anonymized_values_by_form_id, ) from climmob.products.forms.form import create_document_form from climmob.views.classes import privateView @@ -820,6 +821,9 @@ def processView(self): assessmentid, ) + schema = activeProjectUser + "_" + activeProjectCod + remove_anonymized_values_by_form_id(schema, assessmentid) + self.returnRawViewResult = True return HTTPFound(location=self.request.route_url("dashboard")) diff --git a/climmob/views/registry.py b/climmob/views/registry.py index 4e710471..f3e53532 100644 --- a/climmob/views/registry.py +++ b/climmob/views/registry.py @@ -27,6 +27,7 @@ modifyProjectMainLanguage, projectRegStatus, update_project_status, + remove_anonymized_values_by_form_id, ) from climmob.products import stopTasksByProcess from climmob.views.classes import privateView @@ -165,6 +166,9 @@ def processView(self): "", ) + schema = activeProjectUser + "_" + activeProjectCod + remove_anonymized_values_by_form_id(schema, "-") + self.returnRawViewResult = True return HTTPFound(location=self.request.route_url("dashboard")) From 04e9dde96b6feac7dc6540e540a24f46a9a9ce3d Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Tue, 12 Aug 2025 20:46:43 -0600 Subject: [PATCH 30/66] add duplicate entry handling in anonymized --- climmob/processes/db/question.py | 13 +++++- climmob/processes/odk/api.py | 46 +++++++++++---------- climmob/views/Api/projectAssessmentStart.py | 11 ++++- climmob/views/Api/projectRegistryStart.py | 13 +++++- 4 files changed, 57 insertions(+), 26 deletions(-) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index b5849445..51a6aae8 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -614,7 +614,7 @@ def anonymize_questions(request, form, form_id, project_id, schema): questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) registry_id = ( - form["grp_validation/clc_after"] if form_id == "-" else form["grp_1/QST163"] + form.get("grp_validation/clc_after", form["grp_1/QST162"]) if form_id == "-" else form["grp_1/QST163"] ) pattern = r"grp_\d+/(.+)" @@ -678,7 +678,16 @@ def anonymize_questions(request, form, form_id, project_id, schema): anonymized_values.append(value) sql = f"INSERT INTO {schema}.anonymized VALUES {', '.join(anonymized_values)}" - sql_execute(sql) + try: + sql_execute(sql) + return True, "" + except Exception as e: + match = re.search(rf"Duplicate entry '({form_id})-(\d+)-(.+?)'", str(e)) + if match: + form_name = "registry" if form_id == "-" else f"assessment '{form_id}'" + msg = f"Duplicate entry for package '{match.group(2)}' in {form_name}" + return False, msg + return False, "" def remove_anonymized_values_by_form_id(schema, form_id): diff --git a/climmob/processes/odk/api.py b/climmob/processes/odk/api.py index 90db4cc4..118be3b1 100644 --- a/climmob/processes/odk/api.py +++ b/climmob/processes/odk/api.py @@ -104,7 +104,7 @@ def getFormList(userid, enumerator, request, userOwner=None, projectCod=None): if project.project_regstatus == 1: path = os.path.join( request.registry.settings["user.repository"], - *[project.user_name, project.project_cod, "odk", "reg", "*.json"] + *[project.user_name, project.project_cod, "odk", "reg", "*.json"], ) files = glob.glob(path) if files: @@ -141,7 +141,7 @@ def getFormList(userid, enumerator, request, userOwner=None, projectCod=None): "ass", assessment.ass_cod, "*.json", - ] + ], ) files = glob.glob(path) if files: @@ -173,7 +173,7 @@ def getManifest(user, userOwner, projectId, projectCod, request): if prjdat.project_regstatus == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "reg", "media", "*.*"] + *[userOwner, projectCod, "odk", "reg", "media", "*.*"], ) files = glob.glob(path) @@ -211,7 +211,7 @@ def getAssessmentManifest( if prjdat.ass_status == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "ass", assessmentid, "media", "*.*"] + *[userOwner, projectCod, "odk", "ass", assessmentid, "media", "*.*"], ) else: raise HTTPNotFound() @@ -247,7 +247,7 @@ def getXMLForm(userOwner, projectId, projectCod, request): if prjdat.project_regstatus == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "reg", "*.xml"] + *[userOwner, projectCod, "odk", "reg", "*.xml"], ) files = glob.glob(path) @@ -271,7 +271,7 @@ def getAssessmentXMLForm(userOwner, projectId, projectCod, assessmentid, request if prjdat.ass_status == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "ass", assessmentid, "*.xml"] + *[userOwner, projectCod, "odk", "ass", assessmentid, "*.xml"], ) else: raise HTTPNotFound() @@ -293,7 +293,7 @@ def getMediaFile(userOwner, projectId, projectCod, fileid, request): if prjdat.project_regstatus == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "reg", "media", fileid] + *[userOwner, projectCod, "odk", "reg", "media", fileid], ) else: raise HTTPNotFound() @@ -319,7 +319,7 @@ def getAssessmentMediaFile( if prjdat.ass_status == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "ass", assessmentid, "media", fileid] + *[userOwner, projectCod, "odk", "ass", assessmentid, "media", fileid], ) else: raise HTTPNotFound() @@ -395,22 +395,24 @@ def storeJSONInMySQL( form_id = "-" if type == "ASS": form_id = assessmentid - anonymize_questions(request, data, form_id, projectId, schema) + success, msg = anonymize_questions(request, data, form_id, projectId, schema) + if not success: + return False, msg if type == "REG": manifestFile = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "db", "reg", "manifest.xml"] + *[userOwner, projectCod, "db", "reg", "manifest.xml"], ) jsFile = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "db", "reg", "custom.js"] + *[userOwner, projectCod, "db", "reg", "custom.js"], ) else: manifestFile = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "db", "ass", assessmentid, "manifest.xml"] + *[userOwner, projectCod, "db", "ass", assessmentid, "manifest.xml"], ) jsFile = "" @@ -471,7 +473,7 @@ def storeJSONInMySQL( projectId, ) - return True + return True, "" def convertXMLToJSON( @@ -489,12 +491,12 @@ def convertXMLToJSON( if submissionType == "REG": path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "reg", "*.xml"] + *[userOwner, projectCod, "odk", "reg", "*.xml"], ) if submissionType == "ASS": path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "ass", assessmentID, "*.xml"] + *[userOwner, projectCod, "odk", "ass", assessmentID, "*.xml"], ) files = glob.glob(path) @@ -632,7 +634,7 @@ def storeSubmission(userid, userEnum, request): pathTemp = os.path.join( request.registry.settings["user.repository"], - *[userid, "data", "xml", str(iniqueIDTemp)] + *[userid, "data", "xml", str(iniqueIDTemp)], ) os.makedirs(pathTemp) @@ -673,7 +675,7 @@ def storeSubmission(userid, userEnum, request): dirs = glob.glob( os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "data", "reg", "xml"] + *[userOwner, projectCod, "data", "reg", "xml"], ) + "/*", recursive=True, @@ -682,7 +684,7 @@ def storeSubmission(userid, userEnum, request): dirs = glob.glob( os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "data", "ass", assessmentID, "xml"] + *[userOwner, projectCod, "data", "ass", assessmentID, "xml"], ) + "/*", recursive=True, @@ -697,14 +699,14 @@ def storeSubmission(userid, userEnum, request): if submissionType == "REG": path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "data", "reg", "xml", str(iniqueID)] + *[userOwner, projectCod, "data", "reg", "xml", str(iniqueID)], ) if not os.path.exists(path): os.makedirs(path) os.makedirs( os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "data", "reg", "json", str(iniqueID)] + *[userOwner, projectCod, "data", "reg", "json", str(iniqueID)], ) ) @@ -719,7 +721,7 @@ def storeSubmission(userid, userEnum, request): assessmentID, "xml", str(iniqueID), - ] + ], ) if not os.path.exists(path): os.makedirs(path) @@ -734,7 +736,7 @@ def storeSubmission(userid, userEnum, request): assessmentID, "json", str(iniqueID), - ] + ], ) ) diff --git a/climmob/views/Api/projectAssessmentStart.py b/climmob/views/Api/projectAssessmentStart.py index 35ae2976..d673cb3c 100644 --- a/climmob/views/Api/projectAssessmentStart.py +++ b/climmob/views/Api/projectAssessmentStart.py @@ -759,7 +759,7 @@ def ApiAssessmentPushProcess(self, structure, dataworking, activeProjectId): f.write(json.dumps(_json)) f.close() - storeJSONInMySQL( + success, msg = storeJSONInMySQL( self.user.login, "ASS", dataworking["user_owner"], @@ -771,6 +771,15 @@ def ApiAssessmentPushProcess(self, structure, dataworking, activeProjectId): activeProjectId, ) + if not success: + response = Response( + status=401, + body=self._( + "The data could not be saved. ERROR: " + msg + ), + ) + return response + logFile = pathfinal.replace(".json", ".log") if os.path.exists(logFile): doc = minidom.parse(logFile) diff --git a/climmob/views/Api/projectRegistryStart.py b/climmob/views/Api/projectRegistryStart.py index ec2eef78..5020dae6 100644 --- a/climmob/views/Api/projectRegistryStart.py +++ b/climmob/views/Api/projectRegistryStart.py @@ -1226,7 +1226,8 @@ def ApiRegistrationPushProcess(self, structure, dataworking, activeProjectId): f = open(pathfinal, "w") f.write(json.dumps(_json)) f.close() - storeJSONInMySQL( + + success, msg = storeJSONInMySQL( self.user.login, "REG", dataworking["user_owner"], @@ -1238,6 +1239,16 @@ def ApiRegistrationPushProcess(self, structure, dataworking, activeProjectId): activeProjectId, ) + if not success: + response = Response( + status=401, + body=self._( + "The data could not be registered. ERROR: " + + msg + ), + ) + return response + logFile = pathfinal.replace(".json", ".log") if os.path.exists(logFile): doc = minidom.parse(logFile) From 9f18f7949500ef8c483f66ad4167423447fae10a Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 13 Aug 2025 22:15:07 -0600 Subject: [PATCH 31/66] add geo noise anonymization --- climmob/processes/db/question.py | 19 ++++++++++--- climmob/utility/__init__.py | 1 + climmob/utility/anonymization.py | 46 ++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 climmob/utility/anonymization.py diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index 51a6aae8..ba8fd2fa 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -49,7 +49,12 @@ from climmob.models.climmobv4 import AnonymizationParameter from climmob.models.repository import sql_execute -from climmob.utility import get_question_by_field_name, QuestionAnonymity, QuestionType +from climmob.utility import ( + get_question_by_field_name, + QuestionAnonymity, + QuestionType, + add_noise_to_gps_coordinates, +) log = logging.getLogger(__name__) @@ -614,7 +619,9 @@ def anonymize_questions(request, form, form_id, project_id, schema): questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) registry_id = ( - form.get("grp_validation/clc_after", form["grp_1/QST162"]) if form_id == "-" else form["grp_1/QST163"] + form.get("grp_validation/clc_after", form["grp_1/QST162"]) + if form_id == "-" + else form["grp_1/QST163"] ) pattern = r"grp_\d+/(.+)" @@ -666,7 +673,13 @@ def anonymize_questions(request, form, form_id, project_id, schema): dt = datetime.fromisoformat(field["value"]) field["value"] = dt.strftime("%Y-%m") elif field["question"].question_anonymity == QuestionAnonymity.NOISE.value: - pass + geo_point = field["value"].split() + geo_point[0], geo_point[1] = add_noise_to_gps_coordinates( + float(geo_point[0]), float(geo_point[1]), 3000 + ) + if geo_point[0] == "Error" or geo_point[1] == "Error": + return False, "Could not anonymize GeoPoint" + field["value"] = " ".join(geo_point) value = ( f"(" f"'{form_id}', " diff --git a/climmob/utility/__init__.py b/climmob/utility/__init__.py index b68006cb..62e489f4 100644 --- a/climmob/utility/__init__.py +++ b/climmob/utility/__init__.py @@ -1,6 +1,7 @@ from climmob.utility.helpers import * from climmob.utility.factory import * from climmob.utility.question import * +from climmob.utility.anonymization import * def get_enum_as_dict(enum): diff --git a/climmob/utility/anonymization.py b/climmob/utility/anonymization.py new file mode 100644 index 00000000..41b11cf1 --- /dev/null +++ b/climmob/utility/anonymization.py @@ -0,0 +1,46 @@ +import math +import random + + +def add_noise_to_gps_coordinates(lat, lon, radius): + """ + Add noise to a geographical coordinate by choosing a random point within a radius. + + Parameters: + lat (float): Latitude of the original coordinate. + lon (float): Longitude of the original coordinate. + radius (float): Radius in meters within which to choose a random point. + + Returns: + tuple: A tuple containing the new latitude and longitude. + """ + try: + # Earth radius in meters + earth_radius = 6378137 + + # Convert radius from meters to degrees latitude + radius_lat = radius / (earth_radius * (math.pi / 180)) + + # Convert radius from meters to degrees longitude, adjusted by latitude + radius_lon = radius / ( + earth_radius * (math.pi / 180) * math.cos(math.radians(lat)) + ) + + # Random angle in radians + angle = random.uniform(0, 2 * math.pi) + + # Random distance factor for uniform distribution in a circle + factor = math.sqrt(random.uniform(0, 1)) + + # Calculate deltas + delta_lat = factor * radius_lat * math.cos(angle) + delta_lon = factor * radius_lon * math.sin(angle) + + # New latitude and longitude + new_lat = lat + delta_lat + new_lon = lon + delta_lon + + return str(new_lat), str(new_lon) + except Exception as e: + print(e) + return "Error", "Error" From 6ee884aae08482ece442e078583e23bc733b5579 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 14 Aug 2025 16:50:50 -0600 Subject: [PATCH 32/66] fix template format --- climmob/templates/question/library.jinja2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/climmob/templates/question/library.jinja2 b/climmob/templates/question/library.jinja2 index f44d8435..4b22dd2b 100644 --- a/climmob/templates/question/library.jinja2 +++ b/climmob/templates/question/library.jinja2 @@ -649,7 +649,12 @@ function showDeleteQuestion() { var urlAction = '{{ request.application_url }}/question/'+$('#question_id').val()+'/delete' - showDelete(urlAction,'{{ _("Do you really want to remove this question ?") }}','{{ request.session.get_csrf_token() }}', {% if nextPage %}"{{ request.route_url("qlibrary", user_name=activeUser.login, _query={'next': nextPage} ) }}" {% else %} "{{ request.route_url("qlibrary", user_name=activeUser.login ) }}" {% endif %}) + showDelete( + urlAction, + '{{ _("Do you really want to remove this question ?") }}', + '{{ request.session.get_csrf_token() }}', + "{% if nextPage %}{{ request.route_url("qlibrary", user_name=activeUser.login, _query={'next': nextPage} ) }}{% else %}{{ request.route_url("qlibrary", user_name=activeUser.login ) }}{% endif %}" + ) $(this).parent().parent().remove(); } From 36a58e70d9cdef13566a6731860880ace3685d31 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 18 Aug 2025 16:50:09 -0600 Subject: [PATCH 33/66] add foreign key to anonymized referencing REG --- climmob/processes/odk/generator.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/climmob/processes/odk/generator.py b/climmob/processes/odk/generator.py index a8db29c0..d5007be3 100644 --- a/climmob/processes/odk/generator.py +++ b/climmob/processes/odk/generator.py @@ -71,19 +71,22 @@ def create_schema(schema, cnf_file): " DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci", ] error = execute_command(args, "Error creating schema") - if error: - return error + return error +def create_anonymized_table(schema, cnf_file): args = [ "mysql", f"--defaults-file={cnf_file}", f"--execute=CREATE TABLE IF NOT EXISTS {schema}.anonymized " "(`form_id` varchar(255) NOT NULL," - "`reg_id` int NOT NULL," + "`reg_id` varchar(255) NOT NULL," "`col_name` varchar(255) NOT NULL," "`value` varchar(255) DEFAULT NULL," - "PRIMARY KEY (`form_id`,`reg_id`,`col_name`)" - ") ENGINE=InnoDB DEFAULT CHARSET=utf8;", + "PRIMARY KEY (`form_id`,`reg_id`,`col_name`), " + "CONSTRAINT fk_anonymized_reg_id_REG_geninfo FOREIGN KEY (reg_id) " + f"REFERENCES {schema}.REG_geninfo(qst162) " + "ON DELETE CASCADE ON UPDATE CASCADE" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;", ] error = execute_command(args, "Error creating anonymized table") @@ -119,6 +122,11 @@ def buildDatabase( if error: return error + if dropSchema: + error = create_anonymized_table(schema, cnfFile) + if error: + return error + print(f"****buildDatabase**Inserting into lookup tables for {schema}******") with open(insertFile) as input_file: proc = Popen(args, stdin=input_file, stderr=PIPE, stdout=PIPE) From 8a11034ca99760aca2e2db9dea6b72a62c2aedf8 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 18 Aug 2025 16:53:16 -0600 Subject: [PATCH 34/66] add package number extraction from form --- climmob/processes/db/question.py | 33 +++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index ba8fd2fa..50e3e8f8 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -615,14 +615,10 @@ def get_sensitive_questions_anonymity_by_project_id(project_id, request): return query.all() -def anonymize_questions(request, form, form_id, project_id, schema): +def anonymize_questions(request, form, form_id, project_id, user_owner, project_cod): questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) - registry_id = ( - form.get("grp_validation/clc_after", form["grp_1/QST162"]) - if form_id == "-" - else form["grp_1/QST163"] - ) + registry_id = None pattern = r"grp_\d+/(.+)" to_anonymize = [] @@ -631,10 +627,19 @@ def anonymize_questions(request, form, form_id, project_id, schema): match = re.fullmatch(pattern, key) if not match: continue - question = get_question_by_field_name(match.group(1), questions) + field_name = match.group(1) + + if field_name == "QST162" or field_name == "QST163": + match = re.fullmatch(rf"({user_owner}-)?(\d+)(-{project_cod}~)?", form[key]) + if not match: + return False, "Could not anonymize" + registry_id = match.group(2) + continue + + question = get_question_by_field_name(field_name, questions) if question and question.question_anonymity != QuestionAnonymity.REMOVE.value: to_anonymize.append( - {"field_name": match.group(1), "value": form[key], "question": question} + {"field_name": field_name, "value": form[key], "question": question} ) if not to_anonymize: @@ -649,11 +654,11 @@ def anonymize_questions(request, form, form_id, project_id, schema): if field["question"].question_anonymity == QuestionAnonymity.PSEUDONYM.value: field["value"] = params["pseudonym"].replace("{}", registry_id) elif field["question"].question_anonymity == QuestionAnonymity.RANGE.value: - parser = ( - int - if field["question"].question_dtype == QuestionType.INTEGER.value - else float - ) + if field["question"].question_dtype == QuestionType.INTEGER.value: + parser = int + else: + parser = float + field["value"] = parser(field["value"]) params["lower_bound"] = parser(params["lower_bound"]) params["upper_bound"] = parser(params["upper_bound"]) @@ -668,6 +673,7 @@ def anonymize_questions(request, form, form_id, project_id, schema): while i < params["upper_bound"]: if i <= field["value"] < (i + params["interval"]): field["value"] = f'{i}-{i + params["interval"]}' + break i += params["interval"] elif field["question"].question_anonymity == QuestionAnonymity.MONTH_YEAR.value: dt = datetime.fromisoformat(field["value"]) @@ -690,6 +696,7 @@ def anonymize_questions(request, form, form_id, project_id, schema): ) anonymized_values.append(value) + schema = user_owner + "_" + project_cod sql = f"INSERT INTO {schema}.anonymized VALUES {', '.join(anonymized_values)}" try: sql_execute(sql) From d3763ef68adc8c7eaec1fb72c8cc7dad332ab345 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 18 Aug 2025 16:56:46 -0600 Subject: [PATCH 35/66] move anonymization to perform after --- climmob/processes/odk/api.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/climmob/processes/odk/api.py b/climmob/processes/odk/api.py index 118be3b1..a9c1fd7b 100644 --- a/climmob/processes/odk/api.py +++ b/climmob/processes/odk/api.py @@ -390,15 +390,6 @@ def storeJSONInMySQL( ): schema = userOwner + "_" + projectCod - with open(JSONFile, "r", encoding="utf-8") as f: - data = json.load(f) - form_id = "-" - if type == "ASS": - form_id = assessmentid - success, msg = anonymize_questions(request, data, form_id, projectId, schema) - if not success: - return False, msg - if type == "REG": manifestFile = os.path.join( request.registry.settings["user.repository"], @@ -473,6 +464,15 @@ def storeJSONInMySQL( projectId, ) + with open(JSONFile, "r", encoding="utf-8") as f: + data = json.load(f) + form_id = "-" + if type == "ASS": + form_id = assessmentid + success, msg = anonymize_questions(request, data, form_id, projectId, userOwner, projectCod) + if not success: + return False, msg + return True, "" From 55450e31f577e811149076a8290df08c39c0a257 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 21 Aug 2025 13:04:20 -0600 Subject: [PATCH 36/66] remove foreign key to anonymized referencing REG --- climmob/processes/odk/generator.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/climmob/processes/odk/generator.py b/climmob/processes/odk/generator.py index d5007be3..c2dd9243 100644 --- a/climmob/processes/odk/generator.py +++ b/climmob/processes/odk/generator.py @@ -82,10 +82,7 @@ def create_anonymized_table(schema, cnf_file): "`reg_id` varchar(255) NOT NULL," "`col_name` varchar(255) NOT NULL," "`value` varchar(255) DEFAULT NULL," - "PRIMARY KEY (`form_id`,`reg_id`,`col_name`), " - "CONSTRAINT fk_anonymized_reg_id_REG_geninfo FOREIGN KEY (reg_id) " - f"REFERENCES {schema}.REG_geninfo(qst162) " - "ON DELETE CASCADE ON UPDATE CASCADE" + "PRIMARY KEY (`form_id`,`reg_id`,`col_name`)" ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;", ] From 8e6597024507ff47f77f19289ebbc516a9b4d86d Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 22 Aug 2025 09:37:59 -0600 Subject: [PATCH 37/66] refactor anonymization db processes: change location --- climmob/processes/__init__.py | 2 + climmob/processes/db/anonymization_params.py | 50 ++++++ climmob/processes/db/anonymized.py | 134 ++++++++++++++++ climmob/processes/db/question.py | 155 +------------------ climmob/processes/db/registry.py | 3 + climmob/processes/db/results.py | 2 + climmob/processes/odk/api.py | 6 +- climmob/processes/odk/generator.py | 1 + climmob/views/assessment.py | 4 +- climmob/views/registry.py | 4 +- 10 files changed, 202 insertions(+), 159 deletions(-) create mode 100644 climmob/processes/db/anonymization_params.py create mode 100644 climmob/processes/db/anonymized.py diff --git a/climmob/processes/__init__.py b/climmob/processes/__init__.py index a73bd751..540f2719 100644 --- a/climmob/processes/__init__.py +++ b/climmob/processes/__init__.py @@ -50,3 +50,5 @@ from climmob.processes.db.project_location_unit_objective import * from climmob.processes.db.location_unit_of_analysis_objectives import * from climmob.processes.db.affiliation import * +from climmob.processes.db.anonymized import * +from climmob.processes.db.anonymization_params import * diff --git a/climmob/processes/db/anonymization_params.py b/climmob/processes/db/anonymization_params.py new file mode 100644 index 00000000..bab999b1 --- /dev/null +++ b/climmob/processes/db/anonymization_params.py @@ -0,0 +1,50 @@ +__all__ = ["save_anonymization_params", "get_anonymization_params_as_dict"] + +import re + +from climmob.models import mapFromSchema +from climmob.models.climmobv4 import AnonymizationParameter + + +def get_anonymization_params(question_id, request): + result = mapFromSchema( + request.dbsession.query(AnonymizationParameter) + .filter(AnonymizationParameter.question_id == question_id) + .all() + ) + return result + + +def get_anonymization_params_as_dict(question_id, request): + params = get_anonymization_params(question_id, request) + result = {} + for param in params: + result[param["name"]] = param["value"] + return result + + +def save_anonymization_params(question_id, data, request): + delete_existing_anonymization_params(question_id, request) + + params = [] + for key in data.keys(): + pattern = r"anonym_param_([a-z_]+)" + match = re.match(pattern, key) + if match: + params.append({"name": match.group(1), "value": data[key]}) + + for param in params: + new_param = AnonymizationParameter(**param) + new_param.question_id = question_id + request.dbsession.add(new_param) + request.dbsession.flush() + + +def delete_existing_anonymization_params(question_id, request): + try: + request.dbsession.query(AnonymizationParameter).filter( + AnonymizationParameter.question_id == question_id + ).delete() + return True, "" + except Exception as e: + return False, str(e) diff --git a/climmob/processes/db/anonymized.py b/climmob/processes/db/anonymized.py new file mode 100644 index 00000000..8050dcfc --- /dev/null +++ b/climmob/processes/db/anonymized.py @@ -0,0 +1,134 @@ +import re +from datetime import datetime + +from climmob.models.repository import sql_execute +from climmob.processes.db.anonymization_params import get_anonymization_params_as_dict +from climmob.processes.db.question import ( + get_sensitive_questions_anonymity_by_project_id, +) +from climmob.utility import ( + get_question_by_field_name, + QuestionAnonymity, + add_noise_to_gps_coordinates, + QuestionType, +) + +__all__ = [ + "anonymize_questions", + "delete_anonymized_values_by_form_id", + "delete_anonymized_values_by_form_id_and_reg_id", +] + + +def anonymize_questions(request, form, form_id, project_id, user_owner, project_cod): + questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) + + registry_id = None + + schema = user_owner + "_" + project_cod + + pattern = r"grp_\d+/(.+)" + to_anonymize = [] + + for key in form.keys(): + match = re.fullmatch(pattern, key) + if not match: + continue + field_name = match.group(1) + + if field_name == "QST162" or field_name == "QST163": + match = re.fullmatch(rf"({user_owner}-)?(\d+)(-{project_cod}~)?", form[key]) + if not match: + return False, "Could not anonymize" + registry_id = match.group(2) + continue + + question = get_question_by_field_name(field_name, questions) + if question and question.question_anonymity != QuestionAnonymity.REMOVE.value: + to_anonymize.append( + {"field_name": field_name, "value": form[key], "question": question} + ) + + if not to_anonymize: + return True + + for field in to_anonymize: + params = get_anonymization_params_as_dict( + field["question"].question_id, request + ) + if field["question"].question_anonymity == QuestionAnonymity.PSEUDONYM.value: + field["value"] = params["pseudonym"].replace("{}", registry_id) + elif field["question"].question_anonymity == QuestionAnonymity.RANGE.value: + if field["question"].question_dtype == QuestionType.INTEGER.value: + parser = int + else: + parser = float + + field["value"] = parser(field["value"]) + params["lower_bound"] = parser(params["lower_bound"]) + params["upper_bound"] = parser(params["upper_bound"]) + params["interval"] = parser(params["interval"]) + + if field["value"] < params["lower_bound"]: + field["value"] = f'<{params["lower_bound"]}' + elif field["value"] > params["upper_bound"]: + field["value"] = f'>{params["upper_bound"]}' + else: + i = params["lower_bound"] + while i < params["upper_bound"]: + if i <= field["value"] < (i + params["interval"]): + field["value"] = f'{i}-{i + params["interval"]}' + break + i += params["interval"] + elif field["question"].question_anonymity == QuestionAnonymity.MONTH_YEAR.value: + dt = datetime.fromisoformat(field["value"]) + field["value"] = dt.strftime("%Y-%m") + elif field["question"].question_anonymity == QuestionAnonymity.NOISE.value: + geo_point = field["value"].split() + geo_point[0], geo_point[1] = add_noise_to_gps_coordinates( + float(geo_point[0]), float(geo_point[1]), 3000 + ) + if geo_point[0] == "Error" or geo_point[1] == "Error": + return False, "Could not anonymize GeoPoint" + field["value"] = " ".join(geo_point) + field["sql_insert_value"] = ( + f"(" + f"'{form_id}', " + f"'{registry_id}', " + f"'{field['field_name']}', " + f"'{field['value']}'" + f")" + ) + success, msg = execute_anonymization(field, form_id, schema) + if not success: + return False, msg + + return True, "" + + +def execute_anonymization(field, form_id, schema): + sql = f"INSERT INTO {schema}.anonymized VALUES {field['sql_insert_value']}" + try: + sql_execute(sql) + return True, "" + except Exception as e: + match = re.search(rf"Duplicate entry '({form_id})-(\d+)-(.+?)'", str(e)) + if match: + form_name = "registry" if form_id == "-" else f"assessment '{form_id}'" + msg = f"Duplicate entry for package '{match.group(2)}' in {form_name}" + return False, msg + return False, "" + + +def delete_anonymized_values_by_form_id(schema, form_id): + sql = f"DELETE FROM {schema}.anonymized where form_id='{form_id}'" + sql_execute(sql) + + +def delete_anonymized_values_by_form_id_and_reg_id(schema, form_id, reg_id): + query = ( + f"DELETE FROM {schema}.anonymized " + f"WHERE form_id='{form_id}' " + f"AND reg_id='{reg_id}'" + ) + sql_execute(query) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index 50e3e8f8..01686449 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -43,64 +43,13 @@ "getQuestionOwner", "knowIfUserHasCreatedTranslations", "get_sensitive_questions_anonymity_by_project_id", - "anonymize_questions", - "remove_anonymized_values_by_form_id", ] from climmob.models.climmobv4 import AnonymizationParameter -from climmob.models.repository import sql_execute -from climmob.utility import ( - get_question_by_field_name, - QuestionAnonymity, - QuestionType, - add_noise_to_gps_coordinates, -) - -log = logging.getLogger(__name__) - - -def get_anonymization_params(question_id, request): - result = mapFromSchema( - request.dbsession.query(AnonymizationParameter) - .filter(AnonymizationParameter.question_id == question_id) - .all() - ) - return result - - -def get_anonymization_params_as_dict(question_id, request): - params = get_anonymization_params(question_id, request) - result = {} - for param in params: - result[param["name"]] = param["value"] - return result - - -def save_anonymization_params(question_id, data, request): - delete_existing_anonymization_params(question_id, request) - - params = [] - for key in data.keys(): - pattern = r"anonym_param_([a-z_]+)" - match = re.match(pattern, key) - if match: - params.append({"name": match.group(1), "value": data[key]}) - - for param in params: - new_param = AnonymizationParameter(**param) - new_param.question_id = question_id - request.dbsession.add(new_param) - request.dbsession.flush() +from climmob.processes.db.anonymization_params import save_anonymization_params -def delete_existing_anonymization_params(question_id, request): - try: - request.dbsession.query(AnonymizationParameter).filter( - AnonymizationParameter.question_id == question_id - ).delete() - return True, "" - except Exception as e: - return False, str(e) +log = logging.getLogger(__name__) def addQuestion(data, request): @@ -613,103 +562,3 @@ def get_sensitive_questions_anonymity_by_project_id(project_id, request): ) ) return query.all() - - -def anonymize_questions(request, form, form_id, project_id, user_owner, project_cod): - questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) - - registry_id = None - - pattern = r"grp_\d+/(.+)" - to_anonymize = [] - - for key in form.keys(): - match = re.fullmatch(pattern, key) - if not match: - continue - field_name = match.group(1) - - if field_name == "QST162" or field_name == "QST163": - match = re.fullmatch(rf"({user_owner}-)?(\d+)(-{project_cod}~)?", form[key]) - if not match: - return False, "Could not anonymize" - registry_id = match.group(2) - continue - - question = get_question_by_field_name(field_name, questions) - if question and question.question_anonymity != QuestionAnonymity.REMOVE.value: - to_anonymize.append( - {"field_name": field_name, "value": form[key], "question": question} - ) - - if not to_anonymize: - return True - - anonymized_values = [] - - for field in to_anonymize: - params = get_anonymization_params_as_dict( - field["question"].question_id, request - ) - if field["question"].question_anonymity == QuestionAnonymity.PSEUDONYM.value: - field["value"] = params["pseudonym"].replace("{}", registry_id) - elif field["question"].question_anonymity == QuestionAnonymity.RANGE.value: - if field["question"].question_dtype == QuestionType.INTEGER.value: - parser = int - else: - parser = float - - field["value"] = parser(field["value"]) - params["lower_bound"] = parser(params["lower_bound"]) - params["upper_bound"] = parser(params["upper_bound"]) - params["interval"] = parser(params["interval"]) - - if field["value"] < params["lower_bound"]: - field["value"] = f'<{params["lower_bound"]}' - elif field["value"] > params["upper_bound"]: - field["value"] = f'>{params["upper_bound"]}' - else: - i = params["lower_bound"] - while i < params["upper_bound"]: - if i <= field["value"] < (i + params["interval"]): - field["value"] = f'{i}-{i + params["interval"]}' - break - i += params["interval"] - elif field["question"].question_anonymity == QuestionAnonymity.MONTH_YEAR.value: - dt = datetime.fromisoformat(field["value"]) - field["value"] = dt.strftime("%Y-%m") - elif field["question"].question_anonymity == QuestionAnonymity.NOISE.value: - geo_point = field["value"].split() - geo_point[0], geo_point[1] = add_noise_to_gps_coordinates( - float(geo_point[0]), float(geo_point[1]), 3000 - ) - if geo_point[0] == "Error" or geo_point[1] == "Error": - return False, "Could not anonymize GeoPoint" - field["value"] = " ".join(geo_point) - value = ( - f"(" - f"'{form_id}', " - f"'{registry_id}', " - f"'{field['field_name']}', " - f"'{field['value']}'" - f")" - ) - anonymized_values.append(value) - - schema = user_owner + "_" + project_cod - sql = f"INSERT INTO {schema}.anonymized VALUES {', '.join(anonymized_values)}" - try: - sql_execute(sql) - return True, "" - except Exception as e: - match = re.search(rf"Duplicate entry '({form_id})-(\d+)-(.+?)'", str(e)) - if match: - form_name = "registry" if form_id == "-" else f"assessment '{form_id}'" - msg = f"Duplicate entry for package '{match.group(2)}' in {form_name}" - return False, msg - return False, "" - - -def remove_anonymized_values_by_form_id(schema, form_id): - sql = f"DELETE FROM {schema}.anonymized where form_id='{form_id}'" - sql_execute(sql) diff --git a/climmob/processes/db/registry.py b/climmob/processes/db/registry.py index a55f69ec..f5ad3dc1 100644 --- a/climmob/processes/db/registry.py +++ b/climmob/processes/db/registry.py @@ -6,6 +6,9 @@ from climmob.models import Regsection, Registry, Project, Question, userProject from climmob.models.schema import mapFromSchema, mapToSchema from climmob.processes import addRegistryQuestionsToProject +from climmob.processes.db.anonymized import ( + delete_anonymized_values_by_form_id_and_reg_id, +) from climmob.processes.db.assessment import setAssessmentStatus, formattingQuestions import climmob.plugins as p diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index a689e9e9..25a2484e 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -9,6 +9,8 @@ from climmob.models.repository import sql_fetch_all, sql_fetch_one from climmob.processes import ( getCombinations, +) +from climmob.processes.db.question import ( get_sensitive_questions_anonymity_by_project_id, ) diff --git a/climmob/processes/odk/api.py b/climmob/processes/odk/api.py index a9c1fd7b..8e20fbd3 100644 --- a/climmob/processes/odk/api.py +++ b/climmob/processes/odk/api.py @@ -25,7 +25,7 @@ getTheProjectIdForOwner, ) from climmob.processes.db.json import addJsonLog -from climmob.processes.db.question import anonymize_questions +from climmob.processes.db.anonymized import anonymize_questions log = logging.getLogger(__name__) @@ -469,7 +469,9 @@ def storeJSONInMySQL( form_id = "-" if type == "ASS": form_id = assessmentid - success, msg = anonymize_questions(request, data, form_id, projectId, userOwner, projectCod) + success, msg = anonymize_questions( + request, data, form_id, projectId, userOwner, projectCod + ) if not success: return False, msg diff --git a/climmob/processes/odk/generator.py b/climmob/processes/odk/generator.py index c2dd9243..960f1a70 100644 --- a/climmob/processes/odk/generator.py +++ b/climmob/processes/odk/generator.py @@ -73,6 +73,7 @@ def create_schema(schema, cnf_file): error = execute_command(args, "Error creating schema") return error + def create_anonymized_table(schema, cnf_file): args = [ "mysql", diff --git a/climmob/views/assessment.py b/climmob/views/assessment.py index b945e1a8..527fd558 100644 --- a/climmob/views/assessment.py +++ b/climmob/views/assessment.py @@ -34,7 +34,7 @@ getPhraseTranslationInLanguage, update_project_status, clone_assessment, - remove_anonymized_values_by_form_id, + delete_anonymized_values_by_form_id, ) from climmob.products.forms.form import create_document_form from climmob.views.classes import privateView @@ -822,7 +822,7 @@ def processView(self): ) schema = activeProjectUser + "_" + activeProjectCod - remove_anonymized_values_by_form_id(schema, assessmentid) + delete_anonymized_values_by_form_id(schema, assessmentid) self.returnRawViewResult = True return HTTPFound(location=self.request.route_url("dashboard")) diff --git a/climmob/views/registry.py b/climmob/views/registry.py index f3e53532..2b5301a0 100644 --- a/climmob/views/registry.py +++ b/climmob/views/registry.py @@ -27,7 +27,7 @@ modifyProjectMainLanguage, projectRegStatus, update_project_status, - remove_anonymized_values_by_form_id, + delete_anonymized_values_by_form_id, ) from climmob.products import stopTasksByProcess from climmob.views.classes import privateView @@ -167,7 +167,7 @@ def processView(self): ) schema = activeProjectUser + "_" + activeProjectCod - remove_anonymized_values_by_form_id(schema, "-") + delete_anonymized_values_by_form_id(schema, "-") self.returnRawViewResult = True return HTTPFound(location=self.request.route_url("dashboard")) From b5ad929ce011e65dd8c5b97eab8b7934bc48cd79 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 22 Aug 2025 09:41:22 -0600 Subject: [PATCH 38/66] add db processes to delete registry and assessment data --- climmob/processes/db/assessment.py | 24 +++++++++++++++++++++++- climmob/processes/db/registry.py | 12 ++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/climmob/processes/db/assessment.py b/climmob/processes/db/assessment.py index 65f1f2d4..521626d3 100644 --- a/climmob/processes/db/assessment.py +++ b/climmob/processes/db/assessment.py @@ -21,8 +21,11 @@ I18nQstoption, ) -from climmob.models.repository import sql_fetch_one, sql_execute +from climmob.models.repository import sql_fetch_one, sql_execute, execute_two_sqls from climmob.models.schema import mapFromSchema, mapToSchema +from climmob.processes.db.anonymized import ( + delete_anonymized_values_by_form_id_and_reg_id, +) from climmob.processes.db.project import ( addQuestionsToAssessment, numberOfCombinationsForTheProject, @@ -82,6 +85,7 @@ "clone_assessment", "copy_assessment_questions", "copy_assessment_sections", + "delete_assessment_data_by_qst163", ] log = logging.getLogger(__name__) @@ -1746,3 +1750,21 @@ def getFinalizedAssessments(request, userOwner, projectCod, projectId): ) return result + + +def delete_assessment_data_by_qst163(schema, ass_id, qst163, odk_user): + query = ( + "DELETE FROM " + + schema + + ".ASS" + + ass_id + + "_geninfo WHERE qst163='" + + qst163 + + "'" + ) + execute_two_sqls( + "SET @odktools_current_user = '" + odk_user + "'; ", + query, + ) + + delete_anonymized_values_by_form_id_and_reg_id(schema, ass_id, qst163) diff --git a/climmob/processes/db/registry.py b/climmob/processes/db/registry.py index f5ad3dc1..a245d889 100644 --- a/climmob/processes/db/registry.py +++ b/climmob/processes/db/registry.py @@ -4,6 +4,7 @@ from sqlalchemy import func from climmob.models import Regsection, Registry, Project, Question, userProject +from climmob.models.repository import execute_two_sqls from climmob.models.schema import mapFromSchema, mapToSchema from climmob.processes import addRegistryQuestionsToProject from climmob.processes.db.anonymized import ( @@ -41,6 +42,7 @@ "getTheGroupOfThePackageCode", "registryHaveQuestionOfMultimediaType", "deleteRegistryByProjectId", + "delete_registry_data_by_qst162", ] @@ -587,3 +589,13 @@ def registryHaveQuestionOfMultimediaType(request, projectId): return True else: return False + + +def delete_registry_data_by_qst162(schema, qst162, odk_user): + query = "DELETE FROM " + schema + ".REG_geninfo WHERE qst162='" + qst162 + "'" + execute_two_sqls( + "SET @odktools_current_user = '" + odk_user + "'; ", + query, + ) + + delete_anonymized_values_by_form_id_and_reg_id(schema, "-", qst162) From ec9a1ed49172d6026603b9e502856a90746326e2 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 22 Aug 2025 09:42:02 -0600 Subject: [PATCH 39/66] refactor deletion of registry and assessment data --- climmob/views/cleanErrorLogs.py | 85 +++++++++------------------------ 1 file changed, 23 insertions(+), 62 deletions(-) diff --git a/climmob/views/cleanErrorLogs.py b/climmob/views/cleanErrorLogs.py index 55e08bb6..81da6ae5 100644 --- a/climmob/views/cleanErrorLogs.py +++ b/climmob/views/cleanErrorLogs.py @@ -17,6 +17,8 @@ getTheProjectIdForOwner, getActiveProject, getQuestionsStructure, + delete_assessment_data_by_qst163, + delete_registry_data_by_qst162, ) from climmob.processes.odk.api import storeJSONInMySQL from climmob.views.classes import privateView @@ -31,6 +33,8 @@ def processView(self): activeProjectUser = self.request.matchdict["user"] activeProjectCod = self.request.matchdict["project"] + schema = activeProjectUser + "_" + activeProjectCod + activeProjectId = getTheProjectIdForOwner( activeProjectUser, activeProjectCod, self.request ) @@ -94,21 +98,10 @@ def processView(self): if str(dataworking["txt_oldvalue"]) == str( dataworking["newqst"].split("-")[1] ): - - query = ( - "Delete from " - + activeProjectUser - + "_" - + activeProjectCod - + ".REG_geninfo where qst162='" - + dataworking["newqst"].split("-")[1] - + "'" - ) - execute_two_sqls( - "SET @odktools_current_user = '" - + self.user.login - + "';", - query, + delete_registry_data_by_qst162( + schema, + dataworking["newqst"].split("-")[1], + self.user.login, ) storeJSONInMySQL( @@ -159,22 +152,11 @@ def processView(self): if str(dataworking["txt_oldvalue"]) == str( dataworking["newqst2"] ): - query = ( - "Delete from " - + activeProjectUser - + "_" - + activeProjectCod - + ".ASS" - + codeId - + "_geninfo where qst163='" - + dataworking["newqst2"] - + "'" - ) - execute_two_sqls( - "SET @odktools_current_user = '" - + self.user.login - + "'; ", - query, + delete_assessment_data_by_qst163( + schema, + codeId, + dataworking["newqst2"], + self.user.login, ) storeJSONInMySQL( @@ -253,20 +235,10 @@ def processView(self): if str(dataworking["txt_oldvalue"]) == str( dataworking["newqst"].split("-")[1] ): - query = ( - "Delete from " - + activeProjectUser - + "_" - + activeProjectCod - + ".REG_geninfo where qst162='" - + dataworking["newqst"].split("-")[1] - + "'" - ) - execute_two_sqls( - "SET @odktools_current_user = '" - + self.user.login - + "'; ", - query, + delete_registry_data_by_qst162( + schema, + dataworking["newqst"].split("-")[1], + self.user.login, ) update_registry_status_log( @@ -290,22 +262,11 @@ def processView(self): if str(dataworking["txt_oldvalue"]) == str( dataworking["newqst2"] ): - query = ( - "Delete from " - + activeProjectUser - + "_" - + activeProjectCod - + ".ASS" - + codeId - + "_geninfo where qst163='" - + dataworking["newqst2"] - + "'" - ) - execute_two_sqls( - "SET @odktools_current_user = '" - + self.user.login - + "'; ", - query, + delete_assessment_data_by_qst163( + schema, + codeId, + dataworking["newqst2"], + self.user.login, ) update_assessment_status_log( @@ -378,7 +339,7 @@ def processView(self): # Edited by Brandon path = os.path.join( self.request.registry.settings["user.repository"], - *[activeProjectUser, activeProjectCod] + *[activeProjectUser, activeProjectCod], ) paths = ["db", "ass", codeId, "create.xml"] path = os.path.join(path, *paths) From fb130b9cdbf43c2ba800c4a2749d9a732df07f81 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 25 Aug 2025 13:19:17 -0600 Subject: [PATCH 40/66] create functions to select registry and assessments data --- climmob/processes/db/assessment.py | 14 ++++++++++++++ climmob/processes/db/registry.py | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/climmob/processes/db/assessment.py b/climmob/processes/db/assessment.py index 521626d3..c7b65a83 100644 --- a/climmob/processes/db/assessment.py +++ b/climmob/processes/db/assessment.py @@ -86,6 +86,7 @@ "copy_assessment_questions", "copy_assessment_sections", "delete_assessment_data_by_qst163", + "get_assessment_data_by_qst163", ] log = logging.getLogger(__name__) @@ -1752,6 +1753,19 @@ def getFinalizedAssessments(request, userOwner, projectCod, projectId): return result +def get_assessment_data_by_qst163(schema, ass_id, qst163, columns): + query = ( + f"SELECT {','.join(columns)} FROM " + + schema + + ".ASS" + + ass_id + + "_geninfo WHERE qst163='" + + qst163 + + "'" + ) + return sql_execute(query).fetchone() + + def delete_assessment_data_by_qst163(schema, ass_id, qst163, odk_user): query = ( "DELETE FROM " diff --git a/climmob/processes/db/registry.py b/climmob/processes/db/registry.py index a245d889..40fbf9e1 100644 --- a/climmob/processes/db/registry.py +++ b/climmob/processes/db/registry.py @@ -4,7 +4,7 @@ from sqlalchemy import func from climmob.models import Regsection, Registry, Project, Question, userProject -from climmob.models.repository import execute_two_sqls +from climmob.models.repository import execute_two_sqls, sql_execute from climmob.models.schema import mapFromSchema, mapToSchema from climmob.processes import addRegistryQuestionsToProject from climmob.processes.db.anonymized import ( @@ -43,6 +43,7 @@ "registryHaveQuestionOfMultimediaType", "deleteRegistryByProjectId", "delete_registry_data_by_qst162", + "get_registry_data_by_qst162", ] @@ -76,7 +77,7 @@ def setRegistryStatus(userOwner, projectCod, projectId, status, request): try: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "data", "reg"] + *[userOwner, projectCod, "data", "reg"], ) shutil.rmtree(path) except: @@ -591,6 +592,17 @@ def registryHaveQuestionOfMultimediaType(request, projectId): return False +def get_registry_data_by_qst162(schema, qst162, columns): + query = ( + f"SELECT {','.join(columns)} FROM " + + schema + + ".REG_geninfo WHERE qst162='" + + qst162 + + "'" + ) + return sql_execute(query).fetchone() + + def delete_registry_data_by_qst162(schema, qst162, odk_user): query = "DELETE FROM " + schema + ".REG_geninfo WHERE qst162='" + qst162 + "'" execute_two_sqls( From 3a8cb58001a990a1e999d905c4f67ff1ff79a2f0 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 25 Aug 2025 15:29:18 -0600 Subject: [PATCH 41/66] add anonymization to update data --- climmob/processes/db/anonymized.py | 138 ++++++++++++++++++----------- climmob/views/editData.py | 2 + climmob/views/editDataDB.py | 54 ++++++++--- 3 files changed, 134 insertions(+), 60 deletions(-) diff --git a/climmob/processes/db/anonymized.py b/climmob/processes/db/anonymized.py index 8050dcfc..5d264430 100644 --- a/climmob/processes/db/anonymized.py +++ b/climmob/processes/db/anonymized.py @@ -1,5 +1,5 @@ import re -from datetime import datetime +from datetime import datetime, date from climmob.models.repository import sql_execute from climmob.processes.db.anonymization_params import get_anonymization_params_as_dict @@ -17,6 +17,7 @@ "anonymize_questions", "delete_anonymized_values_by_form_id", "delete_anonymized_values_by_form_id_and_reg_id", + "update_anonymized", ] @@ -53,61 +54,65 @@ def anonymize_questions(request, form, form_id, project_id, user_owner, project_ return True for field in to_anonymize: - params = get_anonymization_params_as_dict( - field["question"].question_id, request - ) - if field["question"].question_anonymity == QuestionAnonymity.PSEUDONYM.value: - field["value"] = params["pseudonym"].replace("{}", registry_id) - elif field["question"].question_anonymity == QuestionAnonymity.RANGE.value: - if field["question"].question_dtype == QuestionType.INTEGER.value: - parser = int - else: - parser = float - - field["value"] = parser(field["value"]) - params["lower_bound"] = parser(params["lower_bound"]) - params["upper_bound"] = parser(params["upper_bound"]) - params["interval"] = parser(params["interval"]) - - if field["value"] < params["lower_bound"]: - field["value"] = f'<{params["lower_bound"]}' - elif field["value"] > params["upper_bound"]: - field["value"] = f'>{params["upper_bound"]}' - else: - i = params["lower_bound"] - while i < params["upper_bound"]: - if i <= field["value"] < (i + params["interval"]): - field["value"] = f'{i}-{i + params["interval"]}' - break - i += params["interval"] - elif field["question"].question_anonymity == QuestionAnonymity.MONTH_YEAR.value: - dt = datetime.fromisoformat(field["value"]) - field["value"] = dt.strftime("%Y-%m") - elif field["question"].question_anonymity == QuestionAnonymity.NOISE.value: - geo_point = field["value"].split() - geo_point[0], geo_point[1] = add_noise_to_gps_coordinates( - float(geo_point[0]), float(geo_point[1]), 3000 - ) - if geo_point[0] == "Error" or geo_point[1] == "Error": - return False, "Could not anonymize GeoPoint" - field["value"] = " ".join(geo_point) - field["sql_insert_value"] = ( - f"(" - f"'{form_id}', " - f"'{registry_id}', " - f"'{field['field_name']}', " - f"'{field['value']}'" - f")" - ) - success, msg = execute_anonymization(field, form_id, schema) + anonymize_field_value(field, registry_id, request) + success, msg = insert_anonymized_field(field, form_id, registry_id, schema) if not success: return False, msg return True, "" -def execute_anonymization(field, form_id, schema): - sql = f"INSERT INTO {schema}.anonymized VALUES {field['sql_insert_value']}" +def anonymize_field_value(field, registry_id, request): + params = get_anonymization_params_as_dict(field["question"].question_id, request) + if field["question"].question_anonymity == QuestionAnonymity.PSEUDONYM.value: + field["value"] = params["pseudonym"].replace("{}", registry_id) + elif field["question"].question_anonymity == QuestionAnonymity.RANGE.value: + if field["question"].question_dtype == QuestionType.INTEGER.value: + parser = int + else: + parser = float + + field["value"] = parser(field["value"]) + params["lower_bound"] = parser(params["lower_bound"]) + params["upper_bound"] = parser(params["upper_bound"]) + params["interval"] = parser(params["interval"]) + + if field["value"] < params["lower_bound"]: + field["value"] = f'<{params["lower_bound"]}' + elif field["value"] > params["upper_bound"]: + field["value"] = f'>{params["upper_bound"]}' + else: + i = params["lower_bound"] + while i < params["upper_bound"]: + if i <= field["value"] < (i + params["interval"]): + field["value"] = f'{i}-{i + params["interval"]}' + break + i += params["interval"] + elif field["question"].question_anonymity == QuestionAnonymity.MONTH_YEAR.value: + dt = datetime.fromisoformat(field["value"]) + field["value"] = dt.strftime("%Y-%m") + elif field["question"].question_anonymity == QuestionAnonymity.NOISE.value: + geo_point = field["value"].split() + geo_point[0], geo_point[1] = add_noise_to_gps_coordinates( + float(geo_point[0]), float(geo_point[1]), 3000 + ) + if geo_point[0] == "Error" or geo_point[1] == "Error": + return False, "Could not anonymize GeoPoint" + field["value"] = " ".join(geo_point) + + return True, "" + + +def insert_anonymized_field(field, form_id, registry_id, schema): + sql_insert_value = ( + f"(" + f"'{form_id}', " + f"'{registry_id}', " + f"'{field['field_name']}', " + f"'{field['value']}'" + f")" + ) + sql = f"INSERT INTO {schema}.anonymized VALUES {sql_insert_value}" try: sql_execute(sql) return True, "" @@ -120,6 +125,39 @@ def execute_anonymization(field, form_id, schema): return False, "" +def update_anonymized(to_anonymize, schema, form_id, registry_id, request, current): + for field in to_anonymize: + db_type = type(current[field["field_name"]]) + if db_type == date: + new_value = date.fromisoformat(field["value"]) + elif db_type == datetime: + new_value = datetime.fromisoformat(field["value"]) + else: + new_value = db_type(field["value"]) + if current[field["field_name"]] == new_value: + # Only changed values will be updated to avoid recalculating anonymizations + continue + anonymize_field_value(field, registry_id, request) + success, msg = update_anonymized_field(field, form_id, registry_id, schema) + if not success: + return False, msg + return True, "" + + +def update_anonymized_field(field, form_id, registry_id, schema): + sql = ( + f"UPDATE {schema}.anonymized SET value='{field['value']}' " + f"WHERE form_id='{form_id}' " + f"AND reg_id='{registry_id}' " + f"AND col_name='{field['field_name']}'" + ) + try: + sql_execute(sql) + return True, "" + except Exception as e: + return False, str(e) + + def delete_anonymized_values_by_form_id(schema, form_id): sql = f"DELETE FROM {schema}.anonymized where form_id='{form_id}'" sql_execute(sql) diff --git a/climmob/views/editData.py b/climmob/views/editData.py index a8a90c5f..bbcd7187 100755 --- a/climmob/views/editData.py +++ b/climmob/views/editData.py @@ -263,6 +263,8 @@ def processView(self): path, code, self.user.login, + activeProjectId, + self.request, ) dataXML = getNamesEditByColums(path) diff --git a/climmob/views/editDataDB.py b/climmob/views/editDataDB.py index 17897d13..5e4c4374 100755 --- a/climmob/views/editDataDB.py +++ b/climmob/views/editDataDB.py @@ -2,7 +2,15 @@ import xml.etree.ElementTree as ET from climmob.models.repository import sql_execute, execute_two_sqls -from climmob.processes import getProjectData, getQuestionOptionsByQuestionCode +from climmob.processes import ( + getProjectData, + getQuestionOptionsByQuestionCode, + get_sensitive_questions_anonymity_by_project_id, +) +from climmob.processes.db.anonymized import update_anonymized +from climmob.processes.db.assessment import get_assessment_data_by_qst163 +from climmob.processes.db.registry import get_registry_data_by_qst162 +from climmob.utility import get_question_by_field_name, QuestionAnonymity def get_FieldsByType(types, file): @@ -329,10 +337,15 @@ def fillDataTable( return json.dumps(ret) -def update_edited_data(userOwner, projectCod, form, data, file, code, by): +def update_edited_data( + userOwner, projectCod, form, data, file, code, by, project_id, request +): data = json.loads(data[0]) + schema = userOwner + "_" + projectCod + questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) + for row in data: del row["id"] if row["flag_update"]: @@ -341,6 +354,7 @@ def update_edited_data(userOwner, projectCod, form, data, file, code, by): form.upper() + code, ) del row["flag_update"] + to_anonymize = [] for key in row: val = "" addField = True @@ -352,21 +366,37 @@ def update_edited_data(userOwner, projectCod, form, data, file, code, by): else: if key in get_FieldsByType(["select1"], file): if row[key] and row[key] != "None": - val = ( - "'" - + str(row[key]).replace("[", "").replace("]", "") - + "'" - ) + val = str(row[key]).replace("[", "").replace("]", "") else: addField = False else: if key in get_FieldsByType(["select"], file): - val = "'" + " ".join(row[key]) + "'" + val = " ".join(row[key]) else: - val = "'" + str(row[key]) + "'" + val = str(row[key]) if addField: - query_update += key + "=" + val + ", " + query_update += key + "='" + val + "', " + question = get_question_by_field_name(key, questions) + if ( + question + and question.question_anonymity + != QuestionAnonymity.REMOVE.value + ): + to_anonymize.append( + {"field_name": key, "value": val, "question": question} + ) + + reg_id = row["qst162"] if form == "reg" else row["qst163"] + form_id = "-" if form == "reg" else code + columns = [field["field_name"] for field in to_anonymize] + + if form_id == "-": + current = get_registry_data_by_qst162(schema, reg_id, columns) + else: + current = get_assessment_data_by_qst163( + schema, form_id, reg_id, columns + ) query_update = ( query_update[:-2] + " where rowuuid ='" + str(row["rowuuid"]) + "';" @@ -377,7 +407,11 @@ def update_edited_data(userOwner, projectCod, form, data, file, code, by): execute_two_sqls( "SET @odktools_current_user = '" + by + "'; ", query_update ) + update_anonymized( + to_anonymize, schema, form_id, reg_id, request, current + ) except Exception as e: print(str(e)) return 0, str(e) + return 1, "" From 1660297be8ffa3eb707c8b228fd47f7ba68181c1 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 27 Aug 2025 13:20:19 -0600 Subject: [PATCH 42/66] change download button layout in the dashboard --- climmob/templates/dashboard/dashboard.jinja2 | 13 ++- .../snippets/dashboard/downloads.jinja2 | 85 +++++++++++++++++++ 2 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 climmob/templates/snippets/dashboard/downloads.jinja2 diff --git a/climmob/templates/dashboard/dashboard.jinja2 b/climmob/templates/dashboard/dashboard.jinja2 index b48c10fd..248319e1 100755 --- a/climmob/templates/dashboard/dashboard.jinja2 +++ b/climmob/templates/dashboard/dashboard.jinja2 @@ -41,6 +41,8 @@ {% include "snippets/maze_loader.jinja2" %} +{% from 'snippets/dashboard/downloads.jinja2' import downloads with context %} + {% if request.registry.settings.get('session.show_expired_modal', "false") == 'true' %} {% include 'snippets/expired_session.jinja2' %} {% endif %} @@ -565,10 +567,6 @@ {% if progress.regtotal > 0 %} {{ _("View and edit data") }} - {{ _("Download data in .CSV format") }} - {{ _("Download anonymized data in .CSV format") }} - {{ _("Download data in .XLSX format") }} - {{ _("Download anonymized data in .XLSX format") }} {% block dataprivacy_download_data_registry %} @@ -583,6 +581,8 @@ + + {{ downloads("downloadDataRegistry", "registry") }}
{% endif %} @@ -704,10 +704,6 @@ {% if assessment.asstotal > 0 %} {{ _("View and edit data") }} - {{ _("Download data in .CSV format") }} - {{ _("Download anonymized data in .CSV format") }} - {{ _("Download data in .XLSX format") }} - {{ _("Download anonymized data in .XLSX format") }} {% block dataprivacy_download_data_assessment scoped%} @@ -726,6 +722,7 @@ + {{ downloads("downloadDataAssessment", "assessment", assessment.ass_cod) }}
diff --git a/climmob/templates/snippets/dashboard/downloads.jinja2 b/climmob/templates/snippets/dashboard/downloads.jinja2 new file mode 100644 index 00000000..85a020d7 --- /dev/null +++ b/climmob/templates/snippets/dashboard/downloads.jinja2 @@ -0,0 +1,85 @@ +{% macro downloads(route, form_type, ass_id='') %} + + +

{{ _("Downloads") }}

+ + + + + + + + + + + + + + + + + + + + + + +
{{ _('File Type') }}{{ _('Raw') }}{{ _('Anonymized') }}
+ CSV + + + + + + + + +
+ XLSX + + + + + + + + +
+{% endmacro %} From 1ba4a464657d9f308dcaf0729bd260ff633fb33d Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 27 Aug 2025 16:00:37 -0600 Subject: [PATCH 43/66] make downloads panel collapsible --- .../snippets/dashboard/downloads.jinja2 | 181 ++++++++++-------- 1 file changed, 106 insertions(+), 75 deletions(-) diff --git a/climmob/templates/snippets/dashboard/downloads.jinja2 b/climmob/templates/snippets/dashboard/downloads.jinja2 index 85a020d7..3088d6a7 100644 --- a/climmob/templates/snippets/dashboard/downloads.jinja2 +++ b/climmob/templates/snippets/dashboard/downloads.jinja2 @@ -1,85 +1,116 @@ {% macro downloads(route, form_type, ass_id='') %} -

{{ _("Downloads") }}

- - - - - - - - +
+
+

{{ _("Downloads") }}

+ + + +
+
+
{{ _('File Type') }}{{ _('Raw') }}{{ _('Anonymized') }}
+ + + + + + + - - - - - - + + + + + + - - - - - - -
{{ _('File Type') }}{{ _('Raw') }}{{ _('Anonymized') }}
- CSV - - - - - - - - -
+ CSV + + + + + + + + +
- XLSX - - - - - - - - -
+ + + XLSX + + + + + + + + + + + + + + +
+ {% endmacro %} From 3627f53cf3f0c23ef62f7b2d84090f951df6bac1 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 28 Aug 2025 08:22:38 -0600 Subject: [PATCH 44/66] fix registry and assessment start and cancel tests --- .../test_views_api_project_assessment_start.py | 10 ++++++++-- .../test_views_api_project_registry_start.py | 10 ++++++++-- climmob/tests/test_utils/test_views_registry.py | 2 ++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/climmob/tests/test_utils/test_views_api_project_assessment_start.py b/climmob/tests/test_utils/test_views_api_project_assessment_start.py index 2b12ea2b..a116c09c 100644 --- a/climmob/tests/test_utils/test_views_api_project_assessment_start.py +++ b/climmob/tests/test_utils/test_views_api_project_assessment_start.py @@ -1516,7 +1516,10 @@ def test_api_registration_error_repeated_column(self): @patch("climmob.views.Api.projectAssessmentStart.open") @patch("climmob.views.Api.projectAssessmentStart.uuid.uuid1") @patch("climmob.views.Api.projectAssessmentStart.os.path.join") - @patch("climmob.views.Api.projectAssessmentStart.storeJSONInMySQL") + @patch( + "climmob.views.Api.projectAssessmentStart.storeJSONInMySQL", + return_value=(True, ""), + ) @patch( "climmob.views.Api.projectAssessmentStart.os.path.exists", return_value=False ) @@ -1579,7 +1582,10 @@ def test_api_registration_success( "climmob.views.Api.projectAssessmentStart.os.path.join", side_effect=os.path.join, ) - @patch("climmob.views.Api.projectAssessmentStart.storeJSONInMySQL") + @patch( + "climmob.views.Api.projectAssessmentStart.storeJSONInMySQL", + return_value=(True, ""), + ) def test_api_registration_data_could_not_be_saved( self, mock_storeJSONInMySQL, mock_os_path_join, mock_uuid1, mock_open ): diff --git a/climmob/tests/test_utils/test_views_api_project_registry_start.py b/climmob/tests/test_utils/test_views_api_project_registry_start.py index 489d5c11..07cfd600 100644 --- a/climmob/tests/test_utils/test_views_api_project_registry_start.py +++ b/climmob/tests/test_utils/test_views_api_project_registry_start.py @@ -3134,7 +3134,10 @@ def test_api_registration_no_package_code(self, mock_getProjectNumobs, mock_open ) @patch("climmob.views.Api.projectRegistryStart.open") - @patch("climmob.views.Api.projectRegistryStart.storeJSONInMySQL") + @patch( + "climmob.views.Api.projectRegistryStart.storeJSONInMySQL", + return_value=(True, ""), + ) @patch("climmob.views.Api.projectRegistryStart.getProjectNumobs", return_value=10) @patch("uuid.uuid1", return_value="12345678") def test_api_registration_reads_log_error( @@ -3195,7 +3198,10 @@ def test_api_registration_reads_log_error( mock_storeJSONInMySQL.assert_called() @patch("climmob.views.Api.projectRegistryStart.open") - @patch("climmob.views.Api.projectRegistryStart.storeJSONInMySQL") + @patch( + "climmob.views.Api.projectRegistryStart.storeJSONInMySQL", + return_value=(True, ""), + ) @patch("climmob.views.Api.projectRegistryStart.getProjectNumobs", return_value=10) def test_api_registration_successful( self, mock_getProjectNumobs, mock_storeJSONInMySQL, mock_open diff --git a/climmob/tests/test_utils/test_views_registry.py b/climmob/tests/test_utils/test_views_registry.py index a0e99e38..30f045c8 100644 --- a/climmob/tests/test_utils/test_views_registry.py +++ b/climmob/tests/test_utils/test_views_registry.py @@ -275,6 +275,7 @@ def test_process_view_get( ) mock_getActiveProject.assert_called_once_with("test_user", self.mock_request) + @patch("climmob.views.registry.delete_anonymized_values_by_form_id") @patch( "climmob.views.registry.getActiveProject", return_value={"project": "active_project"}, @@ -294,6 +295,7 @@ def test_process_view_post_cancel_registry( mock_projectExists, mock_getTheProjectIdForOwner, mock_getActiveProject, + mock_delete_anonymized_values_by_form_id, ): self.mock_request.method = "POST" self.mock_request.params = {"cancelRegistry": "1"} From c2b3df6aec81dfdccedc40cb60a740be7586bbe2 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 28 Aug 2025 10:49:00 -0600 Subject: [PATCH 45/66] extract methods for getting registry and assessment key questions --- climmob/processes/db/question.py | 14 ++++++++++++++ climmob/processes/db/results.py | 15 +++++---------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index 01686449..875f97db 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -43,6 +43,8 @@ "getQuestionOwner", "knowIfUserHasCreatedTranslations", "get_sensitive_questions_anonymity_by_project_id", + "get_registry_key_question", + "get_assessment_key_question", ] from climmob.models.climmobv4 import AnonymizationParameter @@ -562,3 +564,15 @@ def get_sensitive_questions_anonymity_by_project_id(project_id, request): ) ) return query.all() + + +def get_registry_key_question(request): + return ( + request.dbsession.query(Question).filter(Question.question_regkey == 1).first() + ) + + +def get_assessment_key_question(request): + return ( + request.dbsession.query(Question).filter(Question.question_asskey == 1).first() + ) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 25a2484e..ff021c2e 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -12,11 +12,13 @@ ) from climmob.processes.db.question import ( get_sensitive_questions_anonymity_by_project_id, + get_registry_key_question, + get_assessment_key_question, ) __all__ = ["getJSONResult", "getCombinationsData"] -from climmob.utility import get_question_by_field_name +from climmob.utility import get_question_by_field_name, QuestionAnonymity def getMiltiSelectLookUpTable(XMLFile, multiSelectTable): @@ -318,16 +320,9 @@ def build(self): def getData( userOwner, project_id, projectCod, registry, assessments, request, anonymize=False ): - from climmob.utility import QuestionAnonymity + registryKey = get_registry_key_question(request).question_code - data = ( - request.dbsession.query(Question).filter(Question.question_regkey == 1).first() - ) - registryKey = data.question_code - data = ( - request.dbsession.query(Question).filter(Question.question_asskey == 1).first() - ) - assessmentKey = data.question_code + assessmentKey = get_assessment_key_question(request).question_code questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) From 721286a138d6675aa947381a3ce863876d81d83d Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 29 Aug 2025 16:10:20 -0600 Subject: [PATCH 46/66] add process anonymize entire project --- climmob/processes/db/anonymized.py | 54 ++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/climmob/processes/db/anonymized.py b/climmob/processes/db/anonymized.py index 5d264430..b0c8812f 100644 --- a/climmob/processes/db/anonymized.py +++ b/climmob/processes/db/anonymized.py @@ -18,9 +18,63 @@ "delete_anonymized_values_by_form_id", "delete_anonymized_values_by_form_id_and_reg_id", "update_anonymized", + "anonymize_project", ] +def anonymize_project(user_owner, project_id, project_code, request): + from climmob.processes import getJSONResult + + questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) + + project_collected_data = getJSONResult( + user_owner, project_id, project_code, request + )["data"] + + schema = user_owner + "_" + project_code + + pattern = r"(REG|(ASS(.+?)))_(.*)" + for entry in project_collected_data: + reg_id = entry["REG_qst162"] + to_anonymize = [] + for key in entry.keys(): + if entry[key] is None: + continue + match = re.match(pattern, key) + if match is None: + continue + question = get_question_by_field_name(match.group(4), questions) + if ( + question + and question.question_anonymity != QuestionAnonymity.REMOVE.value + ): + if match.group(1) == "REG": + form_id = "-" + else: + form_id = match.group(3) + to_anonymize.append( + { + "field_name": match.group(4), + "value": entry[key], + "question": question, + "form_id": form_id, + } + ) + + for field in to_anonymize: + anonymize_field_value(field, reg_id, request) + success, msg = insert_anonymized_field( + field, field["form_id"], reg_id, schema + ) + if not success: + if msg.startswith("Duplicate entry for package"): + # To ignore entries that are already anonymized + continue + return False, msg + + return project_collected_data + + def anonymize_questions(request, form, form_id, project_id, user_owner, project_cod): questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) From 17e7937843d6fc4807f62d5ecbeb2c9ef00d3245 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 29 Aug 2025 16:11:38 -0600 Subject: [PATCH 47/66] set project results for api to be always anonymized set project results for api to be always anonymized --- climmob/views/Api/project_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/climmob/views/Api/project_analysis.py b/climmob/views/Api/project_analysis.py index d3a54f2c..4fef666f 100644 --- a/climmob/views/Api/project_analysis.py +++ b/climmob/views/Api/project_analysis.py @@ -56,6 +56,7 @@ def processView(self): activeProjectId, dataworking["project_cod"], self.request, + anonymize=True, ) ), ) From 3b3dc20abeebe7b4851cb74aa98da2255545fb96 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 1 Sep 2025 10:55:02 -0600 Subject: [PATCH 48/66] hide anonymized products for projects with no anonymized values --- climmob/processes/db/anonymized.py | 7 +++ .../project/productsList/productsList.jinja2 | 47 ++++++++++--------- climmob/views/productsList.py | 14 ++++++ 3 files changed, 45 insertions(+), 23 deletions(-) diff --git a/climmob/processes/db/anonymized.py b/climmob/processes/db/anonymized.py index b0c8812f..d5e9cd35 100644 --- a/climmob/processes/db/anonymized.py +++ b/climmob/processes/db/anonymized.py @@ -19,6 +19,7 @@ "delete_anonymized_values_by_form_id_and_reg_id", "update_anonymized", "anonymize_project", + "does_project_has_anonymized_values", ] @@ -224,3 +225,9 @@ def delete_anonymized_values_by_form_id_and_reg_id(schema, form_id, reg_id): f"AND reg_id='{reg_id}'" ) sql_execute(query) + + +def does_project_has_anonymized_values(schema): + query = f"SELECT COUNT(*) as count FROM {schema}.anonymized" + result = sql_execute(query).first() + return result["count"] > 0 diff --git a/climmob/templates/snippets/project/productsList/productsList.jinja2 b/climmob/templates/snippets/project/productsList/productsList.jinja2 index 991526ff..c9fcf69d 100644 --- a/climmob/templates/snippets/project/productsList/productsList.jinja2 +++ b/climmob/templates/snippets/project/productsList/productsList.jinja2 @@ -360,30 +360,31 @@ {% endif %} - {% if changes["projectDataCollectedXLSX-anonymized"] %} - - {{ _("Information collected in all the project in .XLSX format (anonymized)") }} - - - - {{ _("Create") }} - - - + {% if project_has_anonymized %} + {% if changes["projectDataCollectedXLSX-anonymized"] %} + + {{ _("Information collected in all the project in .XLSX format (anonymized)") }} + + + + {{ _("Create") }} + + + + {% endif %} + + {% if changes["projectDataCollectedCSV-anonymized"] %} + + {{ _("Information collected in all the project in .CSV format (anonymized)") }} + + + + {{ _("Create") }} + + + + {% endif %} {% endif %} - - {% if changes["projectDataCollectedCSV-anonymized"] %} - - {{ _("Information collected in all the project in .CSV format (anonymized)") }} - - - - {{ _("Create") }} - - - - {% endif %} - {% if changes.projectSummary %} {{ _("Project summary") }} diff --git a/climmob/views/productsList.py b/climmob/views/productsList.py index 16fb1a33..e1b90a6d 100644 --- a/climmob/views/productsList.py +++ b/climmob/views/productsList.py @@ -29,6 +29,7 @@ get_registry_logs, get_assessment_logs, getPrjLangDefaultInProject, + does_project_has_anonymized_values, ) from climmob.products import product_found from climmob.products.analysisdata.analysisdata import create_raw_data @@ -84,11 +85,23 @@ def processView(self): productsAvailable = [] assessments = [] + schema = activeProjectData["user_name"] + "_" + activeProjectData["project_cod"] + + project_has_anonymized_values = does_project_has_anonymized_values(schema) + if activeProjectData: products = getDataProduct(activeProjectData["project_id"], self.request) for product in products: + + if ( + product["product_id"] + in ["datacsv-anonymized", "dataxlsx-anonymized"] + and not project_has_anonymized_values + ): + continue + if product_found(product["product_id"]): contentType = product["output_mimetype"] filename = product["output_id"] @@ -146,6 +159,7 @@ def processView(self): "Products": productsAvailable, "assessments": assessments, "sectionActive": "productlist", + "project_has_anonymized": project_has_anonymized_values, } From 03fc9c499199ac4d2a2d401bf0ca7f9906757ff9 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 3 Sep 2025 09:18:47 -0600 Subject: [PATCH 49/66] hide anonymized downloads when module.dataprivacy is false --- .../snippets/dashboard/downloads.jinja2 | 40 +++++++++++++------ .../project/productsList/productsList.jinja2 | 2 +- climmob/views/dashboard.py | 19 +++++++++ climmob/views/productsList.py | 11 +++-- 4 files changed, 54 insertions(+), 18 deletions(-) diff --git a/climmob/templates/snippets/dashboard/downloads.jinja2 b/climmob/templates/snippets/dashboard/downloads.jinja2 index 3088d6a7..add167e1 100644 --- a/climmob/templates/snippets/dashboard/downloads.jinja2 +++ b/climmob/templates/snippets/dashboard/downloads.jinja2 @@ -43,7 +43,9 @@ {{ _('File Type') }} {{ _('Raw') }} - {{ _('Anonymized') }} + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + {{ _('Anonymized') }} + {% endif %} @@ -64,9 +66,14 @@ - - + - - - + + + + {% endif %} @@ -95,9 +103,14 @@ - - + - - - + + + + {% endif %} diff --git a/climmob/templates/snippets/project/productsList/productsList.jinja2 b/climmob/templates/snippets/project/productsList/productsList.jinja2 index c9fcf69d..f2e9969e 100644 --- a/climmob/templates/snippets/project/productsList/productsList.jinja2 +++ b/climmob/templates/snippets/project/productsList/productsList.jinja2 @@ -360,7 +360,7 @@ {% endif %} - {% if project_has_anonymized %} + {% if project_has_anonymized and request.registry.settings.get("module.dataprivacy", "false") != "false"%} {% if changes["projectDataCollectedXLSX-anonymized"] %} {{ _("Information collected in all the project in .XLSX format (anonymized)") }} diff --git a/climmob/views/dashboard.py b/climmob/views/dashboard.py index 8b954b9a..aabc7078 100644 --- a/climmob/views/dashboard.py +++ b/climmob/views/dashboard.py @@ -15,6 +15,7 @@ AssessmentsInformation, seeProgress, getTheProjectIdForOwner, + does_project_has_anonymized_values, ) from climmob.views.classes import privateView, publicView @@ -41,6 +42,16 @@ def processView(self): activeProjectData = getActiveProject(self.user.login, self.request) + schema = ( + activeProjectData["user_name"] + + "_" + + activeProjectData["project_cod"] + ) + + project_has_anonymized_values = does_project_has_anonymized_values( + schema + ) + session = self.request.session session["activeProject"] = activeProjectId @@ -99,6 +110,7 @@ def processView(self): activeProjectCod, self.request, ), + "project_has_anonymized": project_has_anonymized_values, } for plugin in p.PluginImplementations(p.IDashBoard): context = plugin.before_returning_dashboard_context( @@ -108,6 +120,12 @@ def processView(self): else: activeProjectData = getActiveProject(self.user.login, self.request) + schema = ( + activeProjectData["user_name"] + "_" + activeProjectData["project_cod"] + ) + + project_has_anonymized_values = does_project_has_anonymized_values(schema) + if activeProjectData: self.returnRawViewResult = True return HTTPFound( @@ -127,6 +145,7 @@ def processView(self): "progress": {}, "pcompleted": 0, "allassclosed": False, + "project_has_anonymized": project_has_anonymized_values, } for plugin in p.PluginImplementations(p.IDashBoard): context = plugin.before_returning_dashboard_context( diff --git a/climmob/views/productsList.py b/climmob/views/productsList.py index e1b90a6d..b2f5eb14 100644 --- a/climmob/views/productsList.py +++ b/climmob/views/productsList.py @@ -95,10 +95,13 @@ def processView(self): for product in products: - if ( - product["product_id"] - in ["datacsv-anonymized", "dataxlsx-anonymized"] - and not project_has_anonymized_values + if product["product_id"] in [ + "datacsv-anonymized", + "dataxlsx-anonymized", + ] and ( + not project_has_anonymized_values + or self.request.registry.settings.get("module.dataprivacy", "false") + == "false" ): continue From fce85f2a665de82a78c83af19a70a9f9c654d839 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 3 Sep 2025 10:41:23 -0600 Subject: [PATCH 50/66] fix query to determine if project is anonymized --- climmob/processes/db/anonymized.py | 19 +++++++++++++++---- .../snippets/dashboard/downloads.jinja2 | 4 ++-- .../project/productsList/productsList.jinja2 | 2 +- climmob/views/dashboard.py | 12 +++++------- climmob/views/productsList.py | 8 ++++---- 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/climmob/processes/db/anonymized.py b/climmob/processes/db/anonymized.py index d5e9cd35..6cd3c91d 100644 --- a/climmob/processes/db/anonymized.py +++ b/climmob/processes/db/anonymized.py @@ -19,7 +19,7 @@ "delete_anonymized_values_by_form_id_and_reg_id", "update_anonymized", "anonymize_project", - "does_project_has_anonymized_values", + "is_project_anonymized", ] @@ -227,7 +227,18 @@ def delete_anonymized_values_by_form_id_and_reg_id(schema, form_id, reg_id): sql_execute(query) -def does_project_has_anonymized_values(schema): - query = f"SELECT COUNT(*) as count FROM {schema}.anonymized" +def is_project_anonymized(schema): + query = f""" + SELECT + (SELECT + COUNT(DISTINCT reg_id) AS count + FROM + {schema}.anonymized + WHERE + form_id = '-') = (SELECT + COUNT(qst162) AS count + FROM + {schema}.REG_geninfo) AS count_matches """ + result = sql_execute(query).first() - return result["count"] > 0 + return result["count_matches"] == 1 diff --git a/climmob/templates/snippets/dashboard/downloads.jinja2 b/climmob/templates/snippets/dashboard/downloads.jinja2 index add167e1..454b440e 100644 --- a/climmob/templates/snippets/dashboard/downloads.jinja2 +++ b/climmob/templates/snippets/dashboard/downloads.jinja2 @@ -69,7 +69,7 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} {% endif %} - {% if project_has_anonymized and request.registry.settings.get("module.dataprivacy", "false") != "false"%} + {% if project_is_anonymized and request.registry.settings.get("module.dataprivacy", "false") != "false"%} {% if changes["projectDataCollectedXLSX-anonymized"] %} {{ _("Information collected in all the project in .XLSX format (anonymized)") }} diff --git a/climmob/views/dashboard.py b/climmob/views/dashboard.py index aabc7078..1b719aaa 100644 --- a/climmob/views/dashboard.py +++ b/climmob/views/dashboard.py @@ -15,7 +15,7 @@ AssessmentsInformation, seeProgress, getTheProjectIdForOwner, - does_project_has_anonymized_values, + is_project_anonymized, ) from climmob.views.classes import privateView, publicView @@ -48,9 +48,7 @@ def processView(self): + activeProjectData["project_cod"] ) - project_has_anonymized_values = does_project_has_anonymized_values( - schema - ) + project_is_anonymized = is_project_anonymized(schema) session = self.request.session session["activeProject"] = activeProjectId @@ -110,7 +108,7 @@ def processView(self): activeProjectCod, self.request, ), - "project_has_anonymized": project_has_anonymized_values, + "project_is_anonymized": project_is_anonymized, } for plugin in p.PluginImplementations(p.IDashBoard): context = plugin.before_returning_dashboard_context( @@ -124,7 +122,7 @@ def processView(self): activeProjectData["user_name"] + "_" + activeProjectData["project_cod"] ) - project_has_anonymized_values = does_project_has_anonymized_values(schema) + project_is_anonymized = is_project_anonymized(schema) if activeProjectData: self.returnRawViewResult = True @@ -145,7 +143,7 @@ def processView(self): "progress": {}, "pcompleted": 0, "allassclosed": False, - "project_has_anonymized": project_has_anonymized_values, + "project_is_anonymized": project_is_anonymized, } for plugin in p.PluginImplementations(p.IDashBoard): context = plugin.before_returning_dashboard_context( diff --git a/climmob/views/productsList.py b/climmob/views/productsList.py index b2f5eb14..225288d2 100644 --- a/climmob/views/productsList.py +++ b/climmob/views/productsList.py @@ -29,7 +29,7 @@ get_registry_logs, get_assessment_logs, getPrjLangDefaultInProject, - does_project_has_anonymized_values, + is_project_anonymized, ) from climmob.products import product_found from climmob.products.analysisdata.analysisdata import create_raw_data @@ -87,7 +87,7 @@ def processView(self): schema = activeProjectData["user_name"] + "_" + activeProjectData["project_cod"] - project_has_anonymized_values = does_project_has_anonymized_values(schema) + project_is_anonymized = is_project_anonymized(schema) if activeProjectData: @@ -99,7 +99,7 @@ def processView(self): "datacsv-anonymized", "dataxlsx-anonymized", ] and ( - not project_has_anonymized_values + not project_is_anonymized or self.request.registry.settings.get("module.dataprivacy", "false") == "false" ): @@ -162,7 +162,7 @@ def processView(self): "Products": productsAvailable, "assessments": assessments, "sectionActive": "productlist", - "project_has_anonymized": project_has_anonymized_values, + "project_is_anonymized": project_is_anonymized, } From 41058185918ce02bed6d129fa4a9f539564ce63f Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 4 Sep 2025 16:12:56 -0600 Subject: [PATCH 51/66] anonymize packages farmername. Remove IMEI and instancename --- climmob/processes/db/results.py | 38 +++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index ff021c2e..9c6fa8ad 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -7,9 +7,8 @@ from climmob.models import Assessment, Question, Project, mapFromSchema from climmob.models.repository import sql_fetch_all, sql_fetch_one -from climmob.processes import ( - getCombinations, -) +from climmob.processes import getCombinations +from climmob.processes.db.anonymization_params import get_anonymization_params_as_dict from climmob.processes.db.question import ( get_sensitive_questions_anonymity_by_project_id, get_registry_key_question, @@ -113,7 +112,7 @@ def getLookups(XMLFile, userOwner, projectCod, anonymize): return lktables -def getPackageData(userOwner, projectId, projectCod, request): +def getPackageData(userOwner, projectId, projectCod, request, anonymize=False): data = ( request.dbsession.query(Question).filter(Question.question_regkey == 1).first() ) @@ -121,6 +120,7 @@ def getPackageData(userOwner, projectId, projectCod, request): data = ( request.dbsession.query(Question).filter(Question.question_fname == 1).first() ) + farmer_name_qst_id = data.question_id qstFarmer = data.question_code sql = ( @@ -223,6 +223,12 @@ def getPackageData(userOwner, projectId, projectCod, request): ) pkgdetails = sql_fetch_all(sql) + + farmer_name_pseudonym = "" + if anonymize: + params = get_anonymization_params_as_dict(farmer_name_qst_id, request) + farmer_name_pseudonym = params["pseudonym"] + packages = [] pkgcode = -999 for pkg in pkgdetails: @@ -230,7 +236,12 @@ def getPackageData(userOwner, projectId, projectCod, request): aPackage = {} pkgcode = pkg.package_id aPackage["package_id"] = pkg.package_id - aPackage["farmername"] = pkg["farmername"] + if anonymize: + aPackage["farmername"] = farmer_name_pseudonym.replace( + "{}", str(pkg.package_id) + ) + else: + aPackage["farmername"] = pkg["farmername"] aPackage["comps"] = [] for x in range(0, ncombs): aPackage["comps"].append({}) @@ -364,6 +375,19 @@ def getData( select_field_builder.set_sensitive(question is not None) fields.append(select_field_builder.build()) + if anonymize: + to_remove_keys = ["instancename", "deviceimei"] + tmp_fields = fields.copy() + fields = [] + for field in tmp_fields: + append = True + for key in to_remove_keys: + if key in field: + append = False + break + if append: + fields.append(field) + sql = ( "SELECT " + ",".join(fields) @@ -689,7 +713,9 @@ def getJSONResult( if res.project_registration_and_analysis == 1: haveAssessments = True # Get the package information but only for registered farmers - data["packages"] = getPackageData(userOwner, projectId, projectCod, request) + data["packages"] = getPackageData( + userOwner, projectId, projectCod, request, anonymize + ) data["combination"] = getCombinationsData(projectId, request) if haveAssessments: From d44b82e8c71fcd4d8f8107c538fb6a1b29cde028 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 8 Sep 2025 14:17:47 -0600 Subject: [PATCH 52/66] fix error message for pseudonym --- climmob/templates/question/library.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/climmob/templates/question/library.jinja2 b/climmob/templates/question/library.jinja2 index 4b22dd2b..04d66e7a 100644 --- a/climmob/templates/question/library.jinja2 +++ b/climmob/templates/question/library.jinja2 @@ -703,7 +703,7 @@ const regex = /^[^{}]*{}[^{}]*$/; if (!regex.test(anonymity_body["anonym_param_pseudonym"])) { invalid_input = {name: "anonym_param_pseudonym", - msg: "{{ _("Must contain '{}' once.") }}" } + msg: "{{ _("Musts contain '{}' once.") | safe }}" } } } else if (anonymity_body["question_anonymity"] === {{ QuestionAnonymity.RANGE }}) { From 2239262c79b09c67f5f1027fb7d1d18faf12c394 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Wed, 10 Sep 2025 16:32:43 -0600 Subject: [PATCH 53/66] add error message for interval greater than zero --- climmob/templates/question/library.jinja2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/climmob/templates/question/library.jinja2 b/climmob/templates/question/library.jinja2 index 04d66e7a..a1e27f89 100644 --- a/climmob/templates/question/library.jinja2 +++ b/climmob/templates/question/library.jinja2 @@ -724,6 +724,9 @@ if (lower_bound >= upper_bound) { invalid_input = {name: "anonym_param_upper_bound", msg: "{{ _("Must be higher than the lower bound.") }}" } + } else if (interval <= 0) { + invalid_input = {name: "anonym_param_interval", + msg: "{{ _("Must be greater than zero.") }}" } } else if (interval >= upper_bound - lower_bound) { invalid_input = {name: "anonym_param_interval", msg: "{{ _("Must fit within the bounds.") }}" } From 560c7b8ba052e694adc9543fb5e4bc703e7ed8ff Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 6 Oct 2025 11:05:00 -0600 Subject: [PATCH 54/66] remove unnecessary endpoint --- climmob/config/routes.py | 10 ---------- climmob/views/results.py | 17 ----------------- 2 files changed, 27 deletions(-) delete mode 100644 climmob/views/results.py diff --git a/climmob/config/routes.py b/climmob/config/routes.py index 5b77f7e2..cb06dd1d 100644 --- a/climmob/config/routes.py +++ b/climmob/config/routes.py @@ -245,7 +245,6 @@ GetRegistrySectionView, ChangeProjectMainLanguage_view, ) -from climmob.views.results import ResultsView from climmob.views.techaliases import deletealias_view from climmob.views.technologies import ( technologies_view, @@ -2166,15 +2165,6 @@ def loadRoutes(config): ) ) - routes.append( - addRoute( - "results", - "/api/results", - ResultsView, - "json", - ) - ) - # --------------------------------------------------------ClimMob Bot--------------------------------------------------------# # Chat diff --git a/climmob/views/results.py b/climmob/views/results.py deleted file mode 100644 index 1d75005c..00000000 --- a/climmob/views/results.py +++ /dev/null @@ -1,17 +0,0 @@ -from climmob.processes import getJSONResult, getProjectData -from climmob.views.classes import apiView - - -class ResultsView(apiView): - def get(self): - active_project_data = getProjectData( - self.context.active_project_id, self.request - ) - anonymize = str(self.request.params.get("anonymize")).lower() == "true" - return getJSONResult( - self.user.login, - self.context.active_project_id, - active_project_data["project_cod"], - self.request, - anonymize=anonymize, - ) From 6a8fe74733cd94db43d470951821460768cfdec3 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 6 Oct 2025 11:25:59 -0600 Subject: [PATCH 55/66] fix anonymization --- climmob/processes/db/results.py | 15 ++++++++------- climmob/views/productsList.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 9c6fa8ad..7b899974 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -1,7 +1,6 @@ import datetime import decimal import os -import re from lxml import etree @@ -64,11 +63,12 @@ def getFields(XMLFile, table): return fields -def getLookups(XMLFile, userOwner, projectCod, anonymize): +def getLookups(XMLFile, userOwner, projectCod, anonymize, request): lktables = [] tree = etree.parse(XMLFile) root = tree.getroot() elkptables = root.find(".//lkptables") + qst_163_pseudonym = get_anonymization_params_as_dict(163, request)["pseudonym"] if elkptables is not None: etables = elkptables.findall(".//table") for table in etables: @@ -104,9 +104,9 @@ def getLookups(XMLFile, userOwner, projectCod, anonymize): for field in atable["fields"]: avalue[field["name"]] = value[field["name"]] if anonymize and atable["name"].endswith("lkpqst163_opts"): - avalue[ - "qst163_opts_des" - ] = f'Farmer #{avalue["qst163_opts_cod"]}' + avalue["qst163_opts_des"] = qst_163_pseudonym.replace( + "{}", str(avalue["qst163_opts_cod"]) + ) atable["values"].append(avalue) lktables.append(atable) return lktables @@ -376,7 +376,7 @@ def getData( fields.append(select_field_builder.build()) if anonymize: - to_remove_keys = ["instancename", "deviceimei"] + to_remove_keys = ["instancename", "deviceimei", "cal_qst163", "clc_after"] tmp_fields = fields.copy() fields = [] for field in tmp_fields: @@ -658,7 +658,7 @@ def getJSONResult( if os.path.exists(registryXML): data["registry"] = { "lkptables": getLookups( - registryXML, userOwner, projectCod, anonymize + registryXML, userOwner, projectCod, anonymize, request ), "fields": getFields(registryXML, "REG_geninfo"), } @@ -702,6 +702,7 @@ def getJSONResult( userOwner, projectCod, anonymize, + request, ), "fields": getFields( assessmentXML, diff --git a/climmob/views/productsList.py b/climmob/views/productsList.py index 225288d2..9e5390b8 100644 --- a/climmob/views/productsList.py +++ b/climmob/views/productsList.py @@ -135,7 +135,7 @@ def processView(self): product["extraInformation"] = None pattern = re.compile( r".+?(?:(?:Assessment))_" # not captured - r"([a-f0-9]{12})" # captured (group 1) + r"([a-f0-9]+)" # captured (group 1) ) match = pattern.fullmatch(product["process_name"]) if match: From 69c9f5b8191cfd60f26ff59e7efecf8f740ff347 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Tue, 7 Oct 2025 15:16:12 -0600 Subject: [PATCH 56/66] refactor and fix tests for ReadDataOfProjectViewApi --- climmob/tests/test_utils/common.py | 4 +- .../test_views_api_project_analysis.py | 66 ++++------------ climmob/views/Api/project_analysis.py | 75 ++++++------------- climmob/views/context/ApiContext.py | 6 +- 4 files changed, 40 insertions(+), 111 deletions(-) diff --git a/climmob/tests/test_utils/common.py b/climmob/tests/test_utils/common.py index 9c05cc19..ce73dacc 100644 --- a/climmob/tests/test_utils/common.py +++ b/climmob/tests/test_utils/common.py @@ -33,15 +33,17 @@ def get_mock(self, name): class ViewBaseTest(BaseTest): view_class = None request_method = "GET" + body = {} request_body = None def setUp(self): super().setUp() self.request = MagicMock() self.request.translate = self.mock_translation - with patch("climmob.views.classes.ApiContext"), patch( + with patch("climmob.views.classes.ApiContext") as api_context, patch( "climmob.views.classes.PrivateContext" ): + api_context.return_value.body = self.body self.view = self.view_class(self.request) self.view.request.method = self.request_method self.view.user = MagicMock(login="test_user") diff --git a/climmob/tests/test_utils/test_views_api_project_analysis.py b/climmob/tests/test_utils/test_views_api_project_analysis.py index 33c27ba0..d3a43ab8 100644 --- a/climmob/tests/test_utils/test_views_api_project_analysis.py +++ b/climmob/tests/test_utils/test_views_api_project_analysis.py @@ -2,6 +2,7 @@ import unittest from unittest.mock import patch, MagicMock +from climmob.tests.test_utils.common import ViewBaseTest from climmob.views.Api.project_analysis import ( ReadDataOfProjectViewApi, ReadVariablesForAnalysisViewApi, @@ -9,72 +10,31 @@ ) -class TestReadDataOfProjectViewAPI(unittest.TestCase): - def setUp(self): - self.view = ReadDataOfProjectViewApi(MagicMock()) - self.view.request.method = "GET" - self.view.user = MagicMock(login="test_user") - self.view.body = json.dumps({"project_cod": "123", "user_owner": "owner"}) - - def mock_translation(self, message): - return message +class TestReadDataOfProjectViewAPI(ViewBaseTest): + view_class = ReadDataOfProjectViewApi + body = {"project_cod": "123", "user_owner": "owner"} + request_body = json.dumps(body) @patch( "climmob.views.Api.project_analysis.getJSONResult", return_value={"data": "some_data"}, ) - @patch("climmob.views.Api.project_analysis.getTheProjectIdForOwner", return_value=1) - @patch("climmob.views.Api.project_analysis.projectExists", return_value=True) - def test_process_view_success( - self, mock_projectExists, mock_getTheProjectIdForOwner, mock_getJSONResult - ): + def test_get_success(self, mock_get_json_result): self.view._ = self.mock_translation # Mock translation function - response = self.view.processView() + response = self.view.get() self.assertEqual(response.status_code, 200) self.assertIn("some_data", response.body.decode()) - @patch("climmob.views.Api.project_analysis.projectExists", return_value=False) - def test_process_view_project_not_exist(self, mock_projectExists): - self.view._ = self.mock_translation # Mock translation function - - response = self.view.processView() - - self.assertEqual(response.status_code, 401) - self.assertIn("This project does not exist.", response.body.decode()) - - def test_process_view_invalid_json(self): - self.view._ = self.mock_translation # Mock translation function - self.view.body = '{"wrong_key": "value"}' - - response = self.view.processView() - - self.assertEqual(response.status_code, 401) - self.assertIn("Error in the JSON.", response.body.decode()) - - @patch("json.loads", side_effect=json.JSONDecodeError("Expecting value", "", 0)) - def test_process_view_invalid_body(self, mock_json_loads): - self.view._ = self.mock_translation # Mock translation function - self.view.body = "" - - response = self.view.processView() - - self.assertEqual(response.status_code, 401) - self.assertIn( - "Error in the JSON, It does not have the 'body' parameter.", - response.body.decode(), + mock_get_json_result.assert_called_once_with( + self.body["user_owner"], + self.view.context.active_project_id, + self.body["project_cod"], + self.view.request, + anonymize=True, ) - def test_process_view_post_method(self): - self.view._ = self.mock_translation # Mock translation function - self.view.request.method = "POST" - - response = self.view.processView() - - self.assertEqual(response.status_code, 401) - self.assertIn("Only accepts GET method.", response.body.decode()) - class TestReadVariablesForAnalysisViewAPI(unittest.TestCase): def setUp(self): diff --git a/climmob/views/Api/project_analysis.py b/climmob/views/Api/project_analysis.py index 4fef666f..cc445a9c 100644 --- a/climmob/views/Api/project_analysis.py +++ b/climmob/views/Api/project_analysis.py @@ -13,65 +13,32 @@ ) from climmob.views.classes import apiView from climmob.views.project_analysis import processToGenerateTheReport +from climmob.views.validators import TextField +from climmob.views.validators.ProjectExistsValidator import ProjectExistsValidator class ReadDataOfProjectViewApi(apiView): - def processView(self): - - if self.request.method == "GET": - - obligatory = ["project_cod", "user_owner"] - try: - dataworking = json.loads(self.body) - except: - response = Response( - status=401, - body=self._( - "Error in the JSON, It does not have the 'body' parameter." - ), - ) - return response - - if sorted(obligatory) == sorted(dataworking.keys()): - - exitsproject = projectExists( - self.user.login, - dataworking["user_owner"], - dataworking["project_cod"], + validators = (ProjectExistsValidator,) + valid_fields = ( + TextField("project_cod"), + TextField("user_owner"), + TextField("anonymize"), + ) + + def get(self): + response = Response( + status="200", + body=json.dumps( + getJSONResult( + self.context.body["user_owner"], + self.context.active_project_id, + self.context.body["project_cod"], self.request, + anonymize=True, ) - if exitsproject: - - activeProjectId = getTheProjectIdForOwner( - dataworking["user_owner"], - dataworking["project_cod"], - self.request, - ) - - response = Response( - status=200, - body=json.dumps( - getJSONResult( - dataworking["user_owner"], - activeProjectId, - dataworking["project_cod"], - self.request, - anonymize=True, - ) - ), - ) - return response - else: - response = Response( - status=401, body=self._("This project does not exist.") - ) - return response - else: - response = Response(status=401, body=self._("Error in the JSON.")) - return response - else: - response = Response(status=401, body=self._("Only accepts GET method.")) - return response + ), + ) + return response class ReadVariablesForAnalysisViewApi(apiView): diff --git a/climmob/views/context/ApiContext.py b/climmob/views/context/ApiContext.py index a1f11dd9..5f845fde 100644 --- a/climmob/views/context/ApiContext.py +++ b/climmob/views/context/ApiContext.py @@ -12,15 +12,15 @@ def __init__(self, request): super().__init__(request) @cached_property - def __body(self): + def body(self): body = get_body_from_api_request(self.request) return json.loads(body) @cached_property def active_project_id(self): active_project_id = getTheProjectIdForOwner( - self.__body["user_owner"], - self.__body["project_cod"], + self.body["user_owner"], + self.body["project_cod"], self.request, ) return active_project_id From 105f8e8d71dff6fb7fccffe222d88998c34cde1d Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Tue, 7 Oct 2025 15:34:34 -0600 Subject: [PATCH 57/66] create validator for project access --- .../test_views_api_project_analysis.py | 7 ++++++ .../tests/test_utils/test_views_validators.py | 23 ++++++++++++++++++- climmob/views/Api/project_analysis.py | 4 ++-- climmob/views/validators/project/__init__.py | 3 +++ .../has_access_to_project_validator.py | 15 ++++++++++++ 5 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 climmob/views/validators/project/has_access_to_project_validator.py diff --git a/climmob/tests/test_utils/test_views_api_project_analysis.py b/climmob/tests/test_utils/test_views_api_project_analysis.py index d3a43ab8..1cd8cb07 100644 --- a/climmob/tests/test_utils/test_views_api_project_analysis.py +++ b/climmob/tests/test_utils/test_views_api_project_analysis.py @@ -8,6 +8,8 @@ ReadVariablesForAnalysisViewApi, GenerateAnalysisByApiViewApi, ) +from climmob.views.validators.ProjectExistsValidator import ProjectExistsValidator +from climmob.views.validators.project import HasAccessToProjectValidator class TestReadDataOfProjectViewAPI(ViewBaseTest): @@ -15,6 +17,11 @@ class TestReadDataOfProjectViewAPI(ViewBaseTest): body = {"project_cod": "123", "user_owner": "owner"} request_body = json.dumps(body) + def test_has_validators(self): + self.assertEqual( + self.view.validators, (ProjectExistsValidator, HasAccessToProjectValidator) + ) + @patch( "climmob.views.Api.project_analysis.getJSONResult", return_value={"data": "some_data"}, diff --git a/climmob/tests/test_utils/test_views_validators.py b/climmob/tests/test_utils/test_views_validators.py index 57d34d8b..ec61d2cb 100644 --- a/climmob/tests/test_utils/test_views_validators.py +++ b/climmob/tests/test_utils/test_views_validators.py @@ -16,7 +16,10 @@ from climmob.views.validators.ProjectExistsValidator import ProjectExistsValidator from climmob.views.validators.assessment import AssessmentExistsValidator from climmob.views.validators.field.FieldValidator import FieldValidator -from climmob.views.validators.project import CanEditProjectValidator +from climmob.views.validators.project import ( + CanEditProjectValidator, + HasAccessToProjectValidator, +) from climmob.views.validators.question.QuestionMinMaxValidator import ( QuestionMinMaxValidator, @@ -139,6 +142,24 @@ def test_run_invalid(self): self.validator.run() +class TestHasAccessToProjectValidatorRun(unittest.TestCase): + def setUp(self): + self.request = MagicMock() + self.view = MagicMock() + self.view.request = self.request + + self.validator = HasAccessToProjectValidator(self.view) + + def test_run_valid(self): + self.view.context.access_type = MagicMock(int) + self.validator.run() + + def test_run_invalid(self): + self.view.context.access_type = None + with self.assertRaises(HTTPForbidden): + self.validator.run() + + class TestAssessmentExistsValidator(unittest.TestCase): def test_init_for_api(self): view = MagicMock(apiView) diff --git a/climmob/views/Api/project_analysis.py b/climmob/views/Api/project_analysis.py index cc445a9c..f6d5d769 100644 --- a/climmob/views/Api/project_analysis.py +++ b/climmob/views/Api/project_analysis.py @@ -15,14 +15,14 @@ from climmob.views.project_analysis import processToGenerateTheReport from climmob.views.validators import TextField from climmob.views.validators.ProjectExistsValidator import ProjectExistsValidator +from climmob.views.validators.project import HasAccessToProjectValidator class ReadDataOfProjectViewApi(apiView): - validators = (ProjectExistsValidator,) + validators = (ProjectExistsValidator, HasAccessToProjectValidator) valid_fields = ( TextField("project_cod"), TextField("user_owner"), - TextField("anonymize"), ) def get(self): diff --git a/climmob/views/validators/project/__init__.py b/climmob/views/validators/project/__init__.py index 46643210..56297915 100644 --- a/climmob/views/validators/project/__init__.py +++ b/climmob/views/validators/project/__init__.py @@ -1,3 +1,6 @@ from climmob.views.validators.project.CanEditProjectValidator import ( CanEditProjectValidator, ) +from climmob.views.validators.project.has_access_to_project_validator import ( + HasAccessToProjectValidator, +) diff --git a/climmob/views/validators/project/has_access_to_project_validator.py b/climmob/views/validators/project/has_access_to_project_validator.py new file mode 100644 index 00000000..b111dbc3 --- /dev/null +++ b/climmob/views/validators/project/has_access_to_project_validator.py @@ -0,0 +1,15 @@ +from pyramid.httpexceptions import HTTPForbidden + +from climmob.views.validators.BaseValidator import BaseValidator + + +class HasAccessToProjectValidator(BaseValidator): + def run(self): + access_type = self.view.context.access_type + + if access_type is None: + raise HTTPForbidden( + self._( + "The access assigned for this project does not allow you to get the collected data." + ) + ) From 832ebcb332c97fd0063508375c81a3881ab4a3e0 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Thu, 9 Oct 2025 11:46:41 -0600 Subject: [PATCH 58/66] fix circular import error --- climmob/processes/db/anonymized.py | 3 +-- climmob/processes/db/assessment.py | 5 ----- climmob/processes/db/registry.py | 7 +------ climmob/processes/db/results.py | 2 +- climmob/processes/odk/api.py | 13 +++++-------- climmob/views/cleanErrorLogs.py | 18 ++++++++++++++++++ 6 files changed, 26 insertions(+), 22 deletions(-) diff --git a/climmob/processes/db/anonymized.py b/climmob/processes/db/anonymized.py index 6cd3c91d..44b7cbe4 100644 --- a/climmob/processes/db/anonymized.py +++ b/climmob/processes/db/anonymized.py @@ -1,6 +1,7 @@ import re from datetime import datetime, date +from climmob.processes.db.results import getJSONResult from climmob.models.repository import sql_execute from climmob.processes.db.anonymization_params import get_anonymization_params_as_dict from climmob.processes.db.question import ( @@ -24,8 +25,6 @@ def anonymize_project(user_owner, project_id, project_code, request): - from climmob.processes import getJSONResult - questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) project_collected_data = getJSONResult( diff --git a/climmob/processes/db/assessment.py b/climmob/processes/db/assessment.py index c7b65a83..3cda2c0c 100644 --- a/climmob/processes/db/assessment.py +++ b/climmob/processes/db/assessment.py @@ -23,9 +23,6 @@ from climmob.models.repository import sql_fetch_one, sql_execute, execute_two_sqls from climmob.models.schema import mapFromSchema, mapToSchema -from climmob.processes.db.anonymized import ( - delete_anonymized_values_by_form_id_and_reg_id, -) from climmob.processes.db.project import ( addQuestionsToAssessment, numberOfCombinationsForTheProject, @@ -1780,5 +1777,3 @@ def delete_assessment_data_by_qst163(schema, ass_id, qst163, odk_user): "SET @odktools_current_user = '" + odk_user + "'; ", query, ) - - delete_anonymized_values_by_form_id_and_reg_id(schema, ass_id, qst163) diff --git a/climmob/processes/db/registry.py b/climmob/processes/db/registry.py index 40fbf9e1..78fbc37e 100644 --- a/climmob/processes/db/registry.py +++ b/climmob/processes/db/registry.py @@ -6,10 +6,7 @@ from climmob.models import Regsection, Registry, Project, Question, userProject from climmob.models.repository import execute_two_sqls, sql_execute from climmob.models.schema import mapFromSchema, mapToSchema -from climmob.processes import addRegistryQuestionsToProject -from climmob.processes.db.anonymized import ( - delete_anonymized_values_by_form_id_and_reg_id, -) +from climmob.processes.db.project import addRegistryQuestionsToProject from climmob.processes.db.assessment import setAssessmentStatus, formattingQuestions import climmob.plugins as p @@ -609,5 +606,3 @@ def delete_registry_data_by_qst162(schema, qst162, odk_user): "SET @odktools_current_user = '" + odk_user + "'; ", query, ) - - delete_anonymized_values_by_form_id_and_reg_id(schema, "-", qst162) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 7b899974..2e7c2226 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -6,7 +6,7 @@ from climmob.models import Assessment, Question, Project, mapFromSchema from climmob.models.repository import sql_fetch_all, sql_fetch_one -from climmob.processes import getCombinations +from climmob.processes.db.project_combinations import getCombinations from climmob.processes.db.anonymization_params import get_anonymization_params_as_dict from climmob.processes.db.question import ( get_sensitive_questions_anonymity_by_project_id, diff --git a/climmob/processes/odk/api.py b/climmob/processes/odk/api.py index 8e20fbd3..e6aa08fb 100644 --- a/climmob/processes/odk/api.py +++ b/climmob/processes/odk/api.py @@ -16,14 +16,11 @@ from pyramid.response import FileResponse from climmob.models import Project, storageErrors, Assessment -from climmob.processes import ( - isRegistryOpen, - isAssessmentOpen, - assessmentExists, - projectExists, - packageExist, - getTheProjectIdForOwner, -) + +from climmob.processes.db.registry import isRegistryOpen, packageExist +from climmob.processes.db.assessment import isAssessmentOpen, assessmentExists +from climmob.processes.db.validators import projectExists, getTheProjectIdForOwner + from climmob.processes.db.json import addJsonLog from climmob.processes.db.anonymized import anonymize_questions diff --git a/climmob/views/cleanErrorLogs.py b/climmob/views/cleanErrorLogs.py index 81da6ae5..e714e55e 100644 --- a/climmob/views/cleanErrorLogs.py +++ b/climmob/views/cleanErrorLogs.py @@ -19,6 +19,7 @@ getQuestionsStructure, delete_assessment_data_by_qst163, delete_registry_data_by_qst162, + delete_anonymized_values_by_form_id_and_reg_id, ) from climmob.processes.odk.api import storeJSONInMySQL from climmob.views.classes import privateView @@ -103,6 +104,11 @@ def processView(self): dataworking["newqst"].split("-")[1], self.user.login, ) + delete_anonymized_values_by_form_id_and_reg_id( + schema, + "-", + dataworking["newqst"].split("-")[1], + ) storeJSONInMySQL( self.user.login, @@ -159,6 +165,10 @@ def processView(self): self.user.login, ) + delete_anonymized_values_by_form_id_and_reg_id( + schema, codeId, dataworking["newqst2"] + ) + storeJSONInMySQL( self.user.login, "ASS", @@ -240,6 +250,11 @@ def processView(self): dataworking["newqst"].split("-")[1], self.user.login, ) + delete_anonymized_values_by_form_id_and_reg_id( + schema, + "-", + dataworking["newqst"].split("-")[1], + ) update_registry_status_log( self.request, @@ -268,6 +283,9 @@ def processView(self): dataworking["newqst2"], self.user.login, ) + delete_anonymized_values_by_form_id_and_reg_id( + schema, codeId, dataworking["newqst2"] + ) update_assessment_status_log( self.request, From 8ff18584cc4393686f66da2e100208a1885dafd2 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 10 Oct 2025 08:47:38 -0600 Subject: [PATCH 59/66] add script to anonymize project --- climmob/processes/db/anonymized.py | 7 +++- climmob/processes/db/project.py | 11 ++++++ climmob/processes/db/userproject.py | 14 ++++++- climmob/scripts/anonymize_project.py | 59 ++++++++++++++++++++++++++++ setup.py | 1 + 5 files changed, 89 insertions(+), 3 deletions(-) create mode 100644 climmob/scripts/anonymize_project.py diff --git a/climmob/processes/db/anonymized.py b/climmob/processes/db/anonymized.py index 44b7cbe4..9dce2432 100644 --- a/climmob/processes/db/anonymized.py +++ b/climmob/processes/db/anonymized.py @@ -1,6 +1,7 @@ import re from datetime import datetime, date +from climmob.processes import get_project_cod_by_id, get_owner_user_name_by_project_id from climmob.processes.db.results import getJSONResult from climmob.models.repository import sql_execute from climmob.processes.db.anonymization_params import get_anonymization_params_as_dict @@ -24,7 +25,9 @@ ] -def anonymize_project(user_owner, project_id, project_code, request): +def anonymize_project(project_id, request): + project_code = get_project_cod_by_id(project_id, request) + user_owner = get_owner_user_name_by_project_id(project_id, request) questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) project_collected_data = getJSONResult( @@ -72,7 +75,7 @@ def anonymize_project(user_owner, project_id, project_code, request): continue return False, msg - return project_collected_data + return True, "" def anonymize_questions(request, form, form_id, project_id, user_owner, project_cod): diff --git a/climmob/processes/db/project.py b/climmob/processes/db/project.py index b86062d0..d301aa00 100644 --- a/climmob/processes/db/project.py +++ b/climmob/processes/db/project.py @@ -62,9 +62,20 @@ "getProjectFullDetailsById", "getProjectsByUserThatRequireSetup", "update_project_status", + "get_project_cod_by_id", ] +def get_project_cod_by_id(project_id, request): + res = mapFromSchema( + request.dbsession.query(Project.project_cod) + .filter(Project.project_id == project_id) + .first() + ) + + return res["project_cod"] + + def getTotalNumberOfProjectsInClimMob(request): res = request.dbsession.query(Project).count() diff --git a/climmob/processes/db/userproject.py b/climmob/processes/db/userproject.py index 111e464c..2a5f2c6a 100644 --- a/climmob/processes/db/userproject.py +++ b/climmob/processes/db/userproject.py @@ -1,6 +1,8 @@ from climmob.models import userProject, mapFromSchema -__all__ = ["getAllProjectsByUser"] +__all__ = ["getAllProjectsByUser", "get_owner_user_name_by_project_id"] + +from climmob.utility.project import ProjectAccessType def getAllProjectsByUser(user, request): @@ -10,3 +12,13 @@ def getAllProjectsByUser(user, request): .first() ) return mappedData + + +def get_owner_user_name_by_project_id(project_id, request): + mappedData = mapFromSchema( + request.dbsession.query(userProject.user_name) + .filter(userProject.project_id == project_id) + .filter(userProject.access_type == ProjectAccessType.OWNER.value) + .first() + ) + return mappedData["user_name"] diff --git a/climmob/scripts/anonymize_project.py b/climmob/scripts/anonymize_project.py new file mode 100644 index 00000000..058bae49 --- /dev/null +++ b/climmob/scripts/anonymize_project.py @@ -0,0 +1,59 @@ +import sys + +from pyramid.paster import get_appsettings, setup_logging +from climmob.models import ( + get_engine, + Base, + get_tm_session, + get_session_factory, + initialize_schema, +) +import transaction +import requests +import argparse +import pyramid +import os + +from climmob.processes import anonymize_project + + +def main(raw_args=None): + parser = argparse.ArgumentParser() + parser.add_argument("ini_path", help="Path to ini file") + parser.add_argument("project_id", help="Project id") + args = parser.parse_args(raw_args) + + if not os.path.exists(os.path.abspath(args.ini_path)): + print("Ini file does not exists") + sys.exit(1) + + settings = get_appsettings(args.ini_path, "climmob") + + engine = get_engine( + settings, + ) + + Base.metadata.create_all(engine) + session_factory = get_session_factory(engine) + + with transaction.manager: + + dbsession = get_tm_session(session_factory, transaction.manager) + setup_logging(args.ini_path) + + request = requests.Session() + request.dbsession = dbsession + request.registry = pyramid.registry.Registry + request.registry.settings = settings + request.locale_name = "en" + + initialize_schema() + + success, msg = anonymize_project(args.project_id, request) + + if success: + print(f"Successfully anonymized project {args.project_id}") + else: + print(f"Failed to anonymized project {args.project_id}") + + engine.dispose() diff --git a/setup.py b/setup.py index c860bda8..99ee5b68 100644 --- a/setup.py +++ b/setup.py @@ -224,6 +224,7 @@ "update_map_points = climmob.scripts.updatemappoints:main", "mysqldumps_climmob_dbs = climmob.scripts.mysqldumpclimmobdbs:main", "configure_tests = climmob.scripts.configuretests:main", + "anonymize_project = climmob.scripts.anonymize_project:main", ], }, ) From 94e2b8093c9b5a28f7a0627c20d9cbd13b6eb1d5 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 10 Oct 2025 11:51:05 -0600 Subject: [PATCH 60/66] add tests for get_question_by_field_name --- .../tests/test_utils/test_utility_question.py | 185 +++++++++++++++++- 1 file changed, 184 insertions(+), 1 deletion(-) diff --git a/climmob/tests/test_utils/test_utility_question.py b/climmob/tests/test_utils/test_utility_question.py index c88533f7..1b88f438 100644 --- a/climmob/tests/test_utils/test_utility_question.py +++ b/climmob/tests/test_utils/test_utility_question.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import MagicMock -from climmob.utility import is_type_numerical, QuestionType +from climmob.utility import is_type_numerical, QuestionType, get_question_by_field_name class TestIsTypeNumerical(unittest.TestCase): @@ -39,3 +39,186 @@ def test_false(self): result = is_type_numerical(q_type) self.assertFalse(result) + + +class TestGetQuestionByFieldName(unittest.TestCase): + def setUp(self): + self.question_codes = ["qst_a_test", "qst_b_test"] + self.questions = [ + MagicMock(question_code=self.question_codes[0]), + MagicMock(question_code=self.question_codes[1]), + ] + + def test_simple(self): + field_name = self.question_codes[1] + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_a(self): + field_name = self.question_codes[1] + "_a" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_b(self): + field_name = self.question_codes[1] + "_b" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_c(self): + field_name = self.question_codes[1] + "_c" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_a_oth(self): + field_name = self.question_codes[1] + "_a_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_b_oth(self): + field_name = self.question_codes[1] + "_b_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_c_oth(self): + field_name = self.question_codes[1] + "_c_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_oth(self): + field_name = self.question_codes[1] + "_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_perf_1(self): + field_name = "perf_" + self.question_codes[1] + "_1" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_perf_2(self): + field_name = "perf_" + self.question_codes[1] + "_2" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_perf_3(self): + field_name = "perf_" + self.question_codes[1] + "_3" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_char_pos(self): + field_name = "char_" + self.question_codes[1] + "_pos" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_char_neg(self): + field_name = "char_" + self.question_codes[1] + "_neg" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_unknown_suffix(self): + field_name = self.question_codes[1] + "_unknown_suffix" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_prefix(self): + field_name = "unknown_prefix_" + self.question_codes[1] + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_prefix_and_suffix(self): + field_name = "unknown_prefix_" + self.question_codes[1] + "_unknown_suffix" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_simple(self): + field_name = "unknown_question_code" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_a(self): + field_name = "unknown_question_code" + "_a" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_b(self): + field_name = "unknown_question_code" + "_b" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_c(self): + field_name = "unknown_question_code" + "_c" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_a_oth(self): + field_name = "unknown_question_code" + "_a_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_b_oth(self): + field_name = "unknown_question_code" + "_b_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_c_oth(self): + field_name = "unknown_question_code" + "_c_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_oth(self): + field_name = "unknown_question_code" + "_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_perf_1(self): + field_name = "perf_" + "unknown_question_code" + "_1" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_perf_2(self): + field_name = "perf_" + "unknown_question_code" + "_2" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_perf_3(self): + field_name = "perf_" + "unknown_question_code" + "_3" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_char_pos(self): + field_name = "char_" + "unknown_question_code" + "_pos" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_char_neg(self): + field_name = "char_" + "unknown_question_code" + "_neg" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) From 454bf0b3dfa1f9a62eb2b77465c4bdad94d7158c Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 10 Oct 2025 13:26:34 -0600 Subject: [PATCH 61/66] optimization 1 for get_question_by_field_name --- climmob/utility/question.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/climmob/utility/question.py b/climmob/utility/question.py index c561d011..1bed66d5 100644 --- a/climmob/utility/question.py +++ b/climmob/utility/question.py @@ -151,12 +151,13 @@ def get_question_types_with_anonymity_labeled(request): def get_question_by_field_name(field_name, questions): for q in questions: - patterns = [ - rf"^{q.question_code}(_[abc])?(_oth)?$", - rf"^perf_{q.question_code}_[123]$", - rf"^char_{q.question_code}_(pos|neg)$", - ] - for pattern in patterns: - if re.fullmatch(pattern, field_name): - return q + pattern = ( + rf"^" + rf"({q.question_code}(_[abc])?(_oth)?)|" + rf"(perf_{q.question_code}_[123])|" + rf"(char_{q.question_code}_(pos|neg))" + rf"$" + ) + if re.fullmatch(pattern, field_name): + return q return None From 0fd11a697b21b697856aba78bb4acddb93170faf Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Mon, 3 Nov 2025 13:30:31 -0600 Subject: [PATCH 62/66] add tool tips and change texts for question sensitivity --- .../snippets/question/question-form.jinja2 | 52 ++++++++++++++++--- climmob/utility/question.py | 2 +- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/climmob/templates/snippets/question/question-form.jinja2 b/climmob/templates/snippets/question/question-form.jinja2 index d82808cc..fdf09f0e 100644 --- a/climmob/templates/snippets/question/question-form.jinja2 +++ b/climmob/templates/snippets/question/question-form.jinja2 @@ -535,7 +535,7 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %}
- +
- -
+ +
+{# TODO: Alternate tool tips dynamically#} +{# #} +{# #} +{# #} +{# #} + +
- +
- +
- +
@@ -576,10 +603,19 @@
- +
- {{ _("Short template for the pseudonym, e.g., 'Farmer-{}'. The curly braces will be replaced with the package code.") }} + {{ _("Enter the prefix you want for the anonymous ID. + The curly braces {} must be included and will be automatically replaced with the unique package code. + Example: Farmer-{}") }}
diff --git a/climmob/utility/question.py b/climmob/utility/question.py index 1bed66d5..721dbc6c 100644 --- a/climmob/utility/question.py +++ b/climmob/utility/question.py @@ -91,7 +91,7 @@ class QuestionAnonymity(IntEnum): class QuestionAnonymityLabel(Enum): REMOVE = _("Remove") PSEUDONYM = _("Pseudonym") - RANGE = _("Range") + RANGE = _("Binning") NOISE = _("Noise") MASK = _("Mask") MONTH_YEAR = _("Month-Year") From a3d6a22a6ad0a3233ca8c8ff92dd08922787b730 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Fri, 19 Dec 2025 09:05:25 -0600 Subject: [PATCH 63/66] fix: catch database error --- climmob/views/dashboard.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/climmob/views/dashboard.py b/climmob/views/dashboard.py index 1b719aaa..e0af3ca5 100644 --- a/climmob/views/dashboard.py +++ b/climmob/views/dashboard.py @@ -1,4 +1,6 @@ +import mysql.connector from pyramid.httpexceptions import HTTPNotFound, HTTPFound +from sqlalchemy.exc import ProgrammingError import climmob.plugins as p from climmob.processes import ( @@ -43,12 +45,15 @@ def processView(self): activeProjectData = getActiveProject(self.user.login, self.request) schema = ( - activeProjectData["user_name"] + activeProjectData["owner"]["user_name"] + "_" + activeProjectData["project_cod"] ) - project_is_anonymized = is_project_anonymized(schema) + try: + project_is_anonymized = is_project_anonymized(schema) + except ProgrammingError: + project_is_anonymized = False session = self.request.session session["activeProject"] = activeProjectId @@ -119,10 +124,15 @@ def processView(self): activeProjectData = getActiveProject(self.user.login, self.request) schema = ( - activeProjectData["user_name"] + "_" + activeProjectData["project_cod"] + activeProjectData["owner"]["user_name"] + + "_" + + activeProjectData["project_cod"] ) - project_is_anonymized = is_project_anonymized(schema) + try: + project_is_anonymized = is_project_anonymized(schema) + except ProgrammingError: + project_is_anonymized = False if activeProjectData: self.returnRawViewResult = True From 0198e83cedb639c1f4889e7a0970d3cb066ee643 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Tue, 13 Jan 2026 15:16:35 -0600 Subject: [PATCH 64/66] update texts for anonymization tooltips --- .../snippets/dashboard/downloads.jinja2 | 16 +++++----- .../snippets/question/question-form.jinja2 | 30 ++++++++++++++----- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/climmob/templates/snippets/dashboard/downloads.jinja2 b/climmob/templates/snippets/dashboard/downloads.jinja2 index 454b440e..b82065ba 100644 --- a/climmob/templates/snippets/dashboard/downloads.jinja2 +++ b/climmob/templates/snippets/dashboard/downloads.jinja2 @@ -55,7 +55,7 @@ CSV - - + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} - - + {% endif %} @@ -92,7 +92,7 @@ XLSX - - + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} - - + {% endif %} diff --git a/climmob/templates/snippets/question/question-form.jinja2 b/climmob/templates/snippets/question/question-form.jinja2 index fdf09f0e..e33f4ffd 100644 --- a/climmob/templates/snippets/question/question-form.jinja2 +++ b/climmob/templates/snippets/question/question-form.jinja2 @@ -551,18 +551,24 @@
-{# TODO: Alternate tool tips dynamically#} +{# TODO: Alternate tool tips dynamically #} +{# REMOVAL #} {# #} {# #} +{# PSEUDO #} {# #} +{# title="{{ _("Replaces the source ID with a unique code to protect identity while allowing data#} +{# tracking across forms.") }}">#} {# #} +{# BINNING #} +{# NOISE #} +{# #} +{# #}
@@ -598,6 +604,12 @@
+ + {{ _("Define specific ranges to group your data (e.g., age or yield). Setting custom bounds + gives you full control over how your distribution is analyzed while ensuring privacy. + Example: Entering 0 (Lower), 100 (Upper), and 10 (Interval) will automatically group your + data into sets of 10 (0-10, 11-20, etc.).") }} +
@@ -612,10 +624,12 @@ {{ _("Pseudonym") }}
- - {{ _("Enter the prefix you want for the anonymous ID. - The curly braces {} must be included and will be automatically replaced with the unique package code. - Example: Farmer-{}") }} + + + {{ _("Enter the prefix for the anonymous ID. You must include {} as a placeholder, + which will be replaced by a unique number. Example: Farmer-{} becomes Farmer-1") }} +
From c81b95043fef40000b6ad1bd9fd8c842151d6590 Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Tue, 3 Feb 2026 10:51:37 -0600 Subject: [PATCH 65/66] group bound inputs --- .../snippets/question/question-form.jinja2 | 39 +++++++------------ 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/climmob/templates/snippets/question/question-form.jinja2 b/climmob/templates/snippets/question/question-form.jinja2 index e33f4ffd..7a0abc8a 100644 --- a/climmob/templates/snippets/question/question-form.jinja2 +++ b/climmob/templates/snippets/question/question-form.jinja2 @@ -557,13 +557,11 @@ {# #} {# PSEUDO #} {# #} +{# title="{{ _("Replaces the source ID with a unique code to protect identity while allowing data tracking across forms.") }}">#} {# #} {# BINNING #} + title="{{ _("This method anonymizes numerical values by grouping them into bins. Define the lower and upper bounds and an interval to determine the boundaries of each bin.") }}"> {# NOISE #} {#
-
+
-
- -
-
-
- -
- +
+ +
@@ -604,12 +592,11 @@
- - {{ _("Define specific ranges to group your data (e.g., age or yield). Setting custom bounds - gives you full control over how your distribution is analyzed while ensuring privacy. - Example: Entering 0 (Lower), 100 (Upper), and 10 (Interval) will automatically group your - data into sets of 10 (0-10, 11-20, etc.).") }} - + + {{ _("Group your data into specific ranges. Setting custom bounds ensures privacy while + maintaining control over data distribution. Example: 0 (lower), 100 (upper), and 10 (Interval) + creates groups of 10 (0-10, 11-20, etc.).") }} +
From 6ceb998cc6599db8e8c6df88968de586b6368b8e Mon Sep 17 00:00:00 2001 From: JohannMrBot Date: Tue, 3 Feb 2026 16:34:56 -0600 Subject: [PATCH 66/66] alternate tooltip dynamically --- climmob/templates/question/library.jinja2 | 9 ++++++ .../snippets/question/question-form.jinja2 | 28 +++++++++---------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/climmob/templates/question/library.jinja2 b/climmob/templates/question/library.jinja2 index a1e27f89..7dce1049 100644 --- a/climmob/templates/question/library.jinja2 +++ b/climmob/templates/question/library.jinja2 @@ -297,15 +297,24 @@ function setAnonymityInputsDisplay(show, anonymity=-1) { const display = show ? 'block' : 'none' anonymity_container.css('display', display); + $('.anonymity-description').css('display', 'none'); anonymity_inputs_selector.prop('disabled', true); $('#question-anonymization-params > *').css('display', 'none'); if (anonymity === {{ QuestionAnonymity.PSEUDONYM }}) { $('#pseudonym-params').css('display', 'block'); $('#pseudonym-params input').prop('disabled', false); + $('#anonymity-description-pseudo').css('display', 'block'); } else if (anonymity === {{ QuestionAnonymity.RANGE }}) { $('#range-params').css('display', 'block'); $('#range-params input').prop('disabled', false); + $('#anonymity-description-range').css('display', 'block'); + } + else if (anonymity === {{ QuestionAnonymity.REMOVE }}) { + $('#anonymity-description-remove').css('display', 'block'); + } + else if (anonymity === {{ QuestionAnonymity.NOISE }}) { + $('#anonymity-description-noise').css('display', 'block'); } } diff --git a/climmob/templates/snippets/question/question-form.jinja2 b/climmob/templates/snippets/question/question-form.jinja2 index 7a0abc8a..667cfc09 100644 --- a/climmob/templates/snippets/question/question-form.jinja2 +++ b/climmob/templates/snippets/question/question-form.jinja2 @@ -551,22 +551,22 @@
-{# TODO: Alternate tool tips dynamically #} -{# REMOVAL #} -{# #} -{# #} +{# REMOVE #} + {# PSEUDO #} -{# #} -{# #} + {# BINNING #} - + {# NOISE #} -{# #} -{# #} +
@@ -593,9 +593,7 @@
- {{ _("Group your data into specific ranges. Setting custom bounds ensures privacy while - maintaining control over data distribution. Example: 0 (lower), 100 (upper), and 10 (Interval) - creates groups of 10 (0-10, 11-20, etc.).") }} + {{ _("E.g., 0 (lower), 100 (upper), and 10 (interval) create groups of 10.") }}