diff --git a/climmob/models/climmobv4.py b/climmob/models/climmobv4.py index edd4ddba..050b5caf 100644 --- a/climmob/models/climmobv4.py +++ b/climmob/models/climmobv4.py @@ -787,6 +787,14 @@ class Question_subgroup(Base): parent_id = Column(Unicode(80), primary_key=True, nullable=True) +class QuestionType(Base): + __tablename__ = "question_type" + + id = Column(Integer, primary_key=True, nullable=False) + name = Column(Unicode(64), nullable=False) + order = Column(Integer, nullable=False) + + class Question(Base): __tablename__ = "question" __table_args__ = ( @@ -805,7 +813,7 @@ class Question(Base): question_unit = Column(Unicode(120)) question_min = Column(Float, nullable=True) question_max = Column(Float, nullable=True) - question_dtype = Column(Integer) + question_dtype = Column(Integer, ForeignKey("question_type.id")) question_cmp = Column(Unicode(120)) question_reqinreg = Column(Integer, server_default=text("'0'")) question_reqinasses = Column(Integer, server_default=text("'0'")) @@ -835,12 +843,39 @@ class Question(Base): qstgroups_user = Column(Unicode(80), nullable=True) qstgroups_id = Column(Unicode(80), nullable=True) question_sensitive = Column(Integer, server_default=text("'0'")) + question_anonymity = Column(Integer, ForeignKey("question_anonymity.id")) question_lang = Column(ForeignKey("i18n.lang_code"), nullable=True) extra = Column(MEDIUMTEXT(collation="utf8mb4_unicode_ci")) i18n = relationship("I18n") user = relationship("User") +class QuestionAnonymity(Base): + __tablename__ = "question_anonymity" + + id = Column(Integer, primary_key=True, nullable=False) + name = Column(Unicode(64), nullable=False) + + +class QuestionTypeAnonymity(Base): + __tablename__ = "question_type_anonymity" + + type_id = Column( + Integer, ForeignKey("question_type.id"), primary_key=True, nullable=False + ) + anonymity_id = Column( + Integer, ForeignKey("question_anonymity.id"), primary_key=True, nullable=False + ) + + +class AnonymizationParameter(Base): + __tablename__ = "anonymization_parameter" + + question_id = Column(Integer, primary_key=True, nullable=False) + name = Column(Unicode(64), primary_key=True, nullable=False) + value = Column(Unicode(64), nullable=False) + + class Registry(Base): __tablename__ = "registry" __table_args__ = ( diff --git a/climmob/processes/__init__.py b/climmob/processes/__init__.py index a73bd751..540f2719 100644 --- a/climmob/processes/__init__.py +++ b/climmob/processes/__init__.py @@ -50,3 +50,5 @@ from climmob.processes.db.project_location_unit_objective import * from climmob.processes.db.location_unit_of_analysis_objectives import * from climmob.processes.db.affiliation import * +from climmob.processes.db.anonymized import * +from climmob.processes.db.anonymization_params import * diff --git a/climmob/processes/db/anonymization_params.py b/climmob/processes/db/anonymization_params.py new file mode 100644 index 00000000..bab999b1 --- /dev/null +++ b/climmob/processes/db/anonymization_params.py @@ -0,0 +1,50 @@ +__all__ = ["save_anonymization_params", "get_anonymization_params_as_dict"] + +import re + +from climmob.models import mapFromSchema +from climmob.models.climmobv4 import AnonymizationParameter + + +def get_anonymization_params(question_id, request): + result = mapFromSchema( + request.dbsession.query(AnonymizationParameter) + .filter(AnonymizationParameter.question_id == question_id) + .all() + ) + return result + + +def get_anonymization_params_as_dict(question_id, request): + params = get_anonymization_params(question_id, request) + result = {} + for param in params: + result[param["name"]] = param["value"] + return result + + +def save_anonymization_params(question_id, data, request): + delete_existing_anonymization_params(question_id, request) + + params = [] + for key in data.keys(): + pattern = r"anonym_param_([a-z_]+)" + match = re.match(pattern, key) + if match: + params.append({"name": match.group(1), "value": data[key]}) + + for param in params: + new_param = AnonymizationParameter(**param) + new_param.question_id = question_id + request.dbsession.add(new_param) + request.dbsession.flush() + + +def delete_existing_anonymization_params(question_id, request): + try: + request.dbsession.query(AnonymizationParameter).filter( + AnonymizationParameter.question_id == question_id + ).delete() + return True, "" + except Exception as e: + return False, str(e) diff --git a/climmob/processes/db/anonymized.py b/climmob/processes/db/anonymized.py new file mode 100644 index 00000000..9dce2432 --- /dev/null +++ b/climmob/processes/db/anonymized.py @@ -0,0 +1,246 @@ +import re +from datetime import datetime, date + +from climmob.processes import get_project_cod_by_id, get_owner_user_name_by_project_id +from climmob.processes.db.results import getJSONResult +from climmob.models.repository import sql_execute +from climmob.processes.db.anonymization_params import get_anonymization_params_as_dict +from climmob.processes.db.question import ( + get_sensitive_questions_anonymity_by_project_id, +) +from climmob.utility import ( + get_question_by_field_name, + QuestionAnonymity, + add_noise_to_gps_coordinates, + QuestionType, +) + +__all__ = [ + "anonymize_questions", + "delete_anonymized_values_by_form_id", + "delete_anonymized_values_by_form_id_and_reg_id", + "update_anonymized", + "anonymize_project", + "is_project_anonymized", +] + + +def anonymize_project(project_id, request): + project_code = get_project_cod_by_id(project_id, request) + user_owner = get_owner_user_name_by_project_id(project_id, request) + questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) + + project_collected_data = getJSONResult( + user_owner, project_id, project_code, request + )["data"] + + schema = user_owner + "_" + project_code + + pattern = r"(REG|(ASS(.+?)))_(.*)" + for entry in project_collected_data: + reg_id = entry["REG_qst162"] + to_anonymize = [] + for key in entry.keys(): + if entry[key] is None: + continue + match = re.match(pattern, key) + if match is None: + continue + question = get_question_by_field_name(match.group(4), questions) + if ( + question + and question.question_anonymity != QuestionAnonymity.REMOVE.value + ): + if match.group(1) == "REG": + form_id = "-" + else: + form_id = match.group(3) + to_anonymize.append( + { + "field_name": match.group(4), + "value": entry[key], + "question": question, + "form_id": form_id, + } + ) + + for field in to_anonymize: + anonymize_field_value(field, reg_id, request) + success, msg = insert_anonymized_field( + field, field["form_id"], reg_id, schema + ) + if not success: + if msg.startswith("Duplicate entry for package"): + # To ignore entries that are already anonymized + continue + return False, msg + + return True, "" + + +def anonymize_questions(request, form, form_id, project_id, user_owner, project_cod): + questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) + + registry_id = None + + schema = user_owner + "_" + project_cod + + pattern = r"grp_\d+/(.+)" + to_anonymize = [] + + for key in form.keys(): + match = re.fullmatch(pattern, key) + if not match: + continue + field_name = match.group(1) + + if field_name == "QST162" or field_name == "QST163": + match = re.fullmatch(rf"({user_owner}-)?(\d+)(-{project_cod}~)?", form[key]) + if not match: + return False, "Could not anonymize" + registry_id = match.group(2) + continue + + question = get_question_by_field_name(field_name, questions) + if question and question.question_anonymity != QuestionAnonymity.REMOVE.value: + to_anonymize.append( + {"field_name": field_name, "value": form[key], "question": question} + ) + + if not to_anonymize: + return True + + for field in to_anonymize: + anonymize_field_value(field, registry_id, request) + success, msg = insert_anonymized_field(field, form_id, registry_id, schema) + if not success: + return False, msg + + return True, "" + + +def anonymize_field_value(field, registry_id, request): + params = get_anonymization_params_as_dict(field["question"].question_id, request) + if field["question"].question_anonymity == QuestionAnonymity.PSEUDONYM.value: + field["value"] = params["pseudonym"].replace("{}", registry_id) + elif field["question"].question_anonymity == QuestionAnonymity.RANGE.value: + if field["question"].question_dtype == QuestionType.INTEGER.value: + parser = int + else: + parser = float + + field["value"] = parser(field["value"]) + params["lower_bound"] = parser(params["lower_bound"]) + params["upper_bound"] = parser(params["upper_bound"]) + params["interval"] = parser(params["interval"]) + + if field["value"] < params["lower_bound"]: + field["value"] = f'<{params["lower_bound"]}' + elif field["value"] > params["upper_bound"]: + field["value"] = f'>{params["upper_bound"]}' + else: + i = params["lower_bound"] + while i < params["upper_bound"]: + if i <= field["value"] < (i + params["interval"]): + field["value"] = f'{i}-{i + params["interval"]}' + break + i += params["interval"] + elif field["question"].question_anonymity == QuestionAnonymity.MONTH_YEAR.value: + dt = datetime.fromisoformat(field["value"]) + field["value"] = dt.strftime("%Y-%m") + elif field["question"].question_anonymity == QuestionAnonymity.NOISE.value: + geo_point = field["value"].split() + geo_point[0], geo_point[1] = add_noise_to_gps_coordinates( + float(geo_point[0]), float(geo_point[1]), 3000 + ) + if geo_point[0] == "Error" or geo_point[1] == "Error": + return False, "Could not anonymize GeoPoint" + field["value"] = " ".join(geo_point) + + return True, "" + + +def insert_anonymized_field(field, form_id, registry_id, schema): + sql_insert_value = ( + f"(" + f"'{form_id}', " + f"'{registry_id}', " + f"'{field['field_name']}', " + f"'{field['value']}'" + f")" + ) + sql = f"INSERT INTO {schema}.anonymized VALUES {sql_insert_value}" + try: + sql_execute(sql) + return True, "" + except Exception as e: + match = re.search(rf"Duplicate entry '({form_id})-(\d+)-(.+?)'", str(e)) + if match: + form_name = "registry" if form_id == "-" else f"assessment '{form_id}'" + msg = f"Duplicate entry for package '{match.group(2)}' in {form_name}" + return False, msg + return False, "" + + +def update_anonymized(to_anonymize, schema, form_id, registry_id, request, current): + for field in to_anonymize: + db_type = type(current[field["field_name"]]) + if db_type == date: + new_value = date.fromisoformat(field["value"]) + elif db_type == datetime: + new_value = datetime.fromisoformat(field["value"]) + else: + new_value = db_type(field["value"]) + if current[field["field_name"]] == new_value: + # Only changed values will be updated to avoid recalculating anonymizations + continue + anonymize_field_value(field, registry_id, request) + success, msg = update_anonymized_field(field, form_id, registry_id, schema) + if not success: + return False, msg + return True, "" + + +def update_anonymized_field(field, form_id, registry_id, schema): + sql = ( + f"UPDATE {schema}.anonymized SET value='{field['value']}' " + f"WHERE form_id='{form_id}' " + f"AND reg_id='{registry_id}' " + f"AND col_name='{field['field_name']}'" + ) + try: + sql_execute(sql) + return True, "" + except Exception as e: + return False, str(e) + + +def delete_anonymized_values_by_form_id(schema, form_id): + sql = f"DELETE FROM {schema}.anonymized where form_id='{form_id}'" + sql_execute(sql) + + +def delete_anonymized_values_by_form_id_and_reg_id(schema, form_id, reg_id): + query = ( + f"DELETE FROM {schema}.anonymized " + f"WHERE form_id='{form_id}' " + f"AND reg_id='{reg_id}'" + ) + sql_execute(query) + + +def is_project_anonymized(schema): + query = f""" + SELECT + (SELECT + COUNT(DISTINCT reg_id) AS count + FROM + {schema}.anonymized + WHERE + form_id = '-') = (SELECT + COUNT(qst162) AS count + FROM + {schema}.REG_geninfo) AS count_matches """ + + result = sql_execute(query).first() + return result["count_matches"] == 1 diff --git a/climmob/processes/db/assessment.py b/climmob/processes/db/assessment.py index 65f1f2d4..3cda2c0c 100644 --- a/climmob/processes/db/assessment.py +++ b/climmob/processes/db/assessment.py @@ -21,7 +21,7 @@ I18nQstoption, ) -from climmob.models.repository import sql_fetch_one, sql_execute +from climmob.models.repository import sql_fetch_one, sql_execute, execute_two_sqls from climmob.models.schema import mapFromSchema, mapToSchema from climmob.processes.db.project import ( addQuestionsToAssessment, @@ -82,6 +82,8 @@ "clone_assessment", "copy_assessment_questions", "copy_assessment_sections", + "delete_assessment_data_by_qst163", + "get_assessment_data_by_qst163", ] log = logging.getLogger(__name__) @@ -1746,3 +1748,32 @@ def getFinalizedAssessments(request, userOwner, projectCod, projectId): ) return result + + +def get_assessment_data_by_qst163(schema, ass_id, qst163, columns): + query = ( + f"SELECT {','.join(columns)} FROM " + + schema + + ".ASS" + + ass_id + + "_geninfo WHERE qst163='" + + qst163 + + "'" + ) + return sql_execute(query).fetchone() + + +def delete_assessment_data_by_qst163(schema, ass_id, qst163, odk_user): + query = ( + "DELETE FROM " + + schema + + ".ASS" + + ass_id + + "_geninfo WHERE qst163='" + + qst163 + + "'" + ) + execute_two_sqls( + "SET @odktools_current_user = '" + odk_user + "'; ", + query, + ) diff --git a/climmob/processes/db/project.py b/climmob/processes/db/project.py index b86062d0..d301aa00 100644 --- a/climmob/processes/db/project.py +++ b/climmob/processes/db/project.py @@ -62,9 +62,20 @@ "getProjectFullDetailsById", "getProjectsByUserThatRequireSetup", "update_project_status", + "get_project_cod_by_id", ] +def get_project_cod_by_id(project_id, request): + res = mapFromSchema( + request.dbsession.query(Project.project_cod) + .filter(Project.project_id == project_id) + .first() + ) + + return res["project_cod"] + + def getTotalNumberOfProjectsInClimMob(request): res = request.dbsession.query(Project).count() diff --git a/climmob/processes/db/question.py b/climmob/processes/db/question.py index 20aa0655..875f97db 100644 --- a/climmob/processes/db/question.py +++ b/climmob/processes/db/question.py @@ -1,4 +1,6 @@ import json +import re +from datetime import datetime from sqlalchemy import func, or_, and_ @@ -40,8 +42,15 @@ "getDefaultQuestionLanguage", "getQuestionOwner", "knowIfUserHasCreatedTranslations", + "get_sensitive_questions_anonymity_by_project_id", + "get_registry_key_question", + "get_assessment_key_question", ] +from climmob.models.climmobv4 import AnonymizationParameter +from climmob.processes.db.anonymization_params import save_anonymization_params + + log = logging.getLogger(__name__) @@ -53,6 +62,7 @@ def addQuestion(data, request): try: request.dbsession.add(newQuestion) request.dbsession.flush() + save_anonymization_params(newQuestion.question_id, data, request) return True, newQuestion.question_id except DatabaseError as e: save_point.rollback() @@ -129,6 +139,7 @@ def updateQuestion(data, request): request.dbsession.query(Question).filter( Question.user_name == data["user_name"] ).filter(Question.question_id == data["question_id"]).update(mappeData) + save_anonymization_params(data["question_id"], data, request) return True, data["question_id"] except DatabaseError as e: log.error("Error creating the question. The question is very long") @@ -328,6 +339,16 @@ def userQuestionDetailsById(userOwner, questionId, request, language="default"): data["num_options"] = len(options) data["question_options"] = options + if data["question_sensitive"]: + params = ( + request.dbsession.query( + AnonymizationParameter.name, AnonymizationParameter.value + ) + .filter(AnonymizationParameter.question_id == data["question_id"]) + .all() + ) + data.update(params) + return data @@ -514,3 +535,44 @@ def knowIfUserHasCreatedTranslations(request, userId): return True return False + + +def get_sensitive_questions_anonymity_by_project_id(project_id, request): + """ + Retrieve all sensitive questions of a project by its id. Includes the registry and all the assessments. + """ + query = ( + request.dbsession.query( + Question.question_id, + Question.question_dtype, + Question.question_code, + Question.question_anonymity, + ) + .join(Registry, Registry.question_id == Question.question_id) + .filter(Registry.project_id == project_id) + .filter(Question.question_sensitive == 1) + .union( + request.dbsession.query( + Question.question_id, + Question.question_dtype, + Question.question_code, + Question.question_anonymity, + ) + .join(AssDetail, AssDetail.question_id == Question.question_id) + .filter(AssDetail.project_id == project_id) + .filter(Question.question_sensitive == 1) + ) + ) + return query.all() + + +def get_registry_key_question(request): + return ( + request.dbsession.query(Question).filter(Question.question_regkey == 1).first() + ) + + +def get_assessment_key_question(request): + return ( + request.dbsession.query(Question).filter(Question.question_asskey == 1).first() + ) diff --git a/climmob/processes/db/registry.py b/climmob/processes/db/registry.py index a55f69ec..78fbc37e 100644 --- a/climmob/processes/db/registry.py +++ b/climmob/processes/db/registry.py @@ -4,8 +4,9 @@ from sqlalchemy import func from climmob.models import Regsection, Registry, Project, Question, userProject +from climmob.models.repository import execute_two_sqls, sql_execute from climmob.models.schema import mapFromSchema, mapToSchema -from climmob.processes import addRegistryQuestionsToProject +from climmob.processes.db.project import addRegistryQuestionsToProject from climmob.processes.db.assessment import setAssessmentStatus, formattingQuestions import climmob.plugins as p @@ -38,6 +39,8 @@ "getTheGroupOfThePackageCode", "registryHaveQuestionOfMultimediaType", "deleteRegistryByProjectId", + "delete_registry_data_by_qst162", + "get_registry_data_by_qst162", ] @@ -71,7 +74,7 @@ def setRegistryStatus(userOwner, projectCod, projectId, status, request): try: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "data", "reg"] + *[userOwner, projectCod, "data", "reg"], ) shutil.rmtree(path) except: @@ -584,3 +587,22 @@ def registryHaveQuestionOfMultimediaType(request, projectId): return True else: return False + + +def get_registry_data_by_qst162(schema, qst162, columns): + query = ( + f"SELECT {','.join(columns)} FROM " + + schema + + ".REG_geninfo WHERE qst162='" + + qst162 + + "'" + ) + return sql_execute(query).fetchone() + + +def delete_registry_data_by_qst162(schema, qst162, odk_user): + query = "DELETE FROM " + schema + ".REG_geninfo WHERE qst162='" + qst162 + "'" + execute_two_sqls( + "SET @odktools_current_user = '" + odk_user + "'; ", + query, + ) diff --git a/climmob/processes/db/results.py b/climmob/processes/db/results.py index 846faf8b..2e7c2226 100644 --- a/climmob/processes/db/results.py +++ b/climmob/processes/db/results.py @@ -6,10 +6,18 @@ from climmob.models import Assessment, Question, Project, mapFromSchema from climmob.models.repository import sql_fetch_all, sql_fetch_one -from climmob.processes import getCombinations +from climmob.processes.db.project_combinations import getCombinations +from climmob.processes.db.anonymization_params import get_anonymization_params_as_dict +from climmob.processes.db.question import ( + get_sensitive_questions_anonymity_by_project_id, + get_registry_key_question, + get_assessment_key_question, +) __all__ = ["getJSONResult", "getCombinationsData"] +from climmob.utility import get_question_by_field_name, QuestionAnonymity + def getMiltiSelectLookUpTable(XMLFile, multiSelectTable): tree = etree.parse(XMLFile) @@ -55,11 +63,12 @@ def getFields(XMLFile, table): return fields -def getLookups(XMLFile, userOwner, projectCod, request): +def getLookups(XMLFile, userOwner, projectCod, anonymize, request): lktables = [] tree = etree.parse(XMLFile) root = tree.getroot() elkptables = root.find(".//lkptables") + qst_163_pseudonym = get_anonymization_params_as_dict(163, request)["pseudonym"] if elkptables is not None: etables = elkptables.findall(".//table") for table in etables: @@ -94,12 +103,16 @@ def getLookups(XMLFile, userOwner, projectCod, request): avalue = {} for field in atable["fields"]: avalue[field["name"]] = value[field["name"]] + if anonymize and atable["name"].endswith("lkpqst163_opts"): + avalue["qst163_opts_des"] = qst_163_pseudonym.replace( + "{}", str(avalue["qst163_opts_cod"]) + ) atable["values"].append(avalue) lktables.append(atable) return lktables -def getPackageData(userOwner, projectId, projectCod, request): +def getPackageData(userOwner, projectId, projectCod, request, anonymize=False): data = ( request.dbsession.query(Question).filter(Question.question_regkey == 1).first() ) @@ -107,6 +120,7 @@ def getPackageData(userOwner, projectId, projectCod, request): data = ( request.dbsession.query(Question).filter(Question.question_fname == 1).first() ) + farmer_name_qst_id = data.question_id qstFarmer = data.question_code sql = ( @@ -209,6 +223,12 @@ def getPackageData(userOwner, projectId, projectCod, request): ) pkgdetails = sql_fetch_all(sql) + + farmer_name_pseudonym = "" + if anonymize: + params = get_anonymization_params_as_dict(farmer_name_qst_id, request) + farmer_name_pseudonym = params["pseudonym"] + packages = [] pkgcode = -999 for pkg in pkgdetails: @@ -216,7 +236,12 @@ def getPackageData(userOwner, projectId, projectCod, request): aPackage = {} pkgcode = pkg.package_id aPackage["package_id"] = pkg.package_id - aPackage["farmername"] = pkg["farmername"] + if anonymize: + aPackage["farmername"] = farmer_name_pseudonym.replace( + "{}", str(pkg.package_id) + ) + else: + aPackage["farmername"] = pkg["farmername"] aPackage["comps"] = [] for x in range(0, ncombs): aPackage["comps"].append({}) @@ -264,44 +289,104 @@ def getPackageData(userOwner, projectId, projectCod, request): return packages -def getData(userOwner, projectCod, registry, assessments, request): - data = ( - request.dbsession.query(Question).filter(Question.question_regkey == 1).first() - ) - registryKey = data.question_code - data = ( - request.dbsession.query(Question).filter(Question.question_asskey == 1).first() - ) - assessmentKey = data.question_code +class QuestionSelectFieldBuilder: + def __init__(self, anonymize): + self.column = None + self.table = None + self.form_id = None + self.prefix = None + self.sensitive = False + self.anonymize = anonymize + + def set_column(self, column): + self.column = column + + def set_table(self, table): + self.table = table + + def set_form_id(self, form_id): + self.form_id = form_id + + def set_prefix(self, prefix): + self.prefix = prefix + + def set_sensitive(self, sensitive): + self.sensitive = sensitive + + def build(self): + if not self.anonymize or not self.sensitive: + return f"{self.table}.{self.column} AS {self.prefix}_{self.column}" + + query = ( + f"COALESCE(MAX(" + f"CASE WHEN da.col_name = '{self.column}' " + f"AND da.form_id='{self.form_id}'" + f"THEN da.value END)," + f"{self.table}.{self.column}) " + f"AS {self.prefix}_{self.column}" + ) + return query + + +def getData( + userOwner, project_id, projectCod, registry, assessments, request, anonymize=False +): + registryKey = get_registry_key_question(request).question_code + + assessmentKey = get_assessment_key_question(request).question_code + + questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) fields = [] + + reg_alias = "reg" + + select_field_builder = QuestionSelectFieldBuilder(anonymize) + select_field_builder.set_table(reg_alias) + select_field_builder.set_prefix("REG") + select_field_builder.set_form_id("-") + for field in registry["fields"]: - fields.append( - userOwner - + "_" - + projectCod - + ".REG_geninfo." - + field["name"] - + " AS " - + "REG_" - + field["name"] - ) + select_field_builder.set_column(field["name"]) + if anonymize: + question = get_question_by_field_name(field["name"], questions) + if ( + question + and question.question_anonymity == QuestionAnonymity.REMOVE.value + ): + continue + select_field_builder.set_sensitive(question is not None) + fields.append(select_field_builder.build()) + for assessment in assessments: + assessment_alias = "assess_" + assessment["code"] + select_field_builder.set_table(assessment_alias) + select_field_builder.set_prefix(f'ASS{assessment["code"]}') + select_field_builder.set_form_id(f"{assessment['code']}") for field in assessment["fields"]: - fields.append( - userOwner - + "_" - + projectCod - + ".ASS" - + assessment["code"] - + "_geninfo." - + field["name"] - + " AS " - + "ASS" - + assessment["code"] - + "_" - + field["name"] - ) + select_field_builder.set_column(field["name"]) + if anonymize: + question = get_question_by_field_name(field["name"], questions) + if ( + question + and question.question_anonymity == QuestionAnonymity.REMOVE.value + ): + continue + select_field_builder.set_sensitive(question is not None) + fields.append(select_field_builder.build()) + + if anonymize: + to_remove_keys = ["instancename", "deviceimei", "cal_qst163", "clc_after"] + tmp_fields = fields.copy() + fields = [] + for field in tmp_fields: + append = True + for key in to_remove_keys: + if key in field: + append = False + break + if append: + fields.append(field) sql = ( "SELECT " @@ -311,8 +396,11 @@ def getData(userOwner, projectCod, registry, assessments, request): + "_" + projectCod + ".REG_geninfo " + + reg_alias ) + for assessment in assessments: + assessment_alias = "assess_" + assessment["code"] sql = ( sql + " LEFT JOIN " @@ -321,34 +409,29 @@ def getData(userOwner, projectCod, registry, assessments, request): + projectCod + ".ASS" + assessment["code"] - + "_geninfo ON " + + "_geninfo " + + assessment_alias + + " ON " ) sql = ( sql - + userOwner - + "_" - + projectCod - + ".REG_geninfo." + + reg_alias + + "." + registryKey + " = " - + userOwner - + "_" - + projectCod - + ".ASS" - + assessment["code"] - + "_geninfo." + + assessment_alias + + "." + assessmentKey ) - sql = ( - sql - + " ORDER BY cast(" - + userOwner - + "_" - + projectCod - + ".REG_geninfo." - + registryKey - + " AS unsigned)" - ) + if anonymize: + sql = ( + sql + + " LEFT JOIN " + + f"{userOwner}_{projectCod}.anonymized da " + + f"ON da.reg_id = {reg_alias}.qst162 " + + f"GROUP BY {reg_alias}.qst162" + ) + sql = sql + f" ORDER BY cast({reg_alias}.{registryKey} AS unsigned)" data = sql_fetch_all(sql) @@ -544,6 +627,7 @@ def getJSONResult( includeRegistry=True, includeAssessment=True, assessmentCode="", + anonymize=False, ): data = {} res = ( @@ -569,12 +653,12 @@ def getJSONResult( if includeRegistry: registryXML = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "db", "reg", "create.xml"] + *[userOwner, projectCod, "db", "reg", "create.xml"], ) if os.path.exists(registryXML): data["registry"] = { "lkptables": getLookups( - registryXML, userOwner, projectCod, request + registryXML, userOwner, projectCod, anonymize, request ), "fields": getFields(registryXML, "REG_geninfo"), } @@ -605,7 +689,7 @@ def getJSONResult( "ass", assessment.ass_cod, "create.xml", - ] + ], ) if os.path.exists(assessmentXML): data["assessments"].append( @@ -617,6 +701,7 @@ def getJSONResult( assessmentXML, userOwner, projectCod, + anonymize, request, ), "fields": getFields( @@ -629,7 +714,9 @@ def getJSONResult( if res.project_registration_and_analysis == 1: haveAssessments = True # Get the package information but only for registered farmers - data["packages"] = getPackageData(userOwner, projectId, projectCod, request) + data["packages"] = getPackageData( + userOwner, projectId, projectCod, request, anonymize + ) data["combination"] = getCombinationsData(projectId, request) if haveAssessments: @@ -638,10 +725,12 @@ def getJSONResult( ) data["data"] = getData( userOwner, + projectId, projectCod, data["registry"], data["assessments"], request, + anonymize=anonymize, ) data["importantfields"] = getImportantFields(projectId, request) @@ -649,10 +738,12 @@ def getJSONResult( data["specialfields"] = [] data["data"] = getData( userOwner, + projectId, projectCod, data["registry"], data["assessments"], request, + anonymize=anonymize, ) data["importantfields"] = [] diff --git a/climmob/processes/db/userproject.py b/climmob/processes/db/userproject.py index 111e464c..2a5f2c6a 100644 --- a/climmob/processes/db/userproject.py +++ b/climmob/processes/db/userproject.py @@ -1,6 +1,8 @@ from climmob.models import userProject, mapFromSchema -__all__ = ["getAllProjectsByUser"] +__all__ = ["getAllProjectsByUser", "get_owner_user_name_by_project_id"] + +from climmob.utility.project import ProjectAccessType def getAllProjectsByUser(user, request): @@ -10,3 +12,13 @@ def getAllProjectsByUser(user, request): .first() ) return mappedData + + +def get_owner_user_name_by_project_id(project_id, request): + mappedData = mapFromSchema( + request.dbsession.query(userProject.user_name) + .filter(userProject.project_id == project_id) + .filter(userProject.access_type == ProjectAccessType.OWNER.value) + .first() + ) + return mappedData["user_name"] diff --git a/climmob/processes/odk/api.py b/climmob/processes/odk/api.py index 6f32e70a..e6aa08fb 100644 --- a/climmob/processes/odk/api.py +++ b/climmob/processes/odk/api.py @@ -16,15 +16,13 @@ from pyramid.response import FileResponse from climmob.models import Project, storageErrors, Assessment -from climmob.processes import ( - isRegistryOpen, - isAssessmentOpen, - assessmentExists, - projectExists, - packageExist, - getTheProjectIdForOwner, -) + +from climmob.processes.db.registry import isRegistryOpen, packageExist +from climmob.processes.db.assessment import isAssessmentOpen, assessmentExists +from climmob.processes.db.validators import projectExists, getTheProjectIdForOwner + from climmob.processes.db.json import addJsonLog +from climmob.processes.db.anonymized import anonymize_questions log = logging.getLogger(__name__) @@ -103,7 +101,7 @@ def getFormList(userid, enumerator, request, userOwner=None, projectCod=None): if project.project_regstatus == 1: path = os.path.join( request.registry.settings["user.repository"], - *[project.user_name, project.project_cod, "odk", "reg", "*.json"] + *[project.user_name, project.project_cod, "odk", "reg", "*.json"], ) files = glob.glob(path) if files: @@ -140,7 +138,7 @@ def getFormList(userid, enumerator, request, userOwner=None, projectCod=None): "ass", assessment.ass_cod, "*.json", - ] + ], ) files = glob.glob(path) if files: @@ -172,7 +170,7 @@ def getManifest(user, userOwner, projectId, projectCod, request): if prjdat.project_regstatus == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "reg", "media", "*.*"] + *[userOwner, projectCod, "odk", "reg", "media", "*.*"], ) files = glob.glob(path) @@ -210,7 +208,7 @@ def getAssessmentManifest( if prjdat.ass_status == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "ass", assessmentid, "media", "*.*"] + *[userOwner, projectCod, "odk", "ass", assessmentid, "media", "*.*"], ) else: raise HTTPNotFound() @@ -246,7 +244,7 @@ def getXMLForm(userOwner, projectId, projectCod, request): if prjdat.project_regstatus == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "reg", "*.xml"] + *[userOwner, projectCod, "odk", "reg", "*.xml"], ) files = glob.glob(path) @@ -270,7 +268,7 @@ def getAssessmentXMLForm(userOwner, projectId, projectCod, assessmentid, request if prjdat.ass_status == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "ass", assessmentid, "*.xml"] + *[userOwner, projectCod, "odk", "ass", assessmentid, "*.xml"], ) else: raise HTTPNotFound() @@ -292,7 +290,7 @@ def getMediaFile(userOwner, projectId, projectCod, fileid, request): if prjdat.project_regstatus == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "reg", "media", fileid] + *[userOwner, projectCod, "odk", "reg", "media", fileid], ) else: raise HTTPNotFound() @@ -318,7 +316,7 @@ def getAssessmentMediaFile( if prjdat.ass_status == 1: path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "ass", assessmentid, "media", fileid] + *[userOwner, projectCod, "odk", "ass", assessmentid, "media", fileid], ) else: raise HTTPNotFound() @@ -388,20 +386,21 @@ def storeJSONInMySQL( projectId, ): schema = userOwner + "_" + projectCod + if type == "REG": manifestFile = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "db", "reg", "manifest.xml"] + *[userOwner, projectCod, "db", "reg", "manifest.xml"], ) jsFile = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "db", "reg", "custom.js"] + *[userOwner, projectCod, "db", "reg", "custom.js"], ) else: manifestFile = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "db", "ass", assessmentid, "manifest.xml"] + *[userOwner, projectCod, "db", "ass", assessmentid, "manifest.xml"], ) jsFile = "" @@ -462,7 +461,18 @@ def storeJSONInMySQL( projectId, ) - return True + with open(JSONFile, "r", encoding="utf-8") as f: + data = json.load(f) + form_id = "-" + if type == "ASS": + form_id = assessmentid + success, msg = anonymize_questions( + request, data, form_id, projectId, userOwner, projectCod + ) + if not success: + return False, msg + + return True, "" def convertXMLToJSON( @@ -480,12 +490,12 @@ def convertXMLToJSON( if submissionType == "REG": path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "reg", "*.xml"] + *[userOwner, projectCod, "odk", "reg", "*.xml"], ) if submissionType == "ASS": path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "odk", "ass", assessmentID, "*.xml"] + *[userOwner, projectCod, "odk", "ass", assessmentID, "*.xml"], ) files = glob.glob(path) @@ -623,7 +633,7 @@ def storeSubmission(userid, userEnum, request): pathTemp = os.path.join( request.registry.settings["user.repository"], - *[userid, "data", "xml", str(iniqueIDTemp)] + *[userid, "data", "xml", str(iniqueIDTemp)], ) os.makedirs(pathTemp) @@ -664,7 +674,7 @@ def storeSubmission(userid, userEnum, request): dirs = glob.glob( os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "data", "reg", "xml"] + *[userOwner, projectCod, "data", "reg", "xml"], ) + "/*", recursive=True, @@ -673,7 +683,7 @@ def storeSubmission(userid, userEnum, request): dirs = glob.glob( os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "data", "ass", assessmentID, "xml"] + *[userOwner, projectCod, "data", "ass", assessmentID, "xml"], ) + "/*", recursive=True, @@ -688,14 +698,14 @@ def storeSubmission(userid, userEnum, request): if submissionType == "REG": path = os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "data", "reg", "xml", str(iniqueID)] + *[userOwner, projectCod, "data", "reg", "xml", str(iniqueID)], ) if not os.path.exists(path): os.makedirs(path) os.makedirs( os.path.join( request.registry.settings["user.repository"], - *[userOwner, projectCod, "data", "reg", "json", str(iniqueID)] + *[userOwner, projectCod, "data", "reg", "json", str(iniqueID)], ) ) @@ -710,7 +720,7 @@ def storeSubmission(userid, userEnum, request): assessmentID, "xml", str(iniqueID), - ] + ], ) if not os.path.exists(path): os.makedirs(path) @@ -725,7 +735,7 @@ def storeSubmission(userid, userEnum, request): assessmentID, "json", str(iniqueID), - ] + ], ) ) diff --git a/climmob/processes/odk/generator.py b/climmob/processes/odk/generator.py index 49de1ac2..960f1a70 100644 --- a/climmob/processes/odk/generator.py +++ b/climmob/processes/odk/generator.py @@ -40,87 +40,110 @@ ] +def execute_command(args, error_msg): + error = False + try: + check_call(args) + except CalledProcessError as e: + msg = f"{error_msg}\nError:\n{str(e)}\n" + log.error(msg) + print(msg) + error = True + return error + + +def create_schema(schema, cnf_file): + print(f"****buildDatabase**Dropping schema {schema}******") + args = [ + "mysql", + f"--defaults-file={cnf_file}", + f"--execute=DROP SCHEMA IF EXISTS {schema}", + ] + error = execute_command(args, "Error dropping schema") + if error: + return error + + print(f"****buildDatabase**Creating new schema {schema}******") + args = [ + "mysql", + f"--defaults-file={cnf_file}", + f"--execute=CREATE SCHEMA {schema}" + " DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci", + ] + error = execute_command(args, "Error creating schema") + return error + + +def create_anonymized_table(schema, cnf_file): + args = [ + "mysql", + f"--defaults-file={cnf_file}", + f"--execute=CREATE TABLE IF NOT EXISTS {schema}.anonymized " + "(`form_id` varchar(255) NOT NULL," + "`reg_id` varchar(255) NOT NULL," + "`col_name` varchar(255) NOT NULL," + "`value` varchar(255) DEFAULT NULL," + "PRIMARY KEY (`form_id`,`reg_id`,`col_name`)" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;", + ] + + error = execute_command(args, "Error creating anonymized table") + return error + + def buildDatabase( cnfFile, createFile, insertFile, schema, dropSchema, settings, outputDir ): error = False if dropSchema: - print("****buildDatabase**Dropping schema******") - args = [] - args.append("mysql") - args.append("--defaults-file=" + cnfFile) - args.append("--execute=DROP SCHEMA IF EXISTS " + schema) - try: - check_call(args) - except CalledProcessError as e: - msg = "Error dropping schema \n" + error = create_schema(schema, cnfFile) + if error: + return error + + print(f"****buildDatabase**Creating tables {schema}******") + args = ["mysql", f"--defaults-file={cnfFile}", schema] + with open(createFile) as input_file: + proc = Popen(args, stdin=input_file, stderr=PIPE, stdout=PIPE) + output, error = proc.communicate() + # if output != "" or error != "": + if proc.returncode != 0: + # print("3") + msg = "Error creating database \n" + msg = msg + "File: " + createFile + "\n" msg = msg + "Error: \n" - msg = msg + str(e) + "\n" + msg = msg + str(error) + "\n" + msg = msg + "Output: \n" + msg = msg + str(output) + "\n" log.error(msg) - print(msg) error = True + if error: + return error - if not error: - print("****buildDatabase**Creating new schema******") - args = [] - args.append("mysql") - args.append("--defaults-file=" + cnfFile) - args.append( - "--execute=CREATE SCHEMA " - + schema - + " DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci" - ) - try: - check_call(args) - except CalledProcessError as e: - msg = "Error dropping schema \n" - msg = msg + "Error: \n" - msg = msg + str(e) + "\n" - log.error(msg) - error = True - - if not error: - print("****buildDatabase**Creating tables******") - args = [] - args.append("mysql") - args.append("--defaults-file=" + cnfFile) - args.append(schema) - - with open(createFile) as input_file: - proc = Popen(args, stdin=input_file, stderr=PIPE, stdout=PIPE) - output, error = proc.communicate() - # if output != "" or error != "": - if proc.returncode != 0: - # print("3") - msg = "Error creating database \n" - msg = msg + "File: " + createFile + "\n" - msg = msg + "Error: \n" - msg = msg + str(error) + "\n" - msg = msg + "Output: \n" - msg = msg + str(output) + "\n" - log.error(msg) - error = True - - if not error: - print("****buildDatabase**Inserting into lookup tables******") - with open(insertFile) as input_file: - proc = Popen(args, stdin=input_file, stderr=PIPE, stdout=PIPE) - output, error = proc.communicate() - # if output != "" or error != "": - if proc.returncode != 0: - msg = "Error loading lookup tables \n" - msg = msg + "File: " + createFile + "\n" - msg = msg + "Error: \n" - msg = msg + str(error) + "\n" - msg = msg + "Output: \n" - msg = msg + str(output) + "\n" - log.error(msg) - error = True + if dropSchema: + error = create_anonymized_table(schema, cnfFile) + if error: + return error + + print(f"****buildDatabase**Inserting into lookup tables for {schema}******") + with open(insertFile) as input_file: + proc = Popen(args, stdin=input_file, stderr=PIPE, stdout=PIPE) + output, error = proc.communicate() + # if output != "" or error != "": + if proc.returncode != 0: + msg = "Error loading lookup tables \n" + msg = msg + "File: " + createFile + "\n" + msg = msg + "Error: \n" + msg = msg + str(error) + "\n" + msg = msg + "Output: \n" + msg = msg + str(output) + "\n" + log.error(msg) + error = True + if error: + return error - if not error: - print("****buildDatabase**Creating triggers******") - functionForCreateTheTriggers(schema, settings, outputDir, cnfFile) + print(f"****buildDatabase**Creating triggers for {schema}******") + functionForCreateTheTriggers(schema, settings, outputDir, cnfFile) return error diff --git a/climmob/products/analysisdata/analysisdata.py b/climmob/products/analysisdata/analysisdata.py index 9dc66067..5ef460fd 100644 --- a/climmob/products/analysisdata/analysisdata.py +++ b/climmob/products/analysisdata/analysisdata.py @@ -3,51 +3,79 @@ registryHaveQuestionOfMultimediaType, assessmentHaveQuestionOfMultimediaType, ) -from climmob.products.analysisdata.celerytasks import create_CSV +from climmob.products.analysisdata.celerytasks import create_raw_data_file from climmob.products.climmob_products import ( createProductDirectory, registerProductInstance, ) -def create_datacsv(userOwner, projectId, projectCod, info, request, form, code): +def create_raw_data( + user_owner, + project_id, + project_cod, + info, + request, + form, + code, + file_type="csv", + anonymized=False, +): # We create the plugin directory if it does not exists and return it - # The path user.repository in development.ini/user/project/products/product and - # user.repository in development.ini/user/project/products/product/outputs - path = createProductDirectory(request, userOwner, projectCod, "datacsv") + extra = "-anonymized" if anonymized else "" + + name_output = form + f"_data{extra}" + if code != "": + name_output += "_" + code + + name_output += "_" + project_cod + + path = createProductDirectory( + request, user_owner, project_cod, f"data{file_type}{extra}" + ) # We call the Celery task that will generate the output packages.pdf - task = create_CSV.apply_async((path, info, projectCod, form, code), queue="ClimMob") + task = create_raw_data_file.apply_async( + (path, info, name_output, file_type), queue="ClimMob" + ) # We register the instance of the output with the task ID of celery # This will go to the products table that then you can monitor and use # in the nice product interface # u.registerProductInstance(user, project, 'cards', 'cards.pdf', task.id, request) - nameOutput = form + "_data" - if code != "": - nameOutput += "_" + code + + mimetypes = { + "csv": "text/csv", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + } + mimetype = mimetypes.get(file_type) + + process_name = ( + f"create_data{extra}_" + f"{'xlsx_' if file_type == 'xlsx' else ''}" + form + "_" + code + ) registerProductInstance( - projectId, - "datacsv", - nameOutput + "_" + projectCod + ".csv", - "text/csv", - "create_data_" + form + "_" + code, + project_id, + f"data{file_type}{extra}", + name_output + f".{file_type}", + mimetype, + process_name, task.id, request, ) for plugin in p.PluginImplementations(p.IMultimedia): - thereAreMultimedia = False + there_are_multimedia = False if form == "Registration": - thereAreMultimedia = registryHaveQuestionOfMultimediaType( - request, projectId + there_are_multimedia = registryHaveQuestionOfMultimediaType( + request, project_id ) if form == "Assessment": - thereAreMultimedia = assessmentHaveQuestionOfMultimediaType( - request, projectId, code + there_are_multimedia = assessmentHaveQuestionOfMultimediaType( + request, project_id, code ) - if thereAreMultimedia: + if there_are_multimedia: plugin.start_multimedia_download( - request, userOwner, projectId, projectCod, form, code + request, user_owner, project_id, project_cod, form, code ) diff --git a/climmob/products/analysisdata/celerytasks.py b/climmob/products/analysisdata/celerytasks.py index 3d7a45d9..dcdd80e6 100644 --- a/climmob/products/analysisdata/celerytasks.py +++ b/climmob/products/analysisdata/celerytasks.py @@ -1,44 +1,73 @@ -import json import os -import shutil as sh + +import pandas as pd from climmob.config.celery_app import celeryApp from climmob.plugins.utilities import climmobCeleryTask -from climmob.products.analysisdata.exportToCsv import createCSV @celeryApp.task(base=climmobCeleryTask) -def create_CSV(path, info, projectCod, form, code): - - # if os.path.exists(path): - # sh.rmtree(path) - - nameOutput = form + "_data" - if code != "": - nameOutput += "_" + code +def create_raw_data_file(path, info, name_output, file_type): - pathout = os.path.join(path, "outputs") + path_out = os.path.join(path, "outputs") if not os.path.exists(path): os.makedirs(path) - os.makedirs(pathout) + os.makedirs(path_out) + + replace_options_with_labels(info) + + df = pd.DataFrame(info["data"]) + if file_type == "xlsx": + df.to_excel(os.path.join(path_out, name_output) + f".{file_type}", index=False) + elif file_type == "csv": + df.to_csv(os.path.join(path_out, name_output) + f".{file_type}", index=False) + - if os.path.exists(pathout + "/" + nameOutput + "_" + projectCod + ".csv"): - os.remove(pathout + "/" + nameOutput + "_" + projectCod + ".csv") +def replace_options_with_labels(data): + for row in data["data"]: + for field in data["registry"]["fields"]: + if field["rtable"] is not None and row["REG_" + field["name"]] is not None: + result = get_option_label( + data["registry"]["lkptables"], + field["rtable"], + field["rfield"], + row["REG_" + field["name"]], + field["isMultiSelect"], + ) + row["REG_" + field["name"]] = result - pathInputFiles = os.path.join(path, "inputFile") - os.makedirs(pathInputFiles) + for assessment in data["assessments"]: + for field in assessment["fields"]: + if ( + field["rtable"] is not None + and row["ASS" + assessment["code"] + "_" + field["name"]] + is not None + ): + result = get_option_label( + assessment["lkptables"], + field["rtable"], + field["rfield"], + row["ASS" + assessment["code"] + "_" + field["name"]], + field["isMultiSelect"], + ) + row["ASS" + assessment["code"] + "_" + field["name"]] = result - with open(pathInputFiles + "/info.json", "w") as outfile: - jsonString = json.dumps(info, indent=4, ensure_ascii=False) - outfile.write(jsonString) - if os.path.exists(pathInputFiles + "/info.json"): - try: - createCSV( - pathout + "/" + nameOutput + "_" + projectCod + ".csv", - pathInputFiles + "/info.json", - ) - except Exception as e: - print("We can't create the CSV." + str(e)) +def get_option_label(lkptables, rtable, rfield, value, isMultiSelect): + res = None + for lkp in lkptables: + if lkp["name"] == rtable: + for data in lkp["values"]: + if isMultiSelect == "true": + for valueSplit in value.split(" "): + if str(data[rfield]) == str(valueSplit): + if res == None: + res = data[rfield[:-3] + "des"] + else: + res += " - " + data[rfield[:-3] + "des"] + else: + if data[rfield] == value: + res = data[rfield[:-3] + "des"] + break - sh.rmtree(pathInputFiles) + return res diff --git a/climmob/products/analysisdata/exportToCsv.py b/climmob/products/analysisdata/exportToCsv.py deleted file mode 100644 index af7a9d81..00000000 --- a/climmob/products/analysisdata/exportToCsv.py +++ /dev/null @@ -1,89 +0,0 @@ -import csv -import json - - -def getRealData(lkptables, rtable, rfield, value, isMultiSelect): - res = None - for lkp in lkptables: - if lkp["name"] == rtable: - for data in lkp["values"]: - if isMultiSelect == "true": - for valueSplit in value.split(" "): - if str(data[rfield]) == str(valueSplit): - if res == None: - res = data[rfield[:-3] + "des"] - else: - res += " - " + data[rfield[:-3] + "des"] - else: - if data[rfield] == value: - res = data[rfield[:-3] + "des"] - break - - return res - - -def createCSV(outputPath, inputFile): - myFile = open(outputPath, "w") - with myFile: - writer = csv.writer(myFile) - - with open(inputFile) as json_file: - data = json.load(json_file) - # SACO LAS COLUMNAS DEL REGISTRY - columns = [] - for field in data["registry"]["fields"]: - columns.append(field["desc"].replace(",", "")) - - # SACO LAS COLUMNAS DE LOS ASSESSMENTS - for assessment in data["assessments"]: - for field in assessment["fields"]: - columns.append(field["desc"].replace(",", "")) - - # ESCRIBO LAS COLUMNAS - writer.writerow(columns) - - # EMPIEZO A SACAR LOS DATOS - for row in data["data"]: - fieldsDataRow = [] - # DATOS DEL REGISTRO - for field in data["registry"]["fields"]: - # print(field) - if field["rtable"] != None and row["REG_" + field["name"]] != None: - result = getRealData( - data["registry"]["lkptables"], - field["rtable"], - field["rfield"], - row["REG_" + field["name"]], - field["isMultiSelect"], - ) - fieldsDataRow.append(str(result).replace(",", "")) - else: - fieldsDataRow.append( - str(row["REG_" + field["name"]]).replace(",", "") - ) - - for assessment in data["assessments"]: - for field in assessment["fields"]: - if ( - field["rtable"] != None - and row["ASS" + assessment["code"] + "_" + field["name"]] - != None - ): - result = getRealData( - assessment["lkptables"], - field["rtable"], - field["rfield"], - row["ASS" + assessment["code"] + "_" + field["name"]], - field["isMultiSelect"], - ) - fieldsDataRow.append(str(result).replace(",", "")) - else: - fieldsDataRow.append( - str( - row[ - "ASS" + assessment["code"] + "_" + field["name"] - ] - ).replace(",", "") - ) - - writer.writerow(fieldsDataRow) diff --git a/climmob/products/climmob_products.py b/climmob/products/climmob_products.py index 1940b9f6..52c04126 100644 --- a/climmob/products/climmob_products.py +++ b/climmob/products/climmob_products.py @@ -228,6 +228,18 @@ def register_products(config): ) products.append(datacsv) + datacsv_anonymized = addProduct( + "datacsv-anonymized", "Information collected in the project anonymized." + ) + addMetadataToProduct(datacsv_anonymized, "author", "Johann Ávalos") + addMetadataToProduct(datacsv_anonymized, "version", "1.0") + addMetadataToProduct( + datacsv_anonymized, + "Licence", + "Copyright 2025, MrBot Software Solutions", + ) + products.append(datacsv_anonymized) + # FORM documentform = addProduct( "documentform", "Create a document pdf to collect information." @@ -362,6 +374,19 @@ def register_products(config): ) products.append(dataxlsx) + dataxlsx_anonymized = addProduct( + "dataxlsx-anonymized", + "Information collected in the project anonymized in XLSX format.", + ) + addMetadataToProduct(dataxlsx_anonymized, "author", "Johann Ávalos") + addMetadataToProduct(dataxlsx_anonymized, "version", "1.0") + addMetadataToProduct( + dataxlsx_anonymized, + "Licence", + "Copyright 2025, MrBot Software Solutions", + ) + products.append(dataxlsx_anonymized) + # INPUT FILES datajson = addProduct( "datajson", "data.json file used as input for report generation." diff --git a/climmob/products/dataxlsx/__init__.py b/climmob/products/dataxlsx/__init__.py deleted file mode 100644 index 95ed645a..00000000 --- a/climmob/products/dataxlsx/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from climmob.products.dataxlsx.celerytasks import * diff --git a/climmob/products/dataxlsx/celerytasks.py b/climmob/products/dataxlsx/celerytasks.py deleted file mode 100644 index 17909ba2..00000000 --- a/climmob/products/dataxlsx/celerytasks.py +++ /dev/null @@ -1,187 +0,0 @@ -import os -import shutil as sh -import uuid -from climmob.config.celery_app import celeryApp -from climmob.plugins.utilities import climmobCeleryTask -from subprocess import Popen, PIPE -from climmob.models import get_engine -import pandas as pd -import multiprocessing - - -@celeryApp.task(base=climmobCeleryTask) -def create_XLSX( - settings, - path, - userOwner, - projectCod, - projectId, - form, - code, - finalName, - sensitive=False, -): - - num_workers = ( - multiprocessing.cpu_count() - int(settings.get("server:threads", "1")) - 1 - ) - - if num_workers <= 0: - num_workers = 1 - - pathout = os.path.join(path, "outputs") - if not os.path.exists(pathout): - os.makedirs(pathout) - - pathOfTheUser = os.path.join(settings["user.repository"], *[userOwner, projectCod]) - - UuidForTempDirectory = str(uuid.uuid4()) - - pathtmp = os.path.join(path, "tmp_" + UuidForTempDirectory) - if not os.path.exists(pathtmp): - os.makedirs(pathtmp) - - pathtmpout = os.path.join(path, "tmp_out_" + UuidForTempDirectory) - if not os.path.exists(pathtmpout): - os.makedirs(pathtmpout) - - listOfRequiredInformation = [] - listOfGeneratedXLSX = [] - - listOfRequiredInformation.append({"form": "Registration", "code": ""}) - - if form == "Report": - try: - engine = get_engine(settings) - sql = "SELECT * FROM assessment WHERE project_id='{}' AND ass_status > 0 ORDER BY ass_days".format( - projectId - ) - listOfAssessments = engine.execute(sql).fetchall() - - for assessment in listOfAssessments: - listOfRequiredInformation.append( - {"form": "Assessment", "code": assessment[0]} - ) - engine.dispose() - except Exception as e: - print("Error in the query for get the assessments") - else: - if form == "Assessment": - listOfRequiredInformation.append({"form": form, "code": code}) - - for requiredInformation in listOfRequiredInformation: - - nameOutput = requiredInformation["form"] + "_data" - if requiredInformation["code"] != "": - nameOutput += "_" + requiredInformation["code"] - - xlsx_file = os.path.join(pathtmpout, *[nameOutput + "_" + projectCod + ".xlsx"]) - - if requiredInformation["code"] != "": - pathOfTheForm = os.path.join( - pathOfTheUser, *["db", "ass", requiredInformation["code"]] - ) - mainTable = "ASS" + requiredInformation["code"] + "_geninfo" - else: - pathOfTheForm = os.path.join(pathOfTheUser, *["db", "reg"]) - mainTable = "REG_geninfo" - - create_xml = os.path.join(pathOfTheForm, *["create.xml"]) - - mysql_user = settings["odktools.mysql.user"] - mysql_password = settings["odktools.mysql.password"] - mysql_host = settings["odktools.mysql.host"] - mysql_port = settings["odktools.mysql.port"] - odk_tools_dir = settings["odktools.path"] - - paths = ["utilities", "MySQLToXLSX", "mysqltoxlsx"] - mysql_to_xlsx = os.path.join(odk_tools_dir, *paths) - - args = [ - mysql_to_xlsx, - "-H " + mysql_host, - "-P " + mysql_port, - "-u " + mysql_user, - "-p '{}'".format(mysql_password), - "-s " + userOwner + "_" + projectCod, - "-x " + create_xml, - "-o " + xlsx_file, - "-T " + pathtmp, - "-w {}".format(num_workers), - "-r {}".format(3), - ] - if sensitive: - args.append("-c") - - commandToExec = " ".join(map(str, args)) - - # p = Popen(args, stdout=PIPE, stderr=PIPE, shell=True) - p = Popen(commandToExec, stdout=PIPE, stderr=PIPE, shell=True) - stdout, stderr = p.communicate() - if p.returncode == 0: - # os.system("mv " + xlsx_file + " " + pathout) - listOfGeneratedXLSX.append(xlsx_file) - else: - print( - "MySQLToXLSX Error: " - + stderr.decode() - + "-" - + stdout.decode() - + ". Args: " - + " ".join(args) - ) - error = stdout.decode() + stderr.decode() - if error.find("Worksheet name is already in use") >= 0: - print( - "A worksheet name has been repeated. Excel only allow 30 characters in the worksheet name. " - "You can fix this by editing the dictionary and change the description of the tables " - "to a maximum of " - "30 characters." - ) - else: - print( - "Unknown error while creating the XLSX. Sorry about this. " - "Please report this error as an issue on https://github.com/mrbotcr/py3climmob" - ) - - if len(listOfGeneratedXLSX) > 0: - mergeXLSXForms(listOfGeneratedXLSX, pathout, finalName) - - sh.rmtree(pathtmpout) - if os.path.exists(pathtmp): - sh.rmtree(pathtmp) - - -def mergeXLSXForms(listOfGeneratedXLSX, pathout, finalName): - - merged_inner = pd.DataFrame() - - for index, XLSX in enumerate(listOfGeneratedXLSX): - - if index == 0: - registryDF = pd.read_excel( - XLSX, - sheet_name=0, - ) - registryDF = registryDF.add_suffix("_reg") - - merged_inner = registryDF - else: - - assessmentDF = pd.read_excel( - XLSX, - sheet_name=0, - ) - assessmentDF = assessmentDF.add_suffix("_ass_" + str(index)) - if not assessmentDF.empty: - merged_inner = pd.merge( - left=merged_inner, - right=assessmentDF, - left_on="qst162_reg", - right_on="qst163_ass_" + str(index), - how="left", - ) - - merged_inner.to_excel(finalName, sheet_name="Sheet1", index=False) - - os.system("mv " + finalName + " " + pathout) diff --git a/climmob/products/dataxlsx/dataxlsx.py b/climmob/products/dataxlsx/dataxlsx.py deleted file mode 100644 index c05ffe7f..00000000 --- a/climmob/products/dataxlsx/dataxlsx.py +++ /dev/null @@ -1,71 +0,0 @@ -import climmob.plugins as p -from climmob.processes import ( - registryHaveQuestionOfMultimediaType, - assessmentHaveQuestionOfMultimediaType, -) -from climmob.products.dataxlsx.celerytasks import create_XLSX -from climmob.products.climmob_products import ( - createProductDirectory, - registerProductInstance, -) - - -def create_XLSXToDownload(userOwner, projectId, projectCod, request, form, code): - # We create the plugin directory if it does not exists and return it - # The path user.repository in development.ini/user/project/products/product and - # user.repository in development.ini/user/project/products/product/outputs - settings = {} - for key, value in request.registry.settings.items(): - if isinstance(value, str): - settings[key] = value - - path = createProductDirectory(request, userOwner, projectCod, "dataxlsx") - # We call the Celery task that will generate the output packages.pdf - nameOutput = form + "_data" - if code != "": - nameOutput += "_" + code - - task = create_XLSX.apply_async( - ( - settings, - path, - userOwner, - projectCod, - projectId, - form, - code, - nameOutput + "_" + projectCod + ".xlsx", - ), - queue="ClimMob", - ) - # We register the instance of the output with the task ID of celery - # This will go to the products table that then you can monitor and use - # in the nice product interface - # u.registerProductInstance(user, project, 'cards', 'cards.pdf', task.id, request) - - registerProductInstance( - projectId, - "dataxlsx", - nameOutput + "_" + projectCod + ".xlsx", - "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "create_data_xlsx_" + form + "_" + code, - task.id, - request, - ) - - for plugin in p.PluginImplementations(p.IMultimedia): - thereAreMultimedia = False - if form == "Registration": - thereAreMultimedia = registryHaveQuestionOfMultimediaType( - request, projectId - ) - - if form == "Assessment": - thereAreMultimedia = assessmentHaveQuestionOfMultimediaType( - request, projectId, code - ) - - if thereAreMultimedia: - plugin.start_multimedia_download( - request, userOwner, projectId, projectCod, form, code - ) diff --git a/climmob/products/errorLogDocument/celerytasks.py b/climmob/products/errorLogDocument/celerytasks.py index bc701b28..6ed0927d 100644 --- a/climmob/products/errorLogDocument/celerytasks.py +++ b/climmob/products/errorLogDocument/celerytasks.py @@ -6,7 +6,7 @@ from climmob.config.celery_app import celeryApp from climmob.config.celery_class import celeryTask -from climmob.products.analysisdata.exportToCsv import getRealData +from climmob.products.analysisdata.celerytasks import get_option_label @celeryApp.task(base=celeryTask, soft_time_limit=7200, time_limit=7200) @@ -134,7 +134,7 @@ def createErrorLogDocument( and row[firstPart + field["name"]] != None and field["name"] != "qst163" ): - result = getRealData( + result = get_option_label( lkps, field["rtable"], field["rfield"], diff --git a/climmob/scripts/anonymize_project.py b/climmob/scripts/anonymize_project.py new file mode 100644 index 00000000..058bae49 --- /dev/null +++ b/climmob/scripts/anonymize_project.py @@ -0,0 +1,59 @@ +import sys + +from pyramid.paster import get_appsettings, setup_logging +from climmob.models import ( + get_engine, + Base, + get_tm_session, + get_session_factory, + initialize_schema, +) +import transaction +import requests +import argparse +import pyramid +import os + +from climmob.processes import anonymize_project + + +def main(raw_args=None): + parser = argparse.ArgumentParser() + parser.add_argument("ini_path", help="Path to ini file") + parser.add_argument("project_id", help="Project id") + args = parser.parse_args(raw_args) + + if not os.path.exists(os.path.abspath(args.ini_path)): + print("Ini file does not exists") + sys.exit(1) + + settings = get_appsettings(args.ini_path, "climmob") + + engine = get_engine( + settings, + ) + + Base.metadata.create_all(engine) + session_factory = get_session_factory(engine) + + with transaction.manager: + + dbsession = get_tm_session(session_factory, transaction.manager) + setup_logging(args.ini_path) + + request = requests.Session() + request.dbsession = dbsession + request.registry = pyramid.registry.Registry + request.registry.settings = settings + request.locale_name = "en" + + initialize_schema() + + success, msg = anonymize_project(args.project_id, request) + + if success: + print(f"Successfully anonymized project {args.project_id}") + else: + print(f"Failed to anonymized project {args.project_id}") + + engine.dispose() diff --git a/climmob/templates/dashboard/dashboard.jinja2 b/climmob/templates/dashboard/dashboard.jinja2 index d70caf03..248319e1 100755 --- a/climmob/templates/dashboard/dashboard.jinja2 +++ b/climmob/templates/dashboard/dashboard.jinja2 @@ -41,6 +41,8 @@ {% include "snippets/maze_loader.jinja2" %} +{% from 'snippets/dashboard/downloads.jinja2' import downloads with context %} + {% if request.registry.settings.get('session.show_expired_modal', "false") == 'true' %} {% include 'snippets/expired_session.jinja2' %} {% endif %} @@ -565,8 +567,6 @@ {% if progress.regtotal > 0 %} {{ _("View and edit data") }} - {{ _("Download data in .CSV format") }} - {{ _("Download data in .XLSX format") }} {% block dataprivacy_download_data_registry %} @@ -581,6 +581,8 @@ + + {{ downloads("downloadDataRegistry", "registry") }} {% endif %} @@ -702,8 +704,6 @@ {% if assessment.asstotal > 0 %} {{ _("View and edit data") }} - {{ _("Download data in .CSV format") }} - {{ _("Download data in .XLSX format") }} {% block dataprivacy_download_data_assessment scoped%} @@ -722,6 +722,7 @@ + {{ downloads("downloadDataAssessment", "assessment", assessment.ass_cod) }}
diff --git a/climmob/templates/question/library.jinja2 b/climmob/templates/question/library.jinja2 index 5bc5394e..7dce1049 100644 --- a/climmob/templates/question/library.jinja2 +++ b/climmob/templates/question/library.jinja2 @@ -211,8 +211,31 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} - var elem_ckb_question_sensitive = document.querySelector('#ckb_question_as_sensitive'); - var ckb_question_sensitive = new Switchery(elem_ckb_question_sensitive, { color: '#1AB394' }); + let elem_ckb_question_sensitive = document.querySelector('#ckb_question_as_sensitive'); + let ckb_question_sensitive = new Switchery(elem_ckb_question_sensitive, { color: '#1AB394' }); + + let anonymity_select = document.getElementById("question_anonymity"); + + let anonymity_container = $("#question-anonymity-div") + + elem_ckb_question_sensitive.addEventListener('change', function() { + anonymity_select.value = saved_anonymity_id + if (anonymity_select.value === "") { + anonymity_select.selectedIndex = 0; + } + $(anonymity_select).trigger('change.select2'); + setAnonymityInputsDisplay(elem_ckb_question_sensitive.checked, parseInt(anonymity_select.value)) + }); + + $(anonymity_select).on('change', function() { + setAnonymityInputsDisplay(elem_ckb_question_sensitive.checked, parseInt(anonymity_select.value)) + }); + + const anonymity_inputs_selector = $('#question-anonymity-div input') + + anonymity_inputs_selector.on('change', function() { + this.setCustomValidity(""); + }); {% endif %} @@ -240,6 +263,67 @@ $("#question_max").val(""); } + const q_types = {{ question_types | tojson }}; + + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + + let saved_anonymity_id = null; + + function update_anonymity_types(value, current_anonymity_id=-1) { + let option; + anonymity_select.innerHTML = ""; + let anonymity_index = 0; + for (const q_type of q_types) { + if (value === q_type.id) { + q_type.anonymity_opts.forEach((anonymity, j) => { + option = new Option(anonymity.name, anonymity.id); + anonymity_select.appendChild(option); + if (anonymity.id === current_anonymity_id) { + anonymity_index = j; + } + }); + break; + } + } + anonymity_select.selectedIndex = anonymity_index; + $(anonymity_select).trigger('change.select2'); + setAnonymityInputsDisplay(elem_ckb_question_sensitive.checked, parseInt(anonymity_select.value)) + } + + /** + * @param {boolean} show + * @param {int} anonymity + */ + function setAnonymityInputsDisplay(show, anonymity=-1) { + const display = show ? 'block' : 'none' + anonymity_container.css('display', display); + $('.anonymity-description').css('display', 'none'); + anonymity_inputs_selector.prop('disabled', true); + $('#question-anonymization-params > *').css('display', 'none'); + if (anonymity === {{ QuestionAnonymity.PSEUDONYM }}) { + $('#pseudonym-params').css('display', 'block'); + $('#pseudonym-params input').prop('disabled', false); + $('#anonymity-description-pseudo').css('display', 'block'); + } + else if (anonymity === {{ QuestionAnonymity.RANGE }}) { + $('#range-params').css('display', 'block'); + $('#range-params input').prop('disabled', false); + $('#anonymity-description-range').css('display', 'block'); + } + else if (anonymity === {{ QuestionAnonymity.REMOVE }}) { + $('#anonymity-description-remove').css('display', 'block'); + } + else if (anonymity === {{ QuestionAnonymity.NOISE }}) { + $('#anonymity-description-noise').css('display', 'block'); + } + } + + function clean_anonymization_section() { + anonymity_inputs_selector.prop('value', ""); + } + + {% endif %} + $(document).ready(function() { tour = new Tour({ @@ -345,6 +429,10 @@ $("#question_dtype").select2(); var type = $('#question_dtype').val(); + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + update_anonymity_types(parseInt(type)); + {% endif %} + var question_requiredvalue = 0; var objQRequired = $("#ckb_required_value"); if (objQRequired.is(':checked')) @@ -370,7 +458,12 @@ { clean_extra_fields(); - var value = $('#question_dtype').val(); + const value = $('#question_dtype').val(); + + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + const current_anonymity_id = $(anonymity_select).val(); + update_anonymity_types(parseInt(value), parseInt(saved_anonymity_id)); + {% endif %} $("#question"+value).css("display",'initial'); @@ -565,7 +658,12 @@ function showDeleteQuestion() { var urlAction = '{{ request.application_url }}/question/'+$('#question_id').val()+'/delete' - showDelete(urlAction,'{{ _("Do you really want to remove this question ?") }}','{{ request.session.get_csrf_token() }}', {% if nextPage %}"{{ request.route_url("qlibrary", user_name=activeUser.login, _query={'next': nextPage} ) }}" {% else %} "{{ request.route_url("qlibrary", user_name=activeUser.login ) }}" {% endif %}) + showDelete( + urlAction, + '{{ _("Do you really want to remove this question ?") }}', + '{{ request.session.get_csrf_token() }}', + "{% if nextPage %}{{ request.route_url("qlibrary", user_name=activeUser.login, _query={'next': nextPage} ) }}{% else %}{{ request.route_url("qlibrary", user_name=activeUser.login ) }}{% endif %}" + ) $(this).parent().parent().remove(); } @@ -603,6 +701,54 @@ setTimeout(() => checkIsVisibleAndHide(selector), 100); } + /** + * @param {Object} anonymity_body + * @param {int} q_type_id + * @returns {boolean} True if the string is valid, false otherwise. + */ + function validate_anonymization_params(anonymity_body, q_type_id) { + let invalid_input = null; + if (anonymity_body["question_anonymity"] === {{ QuestionAnonymity.PSEUDONYM }}){ + const regex = /^[^{}]*{}[^{}]*$/; + if (!regex.test(anonymity_body["anonym_param_pseudonym"])) { + invalid_input = {name: "anonym_param_pseudonym", + msg: "{{ _("Musts contain '{}' once.") | safe }}" } + } + } + else if (anonymity_body["question_anonymity"] === {{ QuestionAnonymity.RANGE }}) { + const lower_bound = Number(anonymity_body["anonym_param_lower_bound"]) + const upper_bound = Number(anonymity_body["anonym_param_upper_bound"]) + const interval = Number(anonymity_body["anonym_param_interval"]) + + if (q_type_id === {{ QuestionType.INTEGER }}) { + if (!Number.isInteger(lower_bound)) + invalid_input = {name: "anonym_param_lower_bound"} + else if (!Number.isInteger(upper_bound)) + invalid_input = {name: "anonym_param_upper_bound"} + else if (!Number.isInteger(interval)) + invalid_input = {name: "anonym_param_interval"} + if (invalid_input) + invalid_input.msg = "{{ _("Must be integer") }}" + } + if (lower_bound >= upper_bound) { + invalid_input = {name: "anonym_param_upper_bound", + msg: "{{ _("Must be higher than the lower bound.") }}" } + } else if (interval <= 0) { + invalid_input = {name: "anonym_param_interval", + msg: "{{ _("Must be greater than zero.") }}" } + } else if (interval >= upper_bound - lower_bound) { + invalid_input = {name: "anonym_param_interval", + msg: "{{ _("Must fit within the bounds.") }}" } + } + } + if (invalid_input) { + const input = document.querySelector(`input[name="${invalid_input.name}"]`); + input.setCustomValidity(invalid_input.msg); + input.reportValidity(); + } + return invalid_input === null; + } + function actionQuestion(action) { $("#btn_add_question").prop('disabled', true); $("#btn_update_question").prop('disabled', true); @@ -631,9 +777,32 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} - var question_sensitive = 0; - if ($("#ckb_question_as_sensitive").is(':checked')) + let question_sensitive = 0; + let anonymity_body = {}; + if ($("#ckb_question_as_sensitive").is(':checked')) { question_sensitive = 1; + let anonymity_id = parseInt(anonymity_select.value); + anonymity_body["question_anonymity"] = anonymity_id + let params_container_id = null + if (anonymity_id === {{ QuestionAnonymity.PSEUDONYM }}) + params_container_id = "pseudonym-params"; + else if (anonymity_id === {{ QuestionAnonymity.RANGE }}) + params_container_id = "range-params"; + + $(`#${params_container_id} input`).each(function() { + const name = $(this).attr('name'); + const value = $(this).val(); + if (name) + anonymity_body[name] = value; + }); + + if (!validate_anonymization_params(anonymity_body, parseInt($('#question_dtype').val()))) { + $("#btn_add_question").prop('disabled', false); + $("#btn_update_question").prop('disabled', false); + checkIsVisibleAndHide("#carga") + return + } + } {% endif %} @@ -688,6 +857,7 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} "question_sensitive": question_sensitive, + ...anonymity_body {% endif %} @@ -904,13 +1074,17 @@ hideNumericalInputs() + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + update_anonymity_types(q_types[0].id) + {% endif %} + loadLanguages(); $(".inputForQuestion").val("") $("#question_group").val(category); $("#question_forms").val("3"); $('#question_group').trigger("change.select2"); - $("#question_dtype").val("1") + $("#question_dtype").val(q_types[0].id) $('#question_dtype').trigger('change.select2'); $("#question1").css("display",'initial'); @@ -938,6 +1112,7 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} setSwitchery(ckb_question_sensitive, false); + setAnonymityInputsDisplay(false) {% endif %} @@ -971,6 +1146,9 @@ clean_extra_fields(); clean_extra_section(); clean_buttoms_question(); + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + clean_anonymization_section(); + {% endif %} loadLanguages(user_name); @@ -1048,11 +1226,19 @@ {% endif %} {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + const isSensitive = dataJson["question_sensitive"] === 1; - if (dataJson["question_sensitive"] == 1) - setSwitchery(ckb_question_sensitive, true) - else - setSwitchery(ckb_question_sensitive, false) + setSwitchery(ckb_question_sensitive, isSensitive) + saved_anonymity_id = dataJson["question_anonymity"]; + update_anonymity_types(dataJson["question_dtype"], saved_anonymity_id) + + const regex = /anonym_param_([a-z_]+)/; + anonymity_inputs_selector.each(function(index, element) { + const match = regex.exec($(element).attr('name')); + if (match) { + $(element).prop("value", dataJson[match[1]]) + } + }); {% endif %} diff --git a/climmob/templates/snippets/dashboard/downloads.jinja2 b/climmob/templates/snippets/dashboard/downloads.jinja2 new file mode 100644 index 00000000..b82065ba --- /dev/null +++ b/climmob/templates/snippets/dashboard/downloads.jinja2 @@ -0,0 +1,130 @@ +{% macro downloads(route, form_type, ass_id='') %} + + +
+
+

{{ _("Downloads") }}

+ + + +
+
+ + + + + + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + + {% endif %} + + + + + + + + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + + {% endif %} + + + + + + {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %} + + {% endif %} + + +
{{ _('File Type') }}{{ _('Raw') }}{{ _('Anonymized') }}
+ CSV + + + + + + + + +
+ XLSX + + + + + + + + +
+
+
+{% endmacro %} diff --git a/climmob/templates/snippets/project/productsList/productsList.jinja2 b/climmob/templates/snippets/project/productsList/productsList.jinja2 index 4037e63d..7f440443 100644 --- a/climmob/templates/snippets/project/productsList/productsList.jinja2 +++ b/climmob/templates/snippets/project/productsList/productsList.jinja2 @@ -29,11 +29,22 @@ {% set productToFocus = "xx" %} - {% set changes = {'projectSummary': true, 'DataCollectionProgress': false, 'QRPackagesEditable':false,'projectDataCollectedCSV':false,'projectDataCollectedXLSX': false, 'productToFocus':""} %} + {% set changes = { + 'projectSummary': true, + 'DataCollectionProgress': false, + 'QRPackagesEditable':false, + 'projectDataCollectedCSV':false, + 'projectDataCollectedXLSX': false, + 'projectDataCollectedCSV-anonymized':false, + 'projectDataCollectedXLSX-anonymized': false, + 'productToFocus':"" + } %} {% if activeProject.project_regstatus > 0 %} {% if changes.update({'DataCollectionProgress': true }) %} {% endif %} {% if changes.update({'projectDataCollectedCSV': true }) %} {% endif %} {% if changes.update({'projectDataCollectedXLSX': true }) %} {% endif %} + {% if changes.update({'projectDataCollectedCSV-anonymized': true }) %} {% endif %} + {% if changes.update({'projectDataCollectedXLSX-anonymized': true }) %} {% endif %} {% endif %} {% block qr_packages_editable_variable scoped%} @@ -54,6 +65,27 @@ {% if product.product_id == "qrpackage" %} {{ _("List of packages with QR for the participant registration form") }} + + {% elif product.product_id == "datacsv-anonymized" %} + {% if product.process_name == "create_data-anonymized_Report_" %} + {% if changes.update({'projectDataCollectedCSV-anonymized': false }) %}{% endif %} + {{ _("Information collected in all the project in .CSV format (anonymized)") }} + {% elif product.process_name == "create_data-anonymized_Registration_" %} + {{ _("Information collected in the participant registration form in .CSV format (anonymized)") }} + {% else %} + {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .CSV format (anonymized)") }} + {% endif %} + + {% elif product.product_id == "dataxlsx-anonymized" %} + {% if product.process_name == "create_data-anonymized_xlsx_Report_" %} + {% if changes.update({'projectDataCollectedXLSX-anonymized': false }) %}{% endif %} + {{ _("Information collected in all the project in .XLSX format (anonymized)") }} + {% elif product.process_name == "create_data-anonymized_xlsx_Registration_" %} + {{ _("Information collected in the participant registration form in .XLSX format (anonymized)") }} + {% else %} + {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .XLSX format (anonymized)") }} + {% endif %} + {% else %} {% if product.product_id == "packages" %} {{ _("List with randomized trial packages") }} @@ -65,19 +97,15 @@ {{ _("Color cards to explain ClimMob") }} {% else %} {% if product.product_id == "datacsv" %} - {% if product.process_name == "create_data_Report_" %} {% if changes.update({'projectDataCollectedCSV': false }) %}{% endif %} {{ _("Information collected in all the project in .CSV format") }} + {% elif product.process_name == "create_data_Registration_" %} + {{ _("Information collected in the participant registration form in .CSV format") }} {% else %} - {% if product.process_name == "create_data_Registration_" %} - {{ _("Information collected in the participant registration form in .CSV format") }} - {% else %} - {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .CSV format") }} - {% endif %} + {{ _("Information collected in the trial data collection moment form") }}: {{ product.extraInformation.ass_desc }} {{ _("in .CSV format") }} {% endif %} - {% else %} {% if product.product_id == "reports" %} {{ _("Analysis report based on trial data") }} @@ -270,10 +298,18 @@ {{ _("Update") }} {% endif %} + {% if product.product_id == "datacsv-anonymized" and product.process_name == "create_data-anonymized_Report_" %} + {{ _("Update") }} + {% endif %} + {% if product.product_id == "dataxlsx" and product.process_name == "create_data_xlsx_Report_" %} {{ _("Update") }} {% endif %} + {% if product.product_id == "dataxlsx-anonymized" and product.process_name == "create_data-anonymized_xlsx_Report_" %} + {{ _("Update") }} + {% endif %} + {% if product.product_id == "generalreport" %} {{ _("Update") }} {% endif %} @@ -324,6 +360,31 @@ {% endif %} + {% if project_is_anonymized and request.registry.settings.get("module.dataprivacy", "false") != "false"%} + {% if changes["projectDataCollectedXLSX-anonymized"] %} + + {{ _("Information collected in all the project in .XLSX format (anonymized)") }} + + + + {{ _("Create") }} + + + + {% endif %} + + {% if changes["projectDataCollectedCSV-anonymized"] %} + + {{ _("Information collected in all the project in .CSV format (anonymized)") }} + + + + {{ _("Create") }} + + + + {% endif %} + {% endif %} {% if changes.projectSummary %} {{ _("Project summary") }} diff --git a/climmob/templates/snippets/question/question-form.jinja2 b/climmob/templates/snippets/question/question-form.jinja2 index 18e3d885..667cfc09 100644 --- a/climmob/templates/snippets/question/question-form.jinja2 +++ b/climmob/templates/snippets/question/question-form.jinja2 @@ -29,27 +29,9 @@
@@ -553,13 +535,88 @@ {% if request.registry.settings.get("module.dataprivacy", "false") != "false" %}
- +
+ data-off-text="{{ _('No') }}"> +
+
+
+
+
+ +
+ +{# REMOVE #} + +{# PSEUDO #} + +{# BINNING #} + +{# NOISE #} + +
+
+
+
+
+ +
+ + +
+
+
+ +
+ + + {{ _("E.g., 0 (lower), 100 (upper), and 10 (interval) create groups of 10.") }} + +
+
+
+
+
+ +
+ + + {{ _("Enter the prefix for the anonymous ID. You must include {} as a placeholder, + which will be replaced by a unique number. Example: Farmer-{} becomes Farmer-1") }} + +
+
@@ -569,4 +626,4 @@
{% block qstform_extra %} -{% endblock qstform_extra %} \ No newline at end of file +{% endblock qstform_extra %} diff --git a/climmob/tests/test_utils/common.py b/climmob/tests/test_utils/common.py index 9c05cc19..ce73dacc 100644 --- a/climmob/tests/test_utils/common.py +++ b/climmob/tests/test_utils/common.py @@ -33,15 +33,17 @@ def get_mock(self, name): class ViewBaseTest(BaseTest): view_class = None request_method = "GET" + body = {} request_body = None def setUp(self): super().setUp() self.request = MagicMock() self.request.translate = self.mock_translation - with patch("climmob.views.classes.ApiContext"), patch( + with patch("climmob.views.classes.ApiContext") as api_context, patch( "climmob.views.classes.PrivateContext" ): + api_context.return_value.body = self.body self.view = self.view_class(self.request) self.view.request.method = self.request_method self.view.user = MagicMock(login="test_user") diff --git a/climmob/tests/test_utils/test_utility_question.py b/climmob/tests/test_utils/test_utility_question.py index c88533f7..1b88f438 100644 --- a/climmob/tests/test_utils/test_utility_question.py +++ b/climmob/tests/test_utils/test_utility_question.py @@ -1,7 +1,7 @@ import unittest from unittest.mock import MagicMock -from climmob.utility import is_type_numerical, QuestionType +from climmob.utility import is_type_numerical, QuestionType, get_question_by_field_name class TestIsTypeNumerical(unittest.TestCase): @@ -39,3 +39,186 @@ def test_false(self): result = is_type_numerical(q_type) self.assertFalse(result) + + +class TestGetQuestionByFieldName(unittest.TestCase): + def setUp(self): + self.question_codes = ["qst_a_test", "qst_b_test"] + self.questions = [ + MagicMock(question_code=self.question_codes[0]), + MagicMock(question_code=self.question_codes[1]), + ] + + def test_simple(self): + field_name = self.question_codes[1] + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_a(self): + field_name = self.question_codes[1] + "_a" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_b(self): + field_name = self.question_codes[1] + "_b" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_c(self): + field_name = self.question_codes[1] + "_c" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_a_oth(self): + field_name = self.question_codes[1] + "_a_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_b_oth(self): + field_name = self.question_codes[1] + "_b_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_c_oth(self): + field_name = self.question_codes[1] + "_c_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_oth(self): + field_name = self.question_codes[1] + "_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_perf_1(self): + field_name = "perf_" + self.question_codes[1] + "_1" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_perf_2(self): + field_name = "perf_" + self.question_codes[1] + "_2" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_perf_3(self): + field_name = "perf_" + self.question_codes[1] + "_3" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_char_pos(self): + field_name = "char_" + self.question_codes[1] + "_pos" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_with_char_neg(self): + field_name = "char_" + self.question_codes[1] + "_neg" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, self.questions[1]) + + def test_unknown_suffix(self): + field_name = self.question_codes[1] + "_unknown_suffix" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_prefix(self): + field_name = "unknown_prefix_" + self.question_codes[1] + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_prefix_and_suffix(self): + field_name = "unknown_prefix_" + self.question_codes[1] + "_unknown_suffix" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_simple(self): + field_name = "unknown_question_code" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_a(self): + field_name = "unknown_question_code" + "_a" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_b(self): + field_name = "unknown_question_code" + "_b" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_c(self): + field_name = "unknown_question_code" + "_c" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_a_oth(self): + field_name = "unknown_question_code" + "_a_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_b_oth(self): + field_name = "unknown_question_code" + "_b_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_c_oth(self): + field_name = "unknown_question_code" + "_c_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_oth(self): + field_name = "unknown_question_code" + "_oth" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_perf_1(self): + field_name = "perf_" + "unknown_question_code" + "_1" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_perf_2(self): + field_name = "perf_" + "unknown_question_code" + "_2" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_perf_3(self): + field_name = "perf_" + "unknown_question_code" + "_3" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_char_pos(self): + field_name = "char_" + "unknown_question_code" + "_pos" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) + + def test_unknown_question_with_char_neg(self): + field_name = "char_" + "unknown_question_code" + "_neg" + result = get_question_by_field_name(field_name, self.questions) + + self.assertEqual(result, None) diff --git a/climmob/tests/test_utils/test_views_api_project_analysis.py b/climmob/tests/test_utils/test_views_api_project_analysis.py index 33c27ba0..1cd8cb07 100644 --- a/climmob/tests/test_utils/test_views_api_project_analysis.py +++ b/climmob/tests/test_utils/test_views_api_project_analysis.py @@ -2,79 +2,46 @@ import unittest from unittest.mock import patch, MagicMock +from climmob.tests.test_utils.common import ViewBaseTest from climmob.views.Api.project_analysis import ( ReadDataOfProjectViewApi, ReadVariablesForAnalysisViewApi, GenerateAnalysisByApiViewApi, ) +from climmob.views.validators.ProjectExistsValidator import ProjectExistsValidator +from climmob.views.validators.project import HasAccessToProjectValidator -class TestReadDataOfProjectViewAPI(unittest.TestCase): - def setUp(self): - self.view = ReadDataOfProjectViewApi(MagicMock()) - self.view.request.method = "GET" - self.view.user = MagicMock(login="test_user") - self.view.body = json.dumps({"project_cod": "123", "user_owner": "owner"}) +class TestReadDataOfProjectViewAPI(ViewBaseTest): + view_class = ReadDataOfProjectViewApi + body = {"project_cod": "123", "user_owner": "owner"} + request_body = json.dumps(body) - def mock_translation(self, message): - return message + def test_has_validators(self): + self.assertEqual( + self.view.validators, (ProjectExistsValidator, HasAccessToProjectValidator) + ) @patch( "climmob.views.Api.project_analysis.getJSONResult", return_value={"data": "some_data"}, ) - @patch("climmob.views.Api.project_analysis.getTheProjectIdForOwner", return_value=1) - @patch("climmob.views.Api.project_analysis.projectExists", return_value=True) - def test_process_view_success( - self, mock_projectExists, mock_getTheProjectIdForOwner, mock_getJSONResult - ): + def test_get_success(self, mock_get_json_result): self.view._ = self.mock_translation # Mock translation function - response = self.view.processView() + response = self.view.get() self.assertEqual(response.status_code, 200) self.assertIn("some_data", response.body.decode()) - @patch("climmob.views.Api.project_analysis.projectExists", return_value=False) - def test_process_view_project_not_exist(self, mock_projectExists): - self.view._ = self.mock_translation # Mock translation function - - response = self.view.processView() - - self.assertEqual(response.status_code, 401) - self.assertIn("This project does not exist.", response.body.decode()) - - def test_process_view_invalid_json(self): - self.view._ = self.mock_translation # Mock translation function - self.view.body = '{"wrong_key": "value"}' - - response = self.view.processView() - - self.assertEqual(response.status_code, 401) - self.assertIn("Error in the JSON.", response.body.decode()) - - @patch("json.loads", side_effect=json.JSONDecodeError("Expecting value", "", 0)) - def test_process_view_invalid_body(self, mock_json_loads): - self.view._ = self.mock_translation # Mock translation function - self.view.body = "" - - response = self.view.processView() - - self.assertEqual(response.status_code, 401) - self.assertIn( - "Error in the JSON, It does not have the 'body' parameter.", - response.body.decode(), + mock_get_json_result.assert_called_once_with( + self.body["user_owner"], + self.view.context.active_project_id, + self.body["project_cod"], + self.view.request, + anonymize=True, ) - def test_process_view_post_method(self): - self.view._ = self.mock_translation # Mock translation function - self.view.request.method = "POST" - - response = self.view.processView() - - self.assertEqual(response.status_code, 401) - self.assertIn("Only accepts GET method.", response.body.decode()) - class TestReadVariablesForAnalysisViewAPI(unittest.TestCase): def setUp(self): diff --git a/climmob/tests/test_utils/test_views_api_project_assessment_start.py b/climmob/tests/test_utils/test_views_api_project_assessment_start.py index 2b12ea2b..a116c09c 100644 --- a/climmob/tests/test_utils/test_views_api_project_assessment_start.py +++ b/climmob/tests/test_utils/test_views_api_project_assessment_start.py @@ -1516,7 +1516,10 @@ def test_api_registration_error_repeated_column(self): @patch("climmob.views.Api.projectAssessmentStart.open") @patch("climmob.views.Api.projectAssessmentStart.uuid.uuid1") @patch("climmob.views.Api.projectAssessmentStart.os.path.join") - @patch("climmob.views.Api.projectAssessmentStart.storeJSONInMySQL") + @patch( + "climmob.views.Api.projectAssessmentStart.storeJSONInMySQL", + return_value=(True, ""), + ) @patch( "climmob.views.Api.projectAssessmentStart.os.path.exists", return_value=False ) @@ -1579,7 +1582,10 @@ def test_api_registration_success( "climmob.views.Api.projectAssessmentStart.os.path.join", side_effect=os.path.join, ) - @patch("climmob.views.Api.projectAssessmentStart.storeJSONInMySQL") + @patch( + "climmob.views.Api.projectAssessmentStart.storeJSONInMySQL", + return_value=(True, ""), + ) def test_api_registration_data_could_not_be_saved( self, mock_storeJSONInMySQL, mock_os_path_join, mock_uuid1, mock_open ): diff --git a/climmob/tests/test_utils/test_views_api_project_registry_start.py b/climmob/tests/test_utils/test_views_api_project_registry_start.py index 489d5c11..07cfd600 100644 --- a/climmob/tests/test_utils/test_views_api_project_registry_start.py +++ b/climmob/tests/test_utils/test_views_api_project_registry_start.py @@ -3134,7 +3134,10 @@ def test_api_registration_no_package_code(self, mock_getProjectNumobs, mock_open ) @patch("climmob.views.Api.projectRegistryStart.open") - @patch("climmob.views.Api.projectRegistryStart.storeJSONInMySQL") + @patch( + "climmob.views.Api.projectRegistryStart.storeJSONInMySQL", + return_value=(True, ""), + ) @patch("climmob.views.Api.projectRegistryStart.getProjectNumobs", return_value=10) @patch("uuid.uuid1", return_value="12345678") def test_api_registration_reads_log_error( @@ -3195,7 +3198,10 @@ def test_api_registration_reads_log_error( mock_storeJSONInMySQL.assert_called() @patch("climmob.views.Api.projectRegistryStart.open") - @patch("climmob.views.Api.projectRegistryStart.storeJSONInMySQL") + @patch( + "climmob.views.Api.projectRegistryStart.storeJSONInMySQL", + return_value=(True, ""), + ) @patch("climmob.views.Api.projectRegistryStart.getProjectNumobs", return_value=10) def test_api_registration_successful( self, mock_getProjectNumobs, mock_storeJSONInMySQL, mock_open diff --git a/climmob/tests/test_utils/test_views_base_view.py b/climmob/tests/test_utils/test_views_base_view.py index 8a7f29fd..b87de949 100644 --- a/climmob/tests/test_utils/test_views_base_view.py +++ b/climmob/tests/test_utils/test_views_base_view.py @@ -803,8 +803,9 @@ def setUp(self): @classmethod def setUpClass(cls): cls.patchers["validate_register_form"] = { - "patch": patch( - "climmob.views.basic_views.validate_register_form", + "patch": patch.object( + RegisterView, + "validate_register_form", ), "return_value": (False, {}), } @@ -838,7 +839,7 @@ def tearDown(self): super().tearDown() if self.get_mock("validate_register_form").called: self.get_mock("validate_register_form").assert_called_once_with( - self.request.POST, self.view.request, self.view._ + self.request.POST ) if self.get_mock("add_user").called: self.get_mock("add_user").assert_called_once_with( diff --git a/climmob/tests/test_utils/test_views_registry.py b/climmob/tests/test_utils/test_views_registry.py index a0e99e38..30f045c8 100644 --- a/climmob/tests/test_utils/test_views_registry.py +++ b/climmob/tests/test_utils/test_views_registry.py @@ -275,6 +275,7 @@ def test_process_view_get( ) mock_getActiveProject.assert_called_once_with("test_user", self.mock_request) + @patch("climmob.views.registry.delete_anonymized_values_by_form_id") @patch( "climmob.views.registry.getActiveProject", return_value={"project": "active_project"}, @@ -294,6 +295,7 @@ def test_process_view_post_cancel_registry( mock_projectExists, mock_getTheProjectIdForOwner, mock_getActiveProject, + mock_delete_anonymized_values_by_form_id, ): self.mock_request.method = "POST" self.mock_request.params = {"cancelRegistry": "1"} diff --git a/climmob/tests/test_utils/test_views_validators.py b/climmob/tests/test_utils/test_views_validators.py index 57d34d8b..ec61d2cb 100644 --- a/climmob/tests/test_utils/test_views_validators.py +++ b/climmob/tests/test_utils/test_views_validators.py @@ -16,7 +16,10 @@ from climmob.views.validators.ProjectExistsValidator import ProjectExistsValidator from climmob.views.validators.assessment import AssessmentExistsValidator from climmob.views.validators.field.FieldValidator import FieldValidator -from climmob.views.validators.project import CanEditProjectValidator +from climmob.views.validators.project import ( + CanEditProjectValidator, + HasAccessToProjectValidator, +) from climmob.views.validators.question.QuestionMinMaxValidator import ( QuestionMinMaxValidator, @@ -139,6 +142,24 @@ def test_run_invalid(self): self.validator.run() +class TestHasAccessToProjectValidatorRun(unittest.TestCase): + def setUp(self): + self.request = MagicMock() + self.view = MagicMock() + self.view.request = self.request + + self.validator = HasAccessToProjectValidator(self.view) + + def test_run_valid(self): + self.view.context.access_type = MagicMock(int) + self.validator.run() + + def test_run_invalid(self): + self.view.context.access_type = None + with self.assertRaises(HTTPForbidden): + self.validator.run() + + class TestAssessmentExistsValidator(unittest.TestCase): def test_init_for_api(self): view = MagicMock(apiView) diff --git a/climmob/utility/__init__.py b/climmob/utility/__init__.py index e50db0db..62e489f4 100644 --- a/climmob/utility/__init__.py +++ b/climmob/utility/__init__.py @@ -1,4 +1,11 @@ from climmob.utility.helpers import * -from climmob.utility.validators import * from climmob.utility.factory import * from climmob.utility.question import * +from climmob.utility.anonymization import * + + +def get_enum_as_dict(enum): + result = {} + for member in enum: + result[member.name] = member.value + return result diff --git a/climmob/utility/anonymization.py b/climmob/utility/anonymization.py new file mode 100644 index 00000000..41b11cf1 --- /dev/null +++ b/climmob/utility/anonymization.py @@ -0,0 +1,46 @@ +import math +import random + + +def add_noise_to_gps_coordinates(lat, lon, radius): + """ + Add noise to a geographical coordinate by choosing a random point within a radius. + + Parameters: + lat (float): Latitude of the original coordinate. + lon (float): Longitude of the original coordinate. + radius (float): Radius in meters within which to choose a random point. + + Returns: + tuple: A tuple containing the new latitude and longitude. + """ + try: + # Earth radius in meters + earth_radius = 6378137 + + # Convert radius from meters to degrees latitude + radius_lat = radius / (earth_radius * (math.pi / 180)) + + # Convert radius from meters to degrees longitude, adjusted by latitude + radius_lon = radius / ( + earth_radius * (math.pi / 180) * math.cos(math.radians(lat)) + ) + + # Random angle in radians + angle = random.uniform(0, 2 * math.pi) + + # Random distance factor for uniform distribution in a circle + factor = math.sqrt(random.uniform(0, 1)) + + # Calculate deltas + delta_lat = factor * radius_lat * math.cos(angle) + delta_lon = factor * radius_lon * math.sin(angle) + + # New latitude and longitude + new_lat = lat + delta_lat + new_lon = lon + delta_lon + + return str(new_lat), str(new_lon) + except Exception as e: + print(e) + return "Error", "Error" diff --git a/climmob/utility/question.py b/climmob/utility/question.py index 7e538543..721dbc6c 100644 --- a/climmob/utility/question.py +++ b/climmob/utility/question.py @@ -1,18 +1,24 @@ -from enum import Enum +import re +from enum import Enum, IntEnum, auto -class QuestionType(Enum): +def _(x): + return x + + +class QuestionType(IntEnum): TEXT = 1 - RANKING_OF_OPTIONS = 9 - COMPARISON_WITH_CHECK = 10 - LOCATION = 27 DECIMAL = 2 INTEGER = 3 - GEOPOINT = 4 + GEO_POINT = 4 SELECT_ONE = 5 SELECT_MULTIPLE = 6 - GEOTRACE = 11 - GEOSHAPE = 12 + PACKAGE_CODE = 7 + FARMER = 8 + RANKING_OF_OPTIONS = 9 + COMPARISON_WITH_CHECK = 10 + GEO_TRACE = 11 + GEO_SHAPE = 12 DATE = 13 TIME = 14 DATETIME = 15 @@ -20,10 +26,138 @@ class QuestionType(Enum): AUDIO = 17 VIDEO = 18 BARCODE_QR = 19 + LOCATION = 27 + + +class QuestionTypeLabel(Enum): + TEXT = _("Text") + DECIMAL = _("Decimal") + INTEGER = _("Integer") + GEO_POINT = _("GeoPoint") + SELECT_ONE = _("Select one") + SELECT_MULTIPLE = _("Select multiple") + PACKAGE_CODE = _("Package code") + FARMER = _("Farmer") + RANKING_OF_OPTIONS = _("Ranking of options") + COMPARISON_WITH_CHECK = _("Comparison with check") + GEO_TRACE = _("GeoTrace") + GEO_SHAPE = _("GeoShape") + DATE = _("Date") + TIME = _("Time") + DATETIME = _("DateTime") + IMAGE = _("Image") + AUDIO = _("Audio") + VIDEO = _("Video") + BARCODE_QR = _("Barcode/QR") + LOCATION = _("Location") + + +class QuestionTypeOrder(IntEnum): + TEXT = auto() + RANKING_OF_OPTIONS = auto() + COMPARISON_WITH_CHECK = auto() + LOCATION = auto() + DECIMAL = auto() + INTEGER = auto() + GEO_POINT = auto() + SELECT_ONE = auto() + SELECT_MULTIPLE = auto() + GEO_TRACE = auto() + GEO_SHAPE = auto() + DATE = auto() + TIME = auto() + DATETIME = auto() + IMAGE = auto() + AUDIO = auto() + VIDEO = auto() + BARCODE_QR = auto() + PACKAGE_CODE = -1 # Not included as an option + FARMER = -1 # Not included as an option def is_type_numerical(q_type) -> bool: - return ( - int(q_type) == QuestionType.DECIMAL.value - or int(q_type) == QuestionType.INTEGER.value - ) + return int(q_type) == QuestionType.DECIMAL or int(q_type) == QuestionType.INTEGER + + +class QuestionAnonymity(IntEnum): + REMOVE = 1 + PSEUDONYM = 2 + RANGE = 3 + NOISE = 4 + MASK = 5 + MONTH_YEAR = 6 + + +class QuestionAnonymityLabel(Enum): + REMOVE = _("Remove") + PSEUDONYM = _("Pseudonym") + RANGE = _("Binning") + NOISE = _("Noise") + MASK = _("Mask") + MONTH_YEAR = _("Month-Year") + + +QA = QuestionAnonymity + + +class QuestionTypeAnonymity(Enum): + TEXT = [QA.REMOVE, QA.PSEUDONYM] + DECIMAL = [QA.REMOVE, QA.RANGE] + INTEGER = [QA.REMOVE, QA.RANGE] + GEO_POINT = [QA.REMOVE, QA.NOISE] + SELECT_ONE = [QA.REMOVE] + SELECT_MULTIPLE = [QA.REMOVE] + PACKAGE_CODE = [QA.REMOVE] + FARMER = [QA.REMOVE] + RANKING_OF_OPTIONS = [QA.REMOVE] + COMPARISON_WITH_CHECK = [QA.REMOVE] + GEO_TRACE = [QA.REMOVE] + GEO_SHAPE = [QA.REMOVE] + DATE = [QA.REMOVE, QA.MONTH_YEAR] + TIME = [QA.REMOVE] + DATETIME = [QA.REMOVE, QA.MONTH_YEAR] + IMAGE = [QA.REMOVE] + AUDIO = [QA.REMOVE] + VIDEO = [QA.REMOVE] + BARCODE_QR = [QA.REMOVE] + LOCATION = [QA.REMOVE] + + +def get_question_types_with_anonymity_labeled(request): + result = [] + for q_type in QuestionType: + order = QuestionTypeOrder[q_type.name].value + if order == -1: + continue + anonymity_opts = [] + for anonymity in QuestionTypeAnonymity[q_type.name].value: + anonymity_name = QuestionAnonymityLabel[anonymity.name].value + anonymity_name = request.translate(anonymity_name) + anonymity_opts.append({"id": anonymity.value, "name": anonymity_name}) + anonymity_opts = sorted(anonymity_opts, key=lambda x: x["id"]) + q_type_name = QuestionTypeLabel[q_type.name].value + q_type_name = request.translate(q_type_name) + result.append( + { + "id": q_type.value, + "name": q_type_name, + "anonymity_opts": anonymity_opts, + "order": order, + } + ) + result = sorted(result, key=lambda x: x["order"]) + return result + + +def get_question_by_field_name(field_name, questions): + for q in questions: + pattern = ( + rf"^" + rf"({q.question_code}(_[abc])?(_oth)?)|" + rf"(perf_{q.question_code}_[123])|" + rf"(char_{q.question_code}_(pos|neg))" + rf"$" + ) + if re.fullmatch(pattern, field_name): + return q + return None diff --git a/climmob/utility/validators.py b/climmob/utility/validators.py deleted file mode 100644 index 004ffcc5..00000000 --- a/climmob/utility/validators.py +++ /dev/null @@ -1,47 +0,0 @@ -import re - -from climmob.processes import userExists, emailExists - -# Form validation - -__all__ = ["validate_register_form"] - - -def validate_register_form(data, request, _): - error_summary = {} - errors = False - - if data["user_password"] != data["user_password2"]: - error_summary["InvalidPassword"] = _("Invalid password") - errors = True - if userExists(data["user_name"], request): - error_summary["UserExists"] = _("Username already exits") - errors = True - if emailExists(data["user_email"], request): - error_summary["EmailExists"] = _( - "There is already an account using to this email" - ) - errors = True - if data["user_policy"] == "False": - error_summary["CheckPolicy"] = _("You need to accept the terms of service") - errors = True - if data["user_name"] == "": - error_summary["EmptyUser"] = _("User cannot be emtpy") - errors = True - if data["user_password"] == "": - error_summary["EmptyPass"] = _("Password cannot be emtpy") - errors = True - if data["user_fullname"] == "": - error_summary["EmptyName"] = _("Full name cannot be emtpy") - errors = True - if data["user_email"] == "": - error_summary["EmptyEmail"] = _("Email cannot be emtpy") - errors = True - reg = re.compile(r"^[a-z0-9]+$") - if not reg.match(data["user_name"]): - error_summary["Caracters"] = _( - "The username can only use lowercase letters and numbers." - ) - errors = True - - return errors, error_summary diff --git a/climmob/views/Api/projectAssessmentStart.py b/climmob/views/Api/projectAssessmentStart.py index 35ae2976..d673cb3c 100644 --- a/climmob/views/Api/projectAssessmentStart.py +++ b/climmob/views/Api/projectAssessmentStart.py @@ -759,7 +759,7 @@ def ApiAssessmentPushProcess(self, structure, dataworking, activeProjectId): f.write(json.dumps(_json)) f.close() - storeJSONInMySQL( + success, msg = storeJSONInMySQL( self.user.login, "ASS", dataworking["user_owner"], @@ -771,6 +771,15 @@ def ApiAssessmentPushProcess(self, structure, dataworking, activeProjectId): activeProjectId, ) + if not success: + response = Response( + status=401, + body=self._( + "The data could not be saved. ERROR: " + msg + ), + ) + return response + logFile = pathfinal.replace(".json", ".log") if os.path.exists(logFile): doc = minidom.parse(logFile) diff --git a/climmob/views/Api/projectRegistryStart.py b/climmob/views/Api/projectRegistryStart.py index ec2eef78..5020dae6 100644 --- a/climmob/views/Api/projectRegistryStart.py +++ b/climmob/views/Api/projectRegistryStart.py @@ -1226,7 +1226,8 @@ def ApiRegistrationPushProcess(self, structure, dataworking, activeProjectId): f = open(pathfinal, "w") f.write(json.dumps(_json)) f.close() - storeJSONInMySQL( + + success, msg = storeJSONInMySQL( self.user.login, "REG", dataworking["user_owner"], @@ -1238,6 +1239,16 @@ def ApiRegistrationPushProcess(self, structure, dataworking, activeProjectId): activeProjectId, ) + if not success: + response = Response( + status=401, + body=self._( + "The data could not be registered. ERROR: " + + msg + ), + ) + return response + logFile = pathfinal.replace(".json", ".log") if os.path.exists(logFile): doc = minidom.parse(logFile) diff --git a/climmob/views/Api/project_analysis.py b/climmob/views/Api/project_analysis.py index d3a54f2c..f6d5d769 100644 --- a/climmob/views/Api/project_analysis.py +++ b/climmob/views/Api/project_analysis.py @@ -13,64 +13,32 @@ ) from climmob.views.classes import apiView from climmob.views.project_analysis import processToGenerateTheReport +from climmob.views.validators import TextField +from climmob.views.validators.ProjectExistsValidator import ProjectExistsValidator +from climmob.views.validators.project import HasAccessToProjectValidator class ReadDataOfProjectViewApi(apiView): - def processView(self): - - if self.request.method == "GET": - - obligatory = ["project_cod", "user_owner"] - try: - dataworking = json.loads(self.body) - except: - response = Response( - status=401, - body=self._( - "Error in the JSON, It does not have the 'body' parameter." - ), - ) - return response - - if sorted(obligatory) == sorted(dataworking.keys()): - - exitsproject = projectExists( - self.user.login, - dataworking["user_owner"], - dataworking["project_cod"], + validators = (ProjectExistsValidator, HasAccessToProjectValidator) + valid_fields = ( + TextField("project_cod"), + TextField("user_owner"), + ) + + def get(self): + response = Response( + status="200", + body=json.dumps( + getJSONResult( + self.context.body["user_owner"], + self.context.active_project_id, + self.context.body["project_cod"], self.request, + anonymize=True, ) - if exitsproject: - - activeProjectId = getTheProjectIdForOwner( - dataworking["user_owner"], - dataworking["project_cod"], - self.request, - ) - - response = Response( - status=200, - body=json.dumps( - getJSONResult( - dataworking["user_owner"], - activeProjectId, - dataworking["project_cod"], - self.request, - ) - ), - ) - return response - else: - response = Response( - status=401, body=self._("This project does not exist.") - ) - return response - else: - response = Response(status=401, body=self._("Error in the JSON.")) - return response - else: - response = Response(status=401, body=self._("Only accepts GET method.")) - return response + ), + ) + return response class ReadVariablesForAnalysisViewApi(apiView): diff --git a/climmob/views/assessment.py b/climmob/views/assessment.py index b9ef5c26..527fd558 100644 --- a/climmob/views/assessment.py +++ b/climmob/views/assessment.py @@ -34,6 +34,7 @@ getPhraseTranslationInLanguage, update_project_status, clone_assessment, + delete_anonymized_values_by_form_id, ) from climmob.products.forms.form import create_document_form from climmob.views.classes import privateView @@ -820,6 +821,9 @@ def processView(self): assessmentid, ) + schema = activeProjectUser + "_" + activeProjectCod + delete_anonymized_values_by_form_id(schema, assessmentid) + self.returnRawViewResult = True return HTTPFound(location=self.request.route_url("dashboard")) diff --git a/climmob/views/basic_views.py b/climmob/views/basic_views.py index 6a371c89..7c2accf3 100644 --- a/climmob/views/basic_views.py +++ b/climmob/views/basic_views.py @@ -1,3 +1,4 @@ +import re from datetime import datetime import json import logging @@ -27,8 +28,9 @@ getSectorList, getUserCount, getProjectCount, + userExists, + emailExists, ) -from climmob.utility import validate_register_form from climmob.utility.email import build_email_message from climmob.utility.helpers import readble_date from climmob.views.classes import publicView @@ -360,7 +362,7 @@ def post(self): "countries": getCountryList(self.request), "sectors": getSectorList(self.request), } - errors, error_summary = validate_register_form(data, self.request, self._) + errors, error_summary = self.validate_register_form(data) if errors: response["error_summary"] = error_summary @@ -403,3 +405,45 @@ def post(self): location=self.request.route_url("dashboard"), headers=headers, ) + + # Create validator if needed by another view + def validate_register_form(self, data): + error_summary = {} + errors = False + + if data["user_password"] != data["user_password2"]: + error_summary["InvalidPassword"] = self._("Invalid password") + errors = True + if userExists(data["user_name"], self.request): + error_summary["UserExists"] = self._("Username already exits") + errors = True + if emailExists(data["user_email"], self.request): + error_summary["EmailExists"] = self._( + "There is already an account using to this email" + ) + errors = True + if data["user_policy"] == "False": + error_summary["CheckPolicy"] = self._( + "You need to accept the terms of service" + ) + errors = True + if data["user_name"] == "": + error_summary["EmptyUser"] = self._("User cannot be emtpy") + errors = True + if data["user_password"] == "": + error_summary["EmptyPass"] = self._("Password cannot be emtpy") + errors = True + if data["user_fullname"] == "": + error_summary["EmptyName"] = self._("Full name cannot be emtpy") + errors = True + if data["user_email"] == "": + error_summary["EmptyEmail"] = self._("Email cannot be emtpy") + errors = True + reg = re.compile(r"^[a-z0-9]+$") + if not reg.match(data["user_name"]): + error_summary["Caracters"] = self._( + "The username can only use lowercase letters and numbers." + ) + errors = True + + return errors, error_summary diff --git a/climmob/views/cleanErrorLogs.py b/climmob/views/cleanErrorLogs.py index 55e08bb6..e714e55e 100644 --- a/climmob/views/cleanErrorLogs.py +++ b/climmob/views/cleanErrorLogs.py @@ -17,6 +17,9 @@ getTheProjectIdForOwner, getActiveProject, getQuestionsStructure, + delete_assessment_data_by_qst163, + delete_registry_data_by_qst162, + delete_anonymized_values_by_form_id_and_reg_id, ) from climmob.processes.odk.api import storeJSONInMySQL from climmob.views.classes import privateView @@ -31,6 +34,8 @@ def processView(self): activeProjectUser = self.request.matchdict["user"] activeProjectCod = self.request.matchdict["project"] + schema = activeProjectUser + "_" + activeProjectCod + activeProjectId = getTheProjectIdForOwner( activeProjectUser, activeProjectCod, self.request ) @@ -94,21 +99,15 @@ def processView(self): if str(dataworking["txt_oldvalue"]) == str( dataworking["newqst"].split("-")[1] ): - - query = ( - "Delete from " - + activeProjectUser - + "_" - + activeProjectCod - + ".REG_geninfo where qst162='" - + dataworking["newqst"].split("-")[1] - + "'" + delete_registry_data_by_qst162( + schema, + dataworking["newqst"].split("-")[1], + self.user.login, ) - execute_two_sqls( - "SET @odktools_current_user = '" - + self.user.login - + "';", - query, + delete_anonymized_values_by_form_id_and_reg_id( + schema, + "-", + dataworking["newqst"].split("-")[1], ) storeJSONInMySQL( @@ -159,22 +158,15 @@ def processView(self): if str(dataworking["txt_oldvalue"]) == str( dataworking["newqst2"] ): - query = ( - "Delete from " - + activeProjectUser - + "_" - + activeProjectCod - + ".ASS" - + codeId - + "_geninfo where qst163='" - + dataworking["newqst2"] - + "'" + delete_assessment_data_by_qst163( + schema, + codeId, + dataworking["newqst2"], + self.user.login, ) - execute_two_sqls( - "SET @odktools_current_user = '" - + self.user.login - + "'; ", - query, + + delete_anonymized_values_by_form_id_and_reg_id( + schema, codeId, dataworking["newqst2"] ) storeJSONInMySQL( @@ -253,20 +245,15 @@ def processView(self): if str(dataworking["txt_oldvalue"]) == str( dataworking["newqst"].split("-")[1] ): - query = ( - "Delete from " - + activeProjectUser - + "_" - + activeProjectCod - + ".REG_geninfo where qst162='" - + dataworking["newqst"].split("-")[1] - + "'" + delete_registry_data_by_qst162( + schema, + dataworking["newqst"].split("-")[1], + self.user.login, ) - execute_two_sqls( - "SET @odktools_current_user = '" - + self.user.login - + "'; ", - query, + delete_anonymized_values_by_form_id_and_reg_id( + schema, + "-", + dataworking["newqst"].split("-")[1], ) update_registry_status_log( @@ -290,22 +277,14 @@ def processView(self): if str(dataworking["txt_oldvalue"]) == str( dataworking["newqst2"] ): - query = ( - "Delete from " - + activeProjectUser - + "_" - + activeProjectCod - + ".ASS" - + codeId - + "_geninfo where qst163='" - + dataworking["newqst2"] - + "'" + delete_assessment_data_by_qst163( + schema, + codeId, + dataworking["newqst2"], + self.user.login, ) - execute_two_sqls( - "SET @odktools_current_user = '" - + self.user.login - + "'; ", - query, + delete_anonymized_values_by_form_id_and_reg_id( + schema, codeId, dataworking["newqst2"] ) update_assessment_status_log( @@ -378,7 +357,7 @@ def processView(self): # Edited by Brandon path = os.path.join( self.request.registry.settings["user.repository"], - *[activeProjectUser, activeProjectCod] + *[activeProjectUser, activeProjectCod], ) paths = ["db", "ass", codeId, "create.xml"] path = os.path.join(path, *paths) diff --git a/climmob/views/context/ApiContext.py b/climmob/views/context/ApiContext.py index a1f11dd9..5f845fde 100644 --- a/climmob/views/context/ApiContext.py +++ b/climmob/views/context/ApiContext.py @@ -12,15 +12,15 @@ def __init__(self, request): super().__init__(request) @cached_property - def __body(self): + def body(self): body = get_body_from_api_request(self.request) return json.loads(body) @cached_property def active_project_id(self): active_project_id = getTheProjectIdForOwner( - self.__body["user_owner"], - self.__body["project_cod"], + self.body["user_owner"], + self.body["project_cod"], self.request, ) return active_project_id diff --git a/climmob/views/dashboard.py b/climmob/views/dashboard.py index 8b954b9a..e0af3ca5 100644 --- a/climmob/views/dashboard.py +++ b/climmob/views/dashboard.py @@ -1,4 +1,6 @@ +import mysql.connector from pyramid.httpexceptions import HTTPNotFound, HTTPFound +from sqlalchemy.exc import ProgrammingError import climmob.plugins as p from climmob.processes import ( @@ -15,6 +17,7 @@ AssessmentsInformation, seeProgress, getTheProjectIdForOwner, + is_project_anonymized, ) from climmob.views.classes import privateView, publicView @@ -41,6 +44,17 @@ def processView(self): activeProjectData = getActiveProject(self.user.login, self.request) + schema = ( + activeProjectData["owner"]["user_name"] + + "_" + + activeProjectData["project_cod"] + ) + + try: + project_is_anonymized = is_project_anonymized(schema) + except ProgrammingError: + project_is_anonymized = False + session = self.request.session session["activeProject"] = activeProjectId @@ -99,6 +113,7 @@ def processView(self): activeProjectCod, self.request, ), + "project_is_anonymized": project_is_anonymized, } for plugin in p.PluginImplementations(p.IDashBoard): context = plugin.before_returning_dashboard_context( @@ -108,6 +123,17 @@ def processView(self): else: activeProjectData = getActiveProject(self.user.login, self.request) + schema = ( + activeProjectData["owner"]["user_name"] + + "_" + + activeProjectData["project_cod"] + ) + + try: + project_is_anonymized = is_project_anonymized(schema) + except ProgrammingError: + project_is_anonymized = False + if activeProjectData: self.returnRawViewResult = True return HTTPFound( @@ -127,6 +153,7 @@ def processView(self): "progress": {}, "pcompleted": 0, "allassclosed": False, + "project_is_anonymized": project_is_anonymized, } for plugin in p.PluginImplementations(p.IDashBoard): context = plugin.before_returning_dashboard_context( diff --git a/climmob/views/editData.py b/climmob/views/editData.py index d990ccea..bbcd7187 100755 --- a/climmob/views/editData.py +++ b/climmob/views/editData.py @@ -12,8 +12,7 @@ projectExists, getJSONResult, ) -from climmob.products.analysisdata.analysisdata import create_datacsv -from climmob.products.dataxlsx.dataxlsx import create_XLSXToDownload +from climmob.products.analysisdata.analysisdata import create_raw_data from climmob.products.errorLogDocument.errorLogDocument import create_error_log_document from climmob.views.classes import privateView from climmob.views.editDataDB import ( @@ -30,10 +29,10 @@ def processView(self): activeProjectCod = self.request.matchdict["project"] formId = self.request.matchdict["formid"] formatId = self.request.matchdict["formatid"] + anonymize = str(self.request.params.get("anonymize")).lower() == "true" includeRegistry = True includeAssessment = True code = "" - formatExtra = "" if not projectExists( self.user.login, activeProjectUser, activeProjectCod, self.request @@ -64,36 +63,32 @@ def processView(self): includeRegistry, includeAssessment, code, + anonymize=anonymize, ) if formatId not in ["csv", "xlsx"]: raise HTTPNotFound() - if formatId == "csv": - create_datacsv( - activeProjectUser, - activeProjectId, - activeProjectCod, - info, - self.request, - formId, - code, - ) + create_raw_data( + activeProjectUser, + activeProjectId, + activeProjectCod, + info, + self.request, + formId, + code, + file_type=formatId, + anonymized=anonymize, + ) - if formatId == "xlsx": - formatExtra = formatId + "_" - create_XLSXToDownload( - activeProjectUser, - activeProjectId, - activeProjectCod, - self.request, - formId, - code, - ) + format_extra = "xlsx_" if formatId == "xlsx" else "" + product_id_extra = "-anonymized" if anonymize else "" url = self.request.route_url( "productList", - _query={"product1": "create_data_" + formatExtra + formId + "_" + code}, + _query={ + "product1": f"create_data{product_id_extra}_{format_extra}{formId}_{code}" + }, ) self.returnRawViewResult = True return HTTPFound(location=url) @@ -216,7 +211,7 @@ def processView(self): path = os.path.join( self.request.registry.settings["user.repository"], - *[activeProjectUser, activeProjectCod] + *[activeProjectUser, activeProjectCod], ) if code == "": paths = ["db", formId, "create.xml"] @@ -268,6 +263,8 @@ def processView(self): path, code, self.user.login, + activeProjectId, + self.request, ) dataXML = getNamesEditByColums(path) diff --git a/climmob/views/editDataDB.py b/climmob/views/editDataDB.py index 17897d13..5e4c4374 100755 --- a/climmob/views/editDataDB.py +++ b/climmob/views/editDataDB.py @@ -2,7 +2,15 @@ import xml.etree.ElementTree as ET from climmob.models.repository import sql_execute, execute_two_sqls -from climmob.processes import getProjectData, getQuestionOptionsByQuestionCode +from climmob.processes import ( + getProjectData, + getQuestionOptionsByQuestionCode, + get_sensitive_questions_anonymity_by_project_id, +) +from climmob.processes.db.anonymized import update_anonymized +from climmob.processes.db.assessment import get_assessment_data_by_qst163 +from climmob.processes.db.registry import get_registry_data_by_qst162 +from climmob.utility import get_question_by_field_name, QuestionAnonymity def get_FieldsByType(types, file): @@ -329,10 +337,15 @@ def fillDataTable( return json.dumps(ret) -def update_edited_data(userOwner, projectCod, form, data, file, code, by): +def update_edited_data( + userOwner, projectCod, form, data, file, code, by, project_id, request +): data = json.loads(data[0]) + schema = userOwner + "_" + projectCod + questions = get_sensitive_questions_anonymity_by_project_id(project_id, request) + for row in data: del row["id"] if row["flag_update"]: @@ -341,6 +354,7 @@ def update_edited_data(userOwner, projectCod, form, data, file, code, by): form.upper() + code, ) del row["flag_update"] + to_anonymize = [] for key in row: val = "" addField = True @@ -352,21 +366,37 @@ def update_edited_data(userOwner, projectCod, form, data, file, code, by): else: if key in get_FieldsByType(["select1"], file): if row[key] and row[key] != "None": - val = ( - "'" - + str(row[key]).replace("[", "").replace("]", "") - + "'" - ) + val = str(row[key]).replace("[", "").replace("]", "") else: addField = False else: if key in get_FieldsByType(["select"], file): - val = "'" + " ".join(row[key]) + "'" + val = " ".join(row[key]) else: - val = "'" + str(row[key]) + "'" + val = str(row[key]) if addField: - query_update += key + "=" + val + ", " + query_update += key + "='" + val + "', " + question = get_question_by_field_name(key, questions) + if ( + question + and question.question_anonymity + != QuestionAnonymity.REMOVE.value + ): + to_anonymize.append( + {"field_name": key, "value": val, "question": question} + ) + + reg_id = row["qst162"] if form == "reg" else row["qst163"] + form_id = "-" if form == "reg" else code + columns = [field["field_name"] for field in to_anonymize] + + if form_id == "-": + current = get_registry_data_by_qst162(schema, reg_id, columns) + else: + current = get_assessment_data_by_qst163( + schema, form_id, reg_id, columns + ) query_update = ( query_update[:-2] + " where rowuuid ='" + str(row["rowuuid"]) + "';" @@ -377,7 +407,11 @@ def update_edited_data(userOwner, projectCod, form, data, file, code, by): execute_two_sqls( "SET @odktools_current_user = '" + by + "'; ", query_update ) + update_anonymized( + to_anonymize, schema, form_id, reg_id, request, current + ) except Exception as e: print(str(e)) return 0, str(e) + return 1, "" diff --git a/climmob/views/productsList.py b/climmob/views/productsList.py index ca3ee7e2..9e5390b8 100644 --- a/climmob/views/productsList.py +++ b/climmob/views/productsList.py @@ -1,4 +1,5 @@ import os +import re from pyramid.httpexceptions import HTTPFound from pyramid.httpexceptions import HTTPNotFound @@ -28,10 +29,10 @@ get_registry_logs, get_assessment_logs, getPrjLangDefaultInProject, + is_project_anonymized, ) from climmob.products import product_found -from climmob.products.analysisdata.analysisdata import create_datacsv -from climmob.products.dataxlsx.dataxlsx import create_XLSXToDownload +from climmob.products.analysisdata.analysisdata import create_raw_data from climmob.products.colors.colors import create_colors_cards from climmob.products.errorLogDocument.errorLogDocument import create_error_log_document from climmob.products.fieldagents.fieldagents import create_fieldagents_report @@ -84,11 +85,26 @@ def processView(self): productsAvailable = [] assessments = [] + schema = activeProjectData["user_name"] + "_" + activeProjectData["project_cod"] + + project_is_anonymized = is_project_anonymized(schema) + if activeProjectData: products = getDataProduct(activeProjectData["project_id"], self.request) for product in products: + + if product["product_id"] in [ + "datacsv-anonymized", + "dataxlsx-anonymized", + ] and ( + not project_is_anonymized + or self.request.registry.settings.get("module.dataprivacy", "false") + == "false" + ): + continue + if product_found(product["product_id"]): contentType = product["output_mimetype"] filename = product["output_id"] @@ -107,22 +123,28 @@ def processView(self): if product["product_id"] in [ "documentform", "datacsv", + "dataxlsx", + "datacsv-anonymized", + "dataxlsx-anonymized", "errorlogdocument", "multimediadownloads", "uploaddata", - "dataxlsx", "observationcards", "climmobexplanationkit", ]: - assessId = product["process_name"].split("_")[3] - if product["product_id"] == "dataxlsx": - assessId = product["process_name"].split("_")[4] - - product["extraInformation"] = get_project_assessment_info( - activeProjectData["project_id"], - assessId, - self.request, + product["extraInformation"] = None + pattern = re.compile( + r".+?(?:(?:Assessment))_" # not captured + r"([a-f0-9]+)" # captured (group 1) ) + match = pattern.fullmatch(product["process_name"]) + if match: + assess_id = match.group(1) + product["extraInformation"] = get_project_assessment_info( + activeProjectData["project_id"], + assess_id, + self.request, + ) productsAvailable.append(product) @@ -140,6 +162,7 @@ def processView(self): "Products": productsAvailable, "assessments": assessments, "sectionActive": "productlist", + "project_is_anonymized": project_is_anonymized, } @@ -277,9 +300,18 @@ def processView(self): listOfLabels, ) - if productid == "datacsv": - locale = self.request.locale_name + if productid in [ + "datacsv", + "datacsv-anonymized", + "dataxlsx", + "dataxlsx-anonymized", + ]: + anonymized = productid in ["datacsv-anonymized", "dataxlsx-anonymized"] + file_type = "csv" if "csv" in productid else "xlsx" infoProduct = processname.split("_") + if file_type == "xlsx": + infoProduct[2] = infoProduct[3] + infoProduct[3] = infoProduct[4] if infoProduct[2] == "Registration": info = getJSONResult( activeProjectData["owner"]["user_name"], @@ -287,6 +319,7 @@ def processView(self): activeProjectData["project_cod"], self.request, includeAssessment=False, + anonymize=anonymized, ) else: if infoProduct[2] == "Assessment": @@ -296,6 +329,7 @@ def processView(self): activeProjectData["project_cod"], self.request, assessmentCode=infoProduct[3], + anonymize=anonymized, ) else: info = getJSONResult( @@ -303,9 +337,10 @@ def processView(self): activeProjectData["project_id"], activeProjectData["project_cod"], self.request, + anonymize=anonymized, ) - create_datacsv( + create_raw_data( activeProjectData["owner"]["user_name"], activeProjectData["project_id"], activeProjectData["project_cod"], @@ -313,17 +348,8 @@ def processView(self): self.request, infoProduct[2], infoProduct[3], - ) - - if productid == "dataxlsx": - infoProduct = processname.split("_") - create_XLSXToDownload( - activeProjectData["owner"]["user_name"], - activeProjectData["project_id"], - activeProjectData["project_cod"], - self.request, - infoProduct[3], - infoProduct[4], + file_type=file_type, + anonymized=anonymized, ) if productid == "documentform": diff --git a/climmob/views/project_analysis.py b/climmob/views/project_analysis.py index dfd2034f..522c2edd 100644 --- a/climmob/views/project_analysis.py +++ b/climmob/views/project_analysis.py @@ -8,7 +8,7 @@ getProjectProgress, ) from climmob.products.analysis.analysis import create_analysis -from climmob.products.analysisdata.analysisdata import create_datacsv +from climmob.products.analysisdata.analysisdata import create_raw_data from climmob.views.classes import privateView @@ -155,7 +155,7 @@ def processToGenerateTheReport( combinationRerence, ) - create_datacsv( + create_raw_data( activeProjectData["owner"]["user_name"], activeProjectData["project_id"], activeProjectData["project_cod"], diff --git a/climmob/views/question.py b/climmob/views/question.py index cde6f9a3..940ea580 100644 --- a/climmob/views/question.py +++ b/climmob/views/question.py @@ -39,6 +39,12 @@ getPhraseTranslationInLanguage, knowIfUserHasCreatedTranslations, ) +from climmob.utility import ( + get_enum_as_dict, + QuestionAnonymity, + get_question_types_with_anonymity_labeled, + QuestionType, +) from climmob.views.classes import privateView from climmob.views.validators.question.QuestionMinMaxValidator import ( QuestionMinMaxValidator, @@ -785,6 +791,8 @@ def processView(self): nextPage = self.request.params.get("next") + question_types = get_question_types_with_anonymity_labeled(self.request) + regularDict = { "UserQuestion": UserQuestionMoreBioversity(user_name, self.request), "knowIfUserHasCreatedTranslations": knowIfUserHasCreatedTranslations( @@ -799,6 +807,9 @@ def processView(self): "seeQuestion": seeQuestion, "nextPage": nextPage, "sectionActive": "questions", + "question_types": question_types, + "QuestionAnonymity": get_enum_as_dict(QuestionAnonymity), + "QuestionType": get_enum_as_dict(QuestionType), } return regularDict diff --git a/climmob/views/registry.py b/climmob/views/registry.py index 4e710471..2b5301a0 100644 --- a/climmob/views/registry.py +++ b/climmob/views/registry.py @@ -27,6 +27,7 @@ modifyProjectMainLanguage, projectRegStatus, update_project_status, + delete_anonymized_values_by_form_id, ) from climmob.products import stopTasksByProcess from climmob.views.classes import privateView @@ -165,6 +166,9 @@ def processView(self): "", ) + schema = activeProjectUser + "_" + activeProjectCod + delete_anonymized_values_by_form_id(schema, "-") + self.returnRawViewResult = True return HTTPFound(location=self.request.route_url("dashboard")) diff --git a/climmob/views/validators/project/__init__.py b/climmob/views/validators/project/__init__.py index 46643210..56297915 100644 --- a/climmob/views/validators/project/__init__.py +++ b/climmob/views/validators/project/__init__.py @@ -1,3 +1,6 @@ from climmob.views.validators.project.CanEditProjectValidator import ( CanEditProjectValidator, ) +from climmob.views.validators.project.has_access_to_project_validator import ( + HasAccessToProjectValidator, +) diff --git a/climmob/views/validators/project/has_access_to_project_validator.py b/climmob/views/validators/project/has_access_to_project_validator.py new file mode 100644 index 00000000..b111dbc3 --- /dev/null +++ b/climmob/views/validators/project/has_access_to_project_validator.py @@ -0,0 +1,15 @@ +from pyramid.httpexceptions import HTTPForbidden + +from climmob.views.validators.BaseValidator import BaseValidator + + +class HasAccessToProjectValidator(BaseValidator): + def run(self): + access_type = self.view.context.access_type + + if access_type is None: + raise HTTPForbidden( + self._( + "The access assigned for this project does not allow you to get the collected data." + ) + ) diff --git a/setup.py b/setup.py index c860bda8..99ee5b68 100644 --- a/setup.py +++ b/setup.py @@ -224,6 +224,7 @@ "update_map_points = climmob.scripts.updatemappoints:main", "mysqldumps_climmob_dbs = climmob.scripts.mysqldumpclimmobdbs:main", "configure_tests = climmob.scripts.configuretests:main", + "anonymize_project = climmob.scripts.anonymize_project:main", ], }, )