From 57a20c535c8424f58f03b3e4f237afa54bb3fcbe Mon Sep 17 00:00:00 2001 From: Subhash Gupta Date: Thu, 26 Feb 2026 08:53:12 +0530 Subject: [PATCH 1/2] feat: add POST /rest/v1/suggest_cre_mappings API endpoint Signed-off-by: Subhash Gupta --- application/cmd/cre_main.py | 52 ++++++++++++++++++++++++++++++ application/tests/web_main_test.py | 42 ++++++++++++++++++++++++ application/web/web_main.py | 34 +++++++++++++++++++ 3 files changed, 128 insertions(+) diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index ead5a4281..a269b0bbf 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -25,6 +25,7 @@ from alive_progress import alive_bar from application.prompt_client import prompt_client as prompt_client from application.utils import gap_analysis +from application.prompt_client.prompt_client import SIMILARITY_THRESHOLD logging.basicConfig() logger = logging.getLogger(__name__) @@ -314,6 +315,57 @@ def register_standard( redis.wait_for_jobs(jobs) conn.set(standard_hash, value="") +def suggest_cre_mappings( + standard_entries: List[defs.Standard], + collection: db.Node_collection, + confidence_threshold: float = SIMILARITY_THRESHOLD, +) -> Dict[str, Any]: + """ + Given a list of Standard entries, suggest CRE mappings using + cosine similarity on existing embeddings. + + Returns high-confidence matches and flags low-confidence ones + for human review. + + Args: + standard_entries: list of Standard nodes to map + collection: database connection + confidence_threshold: minimum similarity score to auto-map + + Returns: + Dict with 'mapped' (high confidence) and 'needs_review' (low confidence) lists + """ + if not standard_entries: + logger.warning("suggest_cre_mappings() called with no standard_entries") + return {"mapped": [], "needs_review": []} + + ph = prompt_client.PromptHandler(database=collection) + results: Dict[str, Any] = {"mapped": [], "needs_review": []} + + for node in standard_entries: + text = " ".join(filter(None, [node.name, node.section, node.description])) + if not text.strip(): + continue + embedding = ph.get_text_embeddings(text) + cre_id, similarity = ph.get_id_of_most_similar_cre_paginated( + embedding, similarity_threshold=confidence_threshold + ) + entry = { + "standard": node.todict(), + "suggested_cre_id": cre_id, + "confidence": round(float(similarity), 4) if similarity else None, + } + if cre_id and similarity and similarity >= confidence_threshold: + results["mapped"].append(entry) + else: + results["needs_review"].append(entry) + + logger.info( + f"suggest_cre_mappings: {len(results['mapped'])} mapped, " + f"{len(results['needs_review'])} need review" + ) + return results + def parse_standards_from_spreadsheeet( cre_file: List[Dict[str, Any]], diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 9e219b4ce..23c10cb5e 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -21,6 +21,8 @@ from application.defs import cre_defs as defs from application.web import web_main from application.utils.gap_analysis import GAP_ANALYSIS_TIMEOUT +from application.cmd import cre_main +from application.utils import spreadsheet_parsers class MockJob: @@ -973,3 +975,43 @@ def test_get_cre_csv(self) -> None: data.getvalue(), response.data.decode(), ) + + @patch.object(cre_main, "suggest_cre_mappings") + @patch("application.utils.spreadsheet_parsers.parse_export_format") + def test_suggest_cre_mappings_endpoint( + self, + mock_parse, + mock_suggest, + ) -> None: + mock_parse.return_value = { + defs.Credoctypes.Standard.value: [ + defs.Standard(name="PCI-DSS", section="Use strong cryptography") + ] + } + mock_suggest.return_value = { + "mapped": [ + { + "standard": {"name": "PCI-DSS", "section": "Use strong cryptography"}, + "suggested_cre_id": "cre-db-id-123", + "confidence": 0.85, + } + ], + "needs_review": [], + } + + # Create a fake CSV file + csv_content = b"Standard,section,description\nPCI-DSS,Use strong cryptography,Encrypt all data" + data = {"cre_csv": (io.BytesIO(csv_content), "test.csv")} + + with self.app.test_client() as client: + response = client.post( + "/rest/v1/suggest_cre_mappings", + data=data, + content_type="multipart/form-data", + ) + self.assertEqual(response.status_code, 200) + result = json.loads(response.data) + self.assertIn("mapped", result) + self.assertIn("needs_review", result) + self.assertEqual(len(result["mapped"]), 1) + self.assertEqual(result["mapped"][0]["suggested_cre_id"], "cre-db-id-123") diff --git a/application/web/web_main.py b/application/web/web_main.py index 6738a0109..72336d262 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -956,6 +956,40 @@ def import_from_cre_csv() -> Any: } ) +@app.route("/rest/v1/suggest_cre_mappings", methods=["POST"]) +def suggest_cre_mappings() -> Any: + """Given a CSV of standard entries, suggest matching CREs using embeddings. + + Accepts a multipart form upload with a 'cre_csv' file. + Returns high-confidence mappings and low-confidence ones flagged for review. + """ + database = db.Node_collection() + file = request.files.get("cre_csv") + + if file is None: + abort(400, "No file provided") + + contents = file.read() + csv_read = csv.DictReader(contents.decode("utf-8").splitlines()) + try: + documents = spreadsheet_parsers.parse_export_format(list(csv_read)) + except cre_exceptions.DuplicateLinkException as dle: + abort(500, f"error during parsing of the incoming CSV, err:{dle}") + + standards = [] + for _, entries in documents.items(): + if _ != defs.Credoctypes.CRE.value: + standards.extend(list(entries)) + + if not standards: + abort(400, "No standard entries found in CSV") + + result = cre_main.suggest_cre_mappings( + standard_entries=standards, + collection=database, + ) + return jsonify(result) + # /End Importing Handlers From 55968b411b744569b88f237d1d53b57ec5c8313e Mon Sep 17 00:00:00 2001 From: Subhash Gupta Date: Fri, 27 Feb 2026 17:11:33 +0530 Subject: [PATCH 2/2] feat: add AI-suggested CRE mapping UI to MyOpenCRE page Signed-off-by: Subhash Gupta --- .../src/pages/MyOpenCRE/MyOpenCRE.tsx | 210 +++++++++++------- 1 file changed, 133 insertions(+), 77 deletions(-) diff --git a/application/frontend/src/pages/MyOpenCRE/MyOpenCRE.tsx b/application/frontend/src/pages/MyOpenCRE/MyOpenCRE.tsx index 9ccfafa2f..046dca770 100644 --- a/application/frontend/src/pages/MyOpenCRE/MyOpenCRE.tsx +++ b/application/frontend/src/pages/MyOpenCRE/MyOpenCRE.tsx @@ -1,22 +1,25 @@ import './MyOpenCRE.scss'; import React, { useState } from 'react'; -import { Button, Container, Form, Header, Message } from 'semantic-ui-react'; - +import { Button, Container, Form, Header, Label, Loader, Message, Table } from 'semantic-ui-react'; import { useEnvironment } from '../../hooks'; export const MyOpenCRE = () => { const { apiUrl } = useEnvironment(); - - // Upload enabled only for local/dev const isUploadEnabled = apiUrl !== '/rest/v1'; + // CSV import state const [selectedFile, setSelectedFile] = useState(null); - const [loading, setLoading] = useState(false); - const [error, setError] = useState(null); - const [success, setSuccess] = useState(null); + const [importLoading, setImportLoading] = useState(false); + const [importError, setImportError] = useState(null); + const [importSuccess, setImportSuccess] = useState(null); - /* ------------------ CSV DOWNLOAD ------------------ */ + // AI suggest state + const [suggestLoading, setSuggestLoading] = useState(false); + const [suggestError, setSuggestError] = useState(null); + const [mapped, setMapped] = useState([]); + const [needsReview, setNeedsReview] = useState([]); + const [suggestFileName, setSuggestFileName] = useState(''); const downloadCreCsv = async () => { try { @@ -24,20 +27,14 @@ export const MyOpenCRE = () => { method: 'GET', headers: { Accept: 'text/csv' }, }); - - if (!response.ok) { - throw new Error(`HTTP error ${response.status}`); - } - + if (!response.ok) throw new Error(`HTTP error ${response.status}`); const blob = await response.blob(); const url = window.URL.createObjectURL(blob); - const link = document.createElement('a'); link.href = url; link.download = 'opencre-cre-mapping.csv'; document.body.appendChild(link); link.click(); - document.body.removeChild(link); window.URL.revokeObjectURL(url); } catch (err) { @@ -48,16 +45,10 @@ export const MyOpenCRE = () => { const downloadTemplate = () => { const headers = ['standard_name', 'standard_section', 'cre_id', 'notes']; - const csvContent = headers.join(',') + '\n'; - - const blob = new Blob([csvContent], { - type: 'text/csv;charset=utf-8;', - }); - + const blob = new Blob([csvContent], { type: 'text/csv;charset=utf-8;' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); - link.href = url; link.setAttribute('download', 'myopencre_mapping_template.csv'); document.body.appendChild(link); @@ -65,79 +56,81 @@ export const MyOpenCRE = () => { document.body.removeChild(link); }; - /* ------------------ FILE SELECTION ------------------ */ - const onFileChange = (e: React.ChangeEvent) => { - setError(null); - setSuccess(null); - + setImportError(null); + setImportSuccess(null); if (!e.target.files || e.target.files.length === 0) return; - const file = e.target.files[0]; - if (!file.name.toLowerCase().endsWith('.csv')) { - setError('Please upload a valid CSV file.'); + setImportError('Please upload a valid CSV file.'); e.target.value = ''; setSelectedFile(null); return; } - setSelectedFile(file); }; - /* ------------------ CSV UPLOAD ------------------ */ - const uploadCsv = async () => { if (!selectedFile) return; - - setLoading(true); - setError(null); - setSuccess(null); - + setImportLoading(true); + setImportError(null); + setImportSuccess(null); const formData = new FormData(); formData.append('cre_csv', selectedFile); - try { const response = await fetch(`${apiUrl}/cre_csv_import`, { method: 'POST', body: formData, }); - if (response.status === 403) { - throw new Error( - 'CSV import is disabled on hosted environments. Run OpenCRE locally with CRE_ALLOW_IMPORT=true.' - ); + throw new Error('CSV import is disabled on hosted environments. Run OpenCRE locally with CRE_ALLOW_IMPORT=true.'); } - if (!response.ok) { const text = await response.text(); throw new Error(text || 'CSV import failed'); } - const result = await response.json(); - setSuccess(result); + setImportSuccess(result); setSelectedFile(null); } catch (err: any) { - setError(err.message || 'Unexpected error during import'); + setImportError(err.message || 'Unexpected error during import'); } finally { - setLoading(false); + setImportLoading(false); } }; - /* ------------------ UI ------------------ */ + const handleSuggestUpload = async (e: React.ChangeEvent) => { + const file = e.target.files?.[0]; + if (!file) return; + setSuggestFileName(file.name); + setSuggestLoading(true); + setSuggestError(null); + setMapped([]); + setNeedsReview([]); + const formData = new FormData(); + formData.append('cre_csv', file); + try { + const response = await fetch(`${apiUrl}/suggest_cre_mappings`, { + method: 'POST', + body: formData, + }); + if (!response.ok) throw new Error(`Server error: ${response.status}`); + const result = await response.json(); + setMapped(result.mapped || []); + setNeedsReview(result.needs_review || []); + } catch (err: any) { + setSuggestError(err.message || 'An error occurred while processing your file.'); + } finally { + setSuggestLoading(false); + } + }; return (
MyOpenCRE
- -

- MyOpenCRE allows you to map your own security standard (e.g. SOC2) to OpenCRE Common Requirements - using a CSV spreadsheet. -

-

- Start by downloading the CRE catalogue below, then map your standard’s controls or sections to CRE IDs - in the spreadsheet. + MyOpenCRE allows you to map your own security standard (e.g. SOC2) to OpenCRE Common + Requirements using a CSV spreadsheet.

@@ -149,46 +142,109 @@ export const MyOpenCRE = () => {
-
-
Upload Mapping CSV
+ {/* AI Suggest Section */} +
+
AI-Suggested CRE Mappings
+

Upload your standard's CSV to get automatic CRE mapping suggestions powered by AI.

+ + + + {suggestLoading && } + {suggestError && Error

{suggestError}

} + + {mapped.length > 0 && ( + <> +
+ Suggested Mappings +
+ + + + Standard Section + Suggested CRE + Confidence + + + + {mapped.map((item, idx) => ( + + {item.standard?.section || item.standard?.name} + + + {item.suggested_cre_id} + + + + + + + ))} + +
+ + )} + + {needsReview.length > 0 && ( + <> +
+ Needs Review +
+

These controls could not be automatically mapped and require manual review.

+ + + + Standard Section + Description + + + + {needsReview.map((item, idx) => ( + + {item.standard?.section || item.standard?.name} + {item.standard?.description || '—'} + + ))} + +
+ + )} +
+ {/* CSV Import Section */} +
+
Import Standard into OpenCRE

Upload your completed mapping spreadsheet to import your standard into OpenCRE.

{!isUploadEnabled && ( - CSV upload is disabled on hosted environments due to resource constraints. -
- Please run OpenCRE locally to enable standard imports. + CSV upload is disabled on hosted environments. Run OpenCRE locally with CRE_ALLOW_IMPORT=true. )} - - {error && {error}} - - {success && ( + {importError && {importError}} + {importSuccess && ( Import successful
    -
  • New CREs added: {success.new_cres?.length ?? 0}
    • -
  • Standards imported: {success.new_standards}
    • +
  • New CREs added: {importSuccess.new_cres?.length ?? 0}
    • +
  • Standards imported: {importSuccess.new_standards}
 )}
- + - -
); -}; +}; \ No newline at end of file