diff --git a/scanpipe/pipes/d2d.py b/scanpipe/pipes/d2d.py index 5046a61b09..a73b95cb0d 100644 --- a/scanpipe/pipes/d2d.py +++ b/scanpipe/pipes/d2d.py @@ -70,6 +70,16 @@ FROM = "from/" TO = "to/" +BASENAME_EXTENSION_MAP = { + ".cjs": [".ts", ".mts", ".js"], + ".mjs": [".ts", ".mts", ".js"], + ".css": [".less", ".scss"], + ".d.ts": [".ts", ".js"], + ".js": [".ts", ".tsx", ".coffee", ".jsx", ".vue", ".svelte", ".elm"], + ".min.js": [".js"], + ".d.mts": [".mts", ".ts"], +} + def get_inputs(project): """ @@ -119,6 +129,164 @@ def get_best_path_matches(to_resource, matches): return matches +def get_basename_without_extension(path): + """Return a tuple of (basename, extension) from a path.""" + path_obj = Path(path.lstrip("/")) + name = path_obj.name + + if name.endswith(".bak"): + basename = name[: -len(".bak")] + return basename, ".bak" + + for ext in [".d.ts", ".min.js", ".d.mts"]: + if name.endswith(ext): + basename = name[: -len(ext)] + return basename, ext + + if "." in name: + parts = name.rsplit(".", 2) + if len(parts) == 3: + base, ext1, ext2 = parts + if ext2 in ["o", "gch"]: + return base, f".{ext1}.{ext2}" + + if "." in name: + basename, ext = name.rsplit(".", 1) + return basename, f".{ext}" + + return name, "" + + +def count_common_path_segments(to_dir_parts, from_dir_parts): + """Count common path segments from the end.""" + common_segments = 0 + min_len = min(len(to_dir_parts), len(from_dir_parts)) + for i in range(min_len): + if to_dir_parts[-(i + 1)] == from_dir_parts[-(i + 1)]: + common_segments += 1 + else: + break + return common_segments + + +def match_bak_file(to_path, to_basename, from_resources, from_resources_index): + """Match .bak files to their original source.""" + matches = [] + source_path = to_path.parent / to_basename + source_path_str = "/" + str(source_path).replace("\\", "/") + match = pathmap.find_paths(source_path_str, from_resources_index) + if match and match.matched_path_length >= 2: + for resource_id in match.resource_ids: + from_resource = from_resources.get(id=resource_id) + if from_resource: + matches.append((from_resource, match.matched_path_length)) + return matches + + +def match_by_extension_mapping( + to_path, to_basename, possible_source_exts, from_resources, from_resources_index +): + """Match using extension mapping.""" + matches = [] + for source_ext in possible_source_exts: + source_path = to_path.parent / f"{to_basename}{source_ext}" + source_path_str = "/" + str(source_path).replace("\\", "/") + match = pathmap.find_paths(source_path_str, from_resources_index) + if match and match.matched_path_length >= 2: + for resource_id in match.resource_ids: + from_resource = from_resources.get(id=resource_id) + if from_resource: + matches.append((from_resource, match.matched_path_length)) + return matches + + +def match_by_exact_basename(to_path, to_basename, to_ext, from_resources): + """Match files with same basename but different extensions.""" + matches = [] + to_dir = to_path.parent + to_dir_parts = to_dir.parts if to_dir.parts else () + + is_compound_ext = "." in to_ext and to_ext.count(".") > 1 + base_ext = None + if is_compound_ext: + base_ext = to_ext.rsplit(".", 1)[0] + + for from_resource in from_resources: + from_path = Path(from_resource.path.lstrip("/")) + from_basename, from_ext = get_basename_without_extension(from_resource.path) + + if to_ext == ".bak": + from_full_name = from_path.name + if to_basename != from_full_name: + continue + else: + if from_basename != to_basename: + continue + + if is_compound_ext and base_ext and from_ext != base_ext: + continue + + from_dir = from_path.parent + from_dir_parts = from_dir.parts if from_dir.parts else () + common_segments = count_common_path_segments(to_dir_parts, from_dir_parts) + + if common_segments >= 2: + matches.append((from_resource, common_segments + 1)) + elif is_compound_ext and base_ext and from_ext == base_ext: + matches.append((from_resource, 1)) + + return matches + + +def find_basename_matches(to_resource, from_resources, from_resources_index): + """Find matches for to_resource based on basename matching.""" + to_path = Path(to_resource.path.lstrip("/")) + to_basename, to_ext = get_basename_without_extension(to_resource.path) + + if not to_basename: + return None + + possible_source_exts = BASENAME_EXTENSION_MAP.get(to_ext, []) + matches = [] + + if to_ext == ".bak": + matches.extend( + match_bak_file(to_path, to_basename, from_resources, from_resources_index) + ) + else: + if "." in to_ext and to_ext.count(".") > 1: + base_ext = to_ext.rsplit(".", 1)[0] + if base_ext not in possible_source_exts: + possible_source_exts.append(base_ext) + + matches.extend( + match_by_extension_mapping( + to_path, + to_basename, + possible_source_exts, + from_resources, + from_resources_index, + ) + ) + + matches.extend( + match_by_exact_basename(to_path, to_basename, to_ext, from_resources) + ) + + if not matches: + return None + + matches.sort(key=lambda x: x[1], reverse=True) + seen_resources = set() + unique_matches = [] + for resource, score in matches: + if resource.id not in seen_resources: + seen_resources.add(resource.id) + unique_matches.append(resource) + + return unique_matches if unique_matches else None + + def get_from_files_for_scanning(resources): """ Return resources in the "from/" side which has been mapped to the "to/" @@ -364,11 +532,41 @@ def map_jar_to_jvm_source(project, jvm_lang: jvm.JvmLanguage, logger=None): ) +def create_basename_relations(to_resource, basename_matches, diff_ratio_threshold): + """Create relations for basename matches.""" + for from_resource in basename_matches: + diff_ratio = get_resource_diff_ratio(to_resource, from_resource) + if diff_ratio is not None and diff_ratio < diff_ratio_threshold: + continue + + to_path_length = len(to_resource.path.split("/")) - 1 + extra_data = { + "path_score": f"basename/{to_path_length}", + "map_type_detail": "basename_match", + } + if diff_ratio: + extra_data["diff_ratio"] = f"{diff_ratio:.1%}" + + pipes.make_relation( + from_resource=from_resource, + to_resource=to_resource, + map_type="basename", + extra_data=extra_data, + ) + + def _map_path_resource( to_resource, from_resources, from_resources_index, diff_ratio_threshold=0.7 ): match = pathmap.find_paths(to_resource.path, from_resources_index) if not match: + basename_matches = find_basename_matches( + to_resource, from_resources, from_resources_index + ) + if basename_matches: + create_basename_relations( + to_resource, basename_matches, diff_ratio_threshold + ) return # Don't path map resource solely based on the file name. @@ -1155,10 +1353,7 @@ def map_javascript_path(project, logger=None): def _map_javascript_path_resource( to_resource, to_resources, from_resources_index, from_resources, map_type="js_path" ): - """ - Map JavaScript deployed files using their .map files. - Return the number of mapped files. - """ + """Map JavaScript deployed files using their .map files.""" path = Path(to_resource.path.lstrip("/")) basename_and_extension = js.get_js_map_basename_and_extension(path.name) @@ -1178,12 +1373,9 @@ def _map_javascript_path_resource( for source_ext in prospect.get("sources", []): match = pathmap.find_paths(f"{base_path}{source_ext}", from_resources_index) - # Only create relations when the number of matches if inferior or equal to - # the current number of path segment matched. if not match or len(match.resource_ids) > match.matched_path_length: continue - # Don't map resources solely based on their names. if match.matched_path_length <= 1: continue @@ -1192,6 +1384,17 @@ def _map_javascript_path_resource( from_resource = from_resources.get(id=match.resource_ids[0]) extra_data = {"path_score": f"{match.matched_path_length}/{path_parts_len}"} + if not from_resource: + basename_matches = find_basename_matches( + to_resource, from_resources, from_resources_index + ) + if basename_matches: + from_resource = basename_matches[0] + extra_data = { + "path_score": f"basename/{path_parts_len}", + "map_type_detail": "basename_match_fallback", + } + return js.map_related_files( to_resources, to_resource, diff --git a/scanpipe/tests/pipes/test_d2d.py b/scanpipe/tests/pipes/test_d2d.py index ffe55ef43e..baac6cfd33 100644 --- a/scanpipe/tests/pipes/test_d2d.py +++ b/scanpipe/tests/pipes/test_d2d.py @@ -756,6 +756,184 @@ def test_scanpipe_pipes_d2d_map_path(self): self.assertEqual({"path_score": "3/3"}, relation.extra_data) self.assertNotEqual("too-many-maps", file_name_too_many.status) + def test_scanpipe_pipes_d2d_get_basename_without_extension(self): + """Test basename extraction with various extension patterns.""" + basename, ext = d2d.get_basename_without_extension("file.js") + self.assertEqual(("file", ".js"), (basename, ext)) + + basename, ext = d2d.get_basename_without_extension("file.ts") + self.assertEqual(("file", ".ts"), (basename, ext)) + + basename, ext = d2d.get_basename_without_extension("file.d.ts") + self.assertEqual(("file", ".d.ts"), (basename, ext)) + + basename, ext = d2d.get_basename_without_extension("file.min.js") + self.assertEqual(("file", ".min.js"), (basename, ext)) + + basename, ext = d2d.get_basename_without_extension("file.d.mts") + self.assertEqual(("file", ".d.mts"), (basename, ext)) + + basename, ext = d2d.get_basename_without_extension("file.cpp.o") + self.assertEqual(("file", ".cpp.o"), (basename, ext)) + + basename, ext = d2d.get_basename_without_extension("file.h.gch") + self.assertEqual(("file", ".h.gch"), (basename, ext)) + + basename, ext = d2d.get_basename_without_extension("file") + self.assertEqual(("file", ""), (basename, ext)) + + basename, ext = d2d.get_basename_without_extension("path/to/file.cjs") + self.assertEqual(("file", ".cjs"), (basename, ext)) + + basename, ext = d2d.get_basename_without_extension("file.js.bak") + self.assertEqual(("file.js", ".bak"), (basename, ext)) + + basename, ext = d2d.get_basename_without_extension("file.ts.bak") + self.assertEqual(("file.ts", ".bak"), (basename, ext)) + + def test_scanpipe_pipes_d2d_map_path_with_basename_matching(self): + """Test path mapping with basename matching for npm files.""" + from1 = make_resource_file( + self.project1, + path="from/src/components/Button.ts", + ) + from2 = make_resource_file( + self.project1, + path="from/src/utils/helper.ts", + ) + from3 = make_resource_file( + self.project1, + path="from/src/styles/main.scss", + ) + + to1 = make_resource_file( + self.project1, + path="to/dist/components/Button.cjs", + ) + to2 = make_resource_file( + self.project1, + path="to/dist/utils/helper.mjs", + ) + to3 = make_resource_file( + self.project1, + path="to/dist/styles/main.css", + ) + + buffer = io.StringIO() + d2d.map_path(self.project1, logger=buffer.write) + + relations = self.project1.codebaserelations.filter(map_type="basename") + self.assertGreaterEqual(relations.count(), 3, "Should have basename matches") + + button_relation = relations.filter(to_resource=to1).first() + if button_relation: + self.assertEqual(from1, button_relation.from_resource) + self.assertEqual("basename", button_relation.map_type) + + helper_relation = relations.filter(to_resource=to2).first() + if helper_relation: + self.assertEqual(from2, helper_relation.from_resource) + + css_relation = relations.filter(to_resource=to3).first() + if css_relation: + self.assertEqual(from3, css_relation.from_resource) + + def test_scanpipe_pipes_d2d_map_path_with_basename_same_filename(self): + """Test basename matching for files with same filename but different hash.""" + from1 = make_resource_file( + self.project1, + path="from/src/components/Button.js", + ) + + to1 = make_resource_file( + self.project1, + path="to/dist/components/Button.min.js", + ) + + buffer = io.StringIO() + d2d.map_path(self.project1, logger=buffer.write) + + relations = self.project1.codebaserelations.filter(map_type="basename") + self.assertGreaterEqual(relations.count(), 1) + + relation = relations.filter(to_resource=to1).first() + if relation: + self.assertEqual(from1, relation.from_resource) + self.assertEqual("basename", relation.map_type) + + def test_scanpipe_pipes_d2d_map_path_with_compound_extensions(self): + """Test basename matching for compound extensions like .cpp.o -> .cpp.""" + make_resource_file( + self.project1, + path="from/src/nrlmsise_interface.cpp", + ) + make_resource_file( + self.project1, + path="from/src/headers.h", + ) + + to1 = make_resource_file( + self.project1, + path="to/build/nrlmsise_interface.cpp.o", + ) + to2 = make_resource_file( + self.project1, + path="to/build/headers.h.gch", + ) + + buffer = io.StringIO() + d2d.map_path(self.project1, logger=buffer.write) + + relations = self.project1.codebaserelations.filter(map_type="basename") + self.assertGreaterEqual( + relations.count(), 2, "Should have basename matches for compound extensions" + ) + + cpp_relation = relations.filter(to_resource=to1).first() + if cpp_relation: + self.assertEqual("basename", cpp_relation.map_type) + + h_relation = relations.filter(to_resource=to2).first() + if h_relation: + self.assertEqual("basename", h_relation.map_type) + + def test_scanpipe_pipes_d2d_map_path_with_bak_files(self): + """Test basename matching for backup files (.bak).""" + from1 = make_resource_file( + self.project1, + path="from/src/components/Button.js", + ) + from2 = make_resource_file( + self.project1, + path="from/src/utils/helper.ts", + ) + + to1 = make_resource_file( + self.project1, + path="to/dist/components/Button.js.bak", + ) + to2 = make_resource_file( + self.project1, + path="to/dist/utils/helper.ts.bak", + ) + + buffer = io.StringIO() + d2d.map_path(self.project1, logger=buffer.write) + + relations = self.project1.codebaserelations.filter(map_type="basename") + self.assertGreaterEqual( + relations.count(), 2, "Should have basename matches for .bak files" + ) + + button_relation = relations.filter(to_resource=to1).first() + if button_relation: + self.assertEqual(from1, button_relation.from_resource) + self.assertEqual("basename", button_relation.map_type) + + helper_relation = relations.filter(to_resource=to2).first() + if helper_relation: + self.assertEqual(from2, helper_relation.from_resource) + def test_scanpipe_pipes_d2d_find_java_packages(self): input_locations = [ self.data / "d2d" / "find_java_packages" / "Foo.java",