From 4d971903d2c96856a80a2776c91c8ef04e2470f4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 24 Oct 2023 13:59:46 -0400 Subject: [PATCH 01/19] ready for testing --- patchfinder/env.list | 4 +- patchfinder/src/main/java/FixFinderMain.java | 7 ++- .../main/java/fixes/parsers/FixParser.java | 3 ++ .../java/fixes/parsers/cxsecurityParser.java | 43 +++++++++++++++++++ .../urlfinders/VulnerabilityFixUrlFinder.java | 6 +++ 5 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 patchfinder/src/main/java/fixes/parsers/cxsecurityParser.java diff --git a/patchfinder/env.list b/patchfinder/env.list index 8db20f8eb..9d38e7a22 100644 --- a/patchfinder/env.list +++ b/patchfinder/env.list @@ -2,7 +2,7 @@ DB_TYPE=mysql HIKARI_URL=jdbc:mysql://localhost:3306/nvip?useSSL=false&allowPublicKeyRetrieval=true HIKARI_USER=root -HIKARI_PASSWORD=root +HIKARI_PASSWORD=Mack2626 # --- RABBITMQ --- RABBIT_POLL_INTERVAL=60 @@ -23,4 +23,4 @@ CLONE_PATH=nvip_data/patch-repos PATCH_SRC_URL_PATH=nvip_data/source_dict.json # --- FIX FINDER VARS --- -FF_INPUT_MODE=db \ No newline at end of file +FF_INPUT_MODE=dev \ No newline at end of file diff --git a/patchfinder/src/main/java/FixFinderMain.java b/patchfinder/src/main/java/FixFinderMain.java index 6b27af1db..88c05678c 100644 --- a/patchfinder/src/main/java/FixFinderMain.java +++ b/patchfinder/src/main/java/FixFinderMain.java @@ -85,9 +85,7 @@ private void runRabbit() { private void runDev() { // Manually enter CVEs for development List cveIds = new ArrayList<>(); - cveIds.add("CVE-2022-27911"); - cveIds.add("CVE-2023-30367"); - cveIds.add("CVE-2022-0847"); + cveIds.add("CVE-2023-3990"); try { FixFinder.run(cveIds); @@ -97,6 +95,7 @@ private void runDev() { } public static void main(String[] args) { -// run(); + FixFinderMain finder = new FixFinderMain(); + finder.start(); } } diff --git a/patchfinder/src/main/java/fixes/parsers/FixParser.java b/patchfinder/src/main/java/fixes/parsers/FixParser.java index d9fe64067..60e58aade 100644 --- a/patchfinder/src/main/java/fixes/parsers/FixParser.java +++ b/patchfinder/src/main/java/fixes/parsers/FixParser.java @@ -111,6 +111,9 @@ public static FixParser getParser(String cveId, String url) throws MalformedURLE case "access.redhat.com": parser = new RedhatParser(cveId, url); break; + case "cxsecurity.com": + parser = new cxsecurityParser(cveId, url); + break; default: parser = new GenericParser(cveId, url); break; diff --git a/patchfinder/src/main/java/fixes/parsers/cxsecurityParser.java b/patchfinder/src/main/java/fixes/parsers/cxsecurityParser.java new file mode 100644 index 000000000..ff9d687c5 --- /dev/null +++ b/patchfinder/src/main/java/fixes/parsers/cxsecurityParser.java @@ -0,0 +1,43 @@ +package fixes.parsers; + +import fixes.Fix; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +public class cxsecurityParser extends FixParser { + protected cxsecurityParser(String cveId, String url) { + super(cveId, url); + } + + @Override + protected List parseWebPage() throws IOException { + List fixSources = new ArrayList<>(); + + // Retrieve description + String description = String.valueOf(this.DOM.select("h6").first().text()); + + //retrieve references + Document doc = Jsoup.connect(url).get(); + + Elements references = this.DOM.select("table").last().select("td").select("div"); + for(Element row : references){ + String url = row.text(); + fixSources.add(url); + + } + + // For each URL, find the correct parser for it and add the fixes found for that URL + for(String fixSource : fixSources){ + FixParser parser = FixParser.getParser(cveId, fixSource); + this.fixes.addAll(parser.parse()); + } + + return this.fixes; + } + +} diff --git a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java index 091f9a095..fabc13e6a 100644 --- a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java +++ b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.HashSet; +import java.util.Objects; import java.util.Set; /** @@ -26,6 +27,11 @@ public ArrayList run(String cveId) throws IOException { // Test each source for a valid connection for (String source : sources) { + //case for cxsecurityFinder + if(Objects.equals(source, "https://cxsecurity.com/cvemap")){ + source = "https://cxsecurity.com/cveshow/".concat(cveId) ; + } + if (testConnection(source)) { urlList.add(source); } From ad1087426b522c071c7104c94226222cd4d4cdb7 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Wed, 25 Oct 2023 14:28:00 -0400 Subject: [PATCH 02/19] Renamed class to fix naming standards --- .../{cxsecurityParser.java => CXSecurityParser.java} | 7 ++----- patchfinder/src/main/java/fixes/parsers/FixParser.java | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) rename patchfinder/src/main/java/fixes/parsers/{cxsecurityParser.java => CXSecurityParser.java} (84%) diff --git a/patchfinder/src/main/java/fixes/parsers/cxsecurityParser.java b/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java similarity index 84% rename from patchfinder/src/main/java/fixes/parsers/cxsecurityParser.java rename to patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java index ff9d687c5..d077a0e61 100644 --- a/patchfinder/src/main/java/fixes/parsers/cxsecurityParser.java +++ b/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java @@ -9,8 +9,8 @@ import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -public class cxsecurityParser extends FixParser { - protected cxsecurityParser(String cveId, String url) { +public class CXSecurityParser extends FixParser { + protected CXSecurityParser(String cveId, String url) { super(cveId, url); } @@ -21,9 +21,6 @@ protected List parseWebPage() throws IOException { // Retrieve description String description = String.valueOf(this.DOM.select("h6").first().text()); - //retrieve references - Document doc = Jsoup.connect(url).get(); - Elements references = this.DOM.select("table").last().select("td").select("div"); for(Element row : references){ String url = row.text(); diff --git a/patchfinder/src/main/java/fixes/parsers/FixParser.java b/patchfinder/src/main/java/fixes/parsers/FixParser.java index 60e58aade..d16abc442 100644 --- a/patchfinder/src/main/java/fixes/parsers/FixParser.java +++ b/patchfinder/src/main/java/fixes/parsers/FixParser.java @@ -112,7 +112,7 @@ public static FixParser getParser(String cveId, String url) throws MalformedURLE parser = new RedhatParser(cveId, url); break; case "cxsecurity.com": - parser = new cxsecurityParser(cveId, url); + parser = new CXSecurityParser(cveId, url); break; default: parser = new GenericParser(cveId, url); From 5a73870df33cad18c70aa66d34a97b5fb34036b2 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Wed, 25 Oct 2023 14:30:54 -0400 Subject: [PATCH 03/19] Framed out tests for Fixfinder --- .../src/test/java/fixes/FixFinderTest.java | 14 +++++++++----- .../test/java/fixes/parsers/CISAParserTest.java | 15 +++++++++++++++ .../java/fixes/parsers/CXSecurityParserTest.java | 16 ++++++++++++++++ .../test/java/fixes/parsers/FixParserTest.java | 16 ++++++++++++++++ .../java/fixes/urlfinders/FixUrlFinderTest.java | 16 ++++++++++++++++ .../fixes/urlfinders/NvdFixUrlFinderTest.java | 12 ++++++++++++ 6 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 patchfinder/src/test/java/fixes/parsers/CISAParserTest.java create mode 100644 patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java create mode 100644 patchfinder/src/test/java/fixes/parsers/FixParserTest.java create mode 100644 patchfinder/src/test/java/fixes/urlfinders/FixUrlFinderTest.java create mode 100644 patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java diff --git a/patchfinder/src/test/java/fixes/FixFinderTest.java b/patchfinder/src/test/java/fixes/FixFinderTest.java index f12434aa7..f28f59db8 100644 --- a/patchfinder/src/test/java/fixes/FixFinderTest.java +++ b/patchfinder/src/test/java/fixes/FixFinderTest.java @@ -28,11 +28,6 @@ import org.junit.Before; import org.junit.Test; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import static org.junit.jupiter.api.Assertions.*; @@ -50,4 +45,13 @@ public void setUp() { FixFinderEnvVars.initializeEnvVars(true); } + @Test + public void testInit() { + // TODO: Test init + } + + @Test + public void testRun() { + // TODO: Test init + } } diff --git a/patchfinder/src/test/java/fixes/parsers/CISAParserTest.java b/patchfinder/src/test/java/fixes/parsers/CISAParserTest.java new file mode 100644 index 000000000..86bc1cedd --- /dev/null +++ b/patchfinder/src/test/java/fixes/parsers/CISAParserTest.java @@ -0,0 +1,15 @@ +package fixes.parsers; + +import org.junit.Test; + +public class CISAParserTest extends FixParserTest { + public CISAParserTest() { + // TODO: Initialize with test values + super(new CISAParser("", "")); + } + + @Override + public void testParseWebpage() { + // TODO: Test parseWebpage + } +} diff --git a/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java b/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java new file mode 100644 index 000000000..3926eb05a --- /dev/null +++ b/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java @@ -0,0 +1,16 @@ +package fixes.parsers; + +import org.junit.Test; + +public class CXSecurityParserTest extends FixParserTest { + public CXSecurityParserTest() { + // TODO: Initialize with test values + super(new CXSecurityParser("", "")); + } + + @Override + @Test + public void testParseWebpage() { + // TODO: Test parseWebpage + } +} diff --git a/patchfinder/src/test/java/fixes/parsers/FixParserTest.java b/patchfinder/src/test/java/fixes/parsers/FixParserTest.java new file mode 100644 index 000000000..2c86bd54c --- /dev/null +++ b/patchfinder/src/test/java/fixes/parsers/FixParserTest.java @@ -0,0 +1,16 @@ +package fixes.parsers; + +import env.FixFinderEnvVars; +import org.junit.Test; + +public abstract class FixParserTest { + final protected T fixParser; + + protected FixParserTest(T fixParser) { + this.fixParser = fixParser; + FixFinderEnvVars.initializeEnvVars(true); + } + + @Test + public abstract void testParseWebpage(); +} diff --git a/patchfinder/src/test/java/fixes/urlfinders/FixUrlFinderTest.java b/patchfinder/src/test/java/fixes/urlfinders/FixUrlFinderTest.java new file mode 100644 index 000000000..d860ad3f5 --- /dev/null +++ b/patchfinder/src/test/java/fixes/urlfinders/FixUrlFinderTest.java @@ -0,0 +1,16 @@ +package fixes.urlfinders; + +import env.FixFinderEnvVars; +import org.junit.Test; + +public abstract class FixUrlFinderTest { + final protected T fixUrlFinder; + + protected FixUrlFinderTest(T fixUrlFinder) { + this.fixUrlFinder = fixUrlFinder; + FixFinderEnvVars.initializeEnvVars(true); + } + + @Test + public abstract void testRun(); +} diff --git a/patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java b/patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java new file mode 100644 index 000000000..fd4c061cc --- /dev/null +++ b/patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java @@ -0,0 +1,12 @@ +package fixes.urlfinders; + +public class NvdFixUrlFinderTest extends FixUrlFinderTest { + public NvdFixUrlFinderTest() { + super(new NvdFixUrlFinder()); + } + + @Override + public void testRun() { + // TODO: Test run + } +} From da4042df486ab15a66ff6eb5b1c20c524000aabe Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Wed, 25 Oct 2023 16:13:27 -0400 Subject: [PATCH 04/19] Added FixParser tests --- .../src/test/java/fixes/parsers/FixParserTest.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/patchfinder/src/test/java/fixes/parsers/FixParserTest.java b/patchfinder/src/test/java/fixes/parsers/FixParserTest.java index 2c86bd54c..52082d2e1 100644 --- a/patchfinder/src/test/java/fixes/parsers/FixParserTest.java +++ b/patchfinder/src/test/java/fixes/parsers/FixParserTest.java @@ -13,4 +13,14 @@ protected FixParserTest(T fixParser) { @Test public abstract void testParseWebpage(); + + @Test + public void testParse() { + // TODO: Test parse + } + + @Test + public void testGetParser() { + // TODO: Test getParser + } } From 36642d681941b17e9f1f27ac95d6292a8099c2b7 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Wed, 25 Oct 2023 16:32:43 -0400 Subject: [PATCH 05/19] Improved test structure --- .../java/fixes/parsers/CISAParserTest.java | 7 ++++++- .../fixes/parsers/CXSecurityParserTest.java | 19 +++++++++++++++++-- .../java/fixes/parsers/FixParserTest.java | 12 +++++++++--- 3 files changed, 32 insertions(+), 6 deletions(-) diff --git a/patchfinder/src/test/java/fixes/parsers/CISAParserTest.java b/patchfinder/src/test/java/fixes/parsers/CISAParserTest.java index 86bc1cedd..db50f6440 100644 --- a/patchfinder/src/test/java/fixes/parsers/CISAParserTest.java +++ b/patchfinder/src/test/java/fixes/parsers/CISAParserTest.java @@ -5,7 +5,12 @@ public class CISAParserTest extends FixParserTest { public CISAParserTest() { // TODO: Initialize with test values - super(new CISAParser("", "")); + this.setFixParser(getNewParser("", "")); + } + + @Override + protected CISAParser getNewParser(String cveId, String url) { + return new CISAParser(cveId, url); } @Override diff --git a/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java b/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java index 3926eb05a..1dcdbdbe3 100644 --- a/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java +++ b/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java @@ -1,16 +1,31 @@ package fixes.parsers; +import fixes.Fix; import org.junit.Test; +import java.util.List; + public class CXSecurityParserTest extends FixParserTest { public CXSecurityParserTest() { // TODO: Initialize with test values - super(new CXSecurityParser("", "")); + this.setFixParser(getNewParser("", "")); + } + + @Override + protected CXSecurityParser getNewParser(String cveId, String url) { + return new CXSecurityParser(cveId, url); } @Override - @Test public void testParseWebpage() { // TODO: Test parseWebpage + final List fixes = this.fixParser().parse(); + } + + @Test + public void testParseWebpageNoFixes() { + // TODO: Test parseWebpage with second cve/url + this.setFixParser(getNewParser("", "")); + final List fixes = this.fixParser().parse(); } } diff --git a/patchfinder/src/test/java/fixes/parsers/FixParserTest.java b/patchfinder/src/test/java/fixes/parsers/FixParserTest.java index 52082d2e1..9247d3e64 100644 --- a/patchfinder/src/test/java/fixes/parsers/FixParserTest.java +++ b/patchfinder/src/test/java/fixes/parsers/FixParserTest.java @@ -4,13 +4,19 @@ import org.junit.Test; public abstract class FixParserTest { - final protected T fixParser; + private T fixParser; - protected FixParserTest(T fixParser) { - this.fixParser = fixParser; + protected FixParserTest() { +// this.fixParser = fixParser; FixFinderEnvVars.initializeEnvVars(true); } + public T fixParser() { return fixParser; } + + public void setFixParser(T fixParser) { this.fixParser = fixParser; } + + protected abstract T getNewParser(String cveId, String url); + @Test public abstract void testParseWebpage(); From 9c35f28333567180a76df9fc5a656dd4f914cd5d Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 31 Oct 2023 12:44:21 -0400 Subject: [PATCH 06/19] Test suite completed for CXSecurityParser --- patchfinder/src/main/java/FixFinderMain.java | 2 +- .../java/fixes/parsers/CXSecurityParser.java | 1 - .../fixes/parsers/CXSecurityParserTest.java | 21 ++++++++++++++----- .../java/fixes/parsers/FixParserTest.java | 3 ++- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/patchfinder/src/main/java/FixFinderMain.java b/patchfinder/src/main/java/FixFinderMain.java index 88c05678c..bf2ac9517 100644 --- a/patchfinder/src/main/java/FixFinderMain.java +++ b/patchfinder/src/main/java/FixFinderMain.java @@ -85,7 +85,7 @@ private void runRabbit() { private void runDev() { // Manually enter CVEs for development List cveIds = new ArrayList<>(); - cveIds.add("CVE-2023-3990"); + cveIds.add("CVE-2023-38571"); try { FixFinder.run(cveIds); diff --git a/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java b/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java index d077a0e61..02fb48108 100644 --- a/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java +++ b/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java @@ -33,7 +33,6 @@ protected List parseWebPage() throws IOException { FixParser parser = FixParser.getParser(cveId, fixSource); this.fixes.addAll(parser.parse()); } - return this.fixes; } diff --git a/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java b/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java index 1dcdbdbe3..929e1e542 100644 --- a/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java +++ b/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java @@ -1,14 +1,19 @@ package fixes.parsers; import fixes.Fix; +import org.jsoup.Jsoup; import org.junit.Test; +import java.io.IOException; +import java.net.URL; +import java.util.ArrayList; import java.util.List; +import static org.junit.Assert.assertEquals; public class CXSecurityParserTest extends FixParserTest { public CXSecurityParserTest() { // TODO: Initialize with test values - this.setFixParser(getNewParser("", "")); +// this.setFixParser(getNewParser("", "")); } @Override @@ -17,15 +22,21 @@ protected CXSecurityParser getNewParser(String cveId, String url) { } @Override - public void testParseWebpage() { + //zero fixes are found + public void testParseWebpage() throws IOException { // TODO: Test parseWebpage - final List fixes = this.fixParser().parse(); } @Test public void testParseWebpageNoFixes() { // TODO: Test parseWebpage with second cve/url - this.setFixParser(getNewParser("", "")); - final List fixes = this.fixParser().parse(); + String cveId ="CVE-2023-3990"; + String url ="https://cxsecurity.com/cveshow/CVE-2023-3990"; + this.setFixParser(getNewParser(cveId, url)); + + List actual = this.fixParser().parse(); + List expected = new ArrayList<>(); + + assertEquals(expected, actual); } } diff --git a/patchfinder/src/test/java/fixes/parsers/FixParserTest.java b/patchfinder/src/test/java/fixes/parsers/FixParserTest.java index 9247d3e64..b3237934f 100644 --- a/patchfinder/src/test/java/fixes/parsers/FixParserTest.java +++ b/patchfinder/src/test/java/fixes/parsers/FixParserTest.java @@ -3,6 +3,7 @@ import env.FixFinderEnvVars; import org.junit.Test; +import java.io.IOException; public abstract class FixParserTest { private T fixParser; @@ -18,7 +19,7 @@ protected FixParserTest() { protected abstract T getNewParser(String cveId, String url); @Test - public abstract void testParseWebpage(); + public abstract void testParseWebpage() throws IOException; @Test public void testParse() { From 58b628d4593a1e1f3bff5778b2f7f325233e66b6 Mon Sep 17 00:00:00 2001 From: mcw7034 <89413825+mcw7034@users.noreply.github.com> Date: Tue, 31 Oct 2023 14:12:57 -0400 Subject: [PATCH 07/19] Update env.list --- patchfinder/env.list | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/patchfinder/env.list b/patchfinder/env.list index 9d38e7a22..1bca8eec6 100644 --- a/patchfinder/env.list +++ b/patchfinder/env.list @@ -2,7 +2,7 @@ DB_TYPE=mysql HIKARI_URL=jdbc:mysql://localhost:3306/nvip?useSSL=false&allowPublicKeyRetrieval=true HIKARI_USER=root -HIKARI_PASSWORD=Mack2626 +HIKARI_PASSWORD=root # --- RABBITMQ --- RABBIT_POLL_INTERVAL=60 @@ -23,4 +23,4 @@ CLONE_PATH=nvip_data/patch-repos PATCH_SRC_URL_PATH=nvip_data/source_dict.json # --- FIX FINDER VARS --- -FF_INPUT_MODE=dev \ No newline at end of file +FF_INPUT_MODE=dev From bec42a1325bb1f6e612e712e0f2ce3a5cb57ded5 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Wed, 1 Nov 2023 12:48:11 -0400 Subject: [PATCH 08/19] Bugfixes --- .../java/fixes/parsers/GenericParser.java | 28 +++++++++++++++++-- .../fixes/parsers/CXSecurityParserTest.java | 2 +- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/patchfinder/src/main/java/fixes/parsers/GenericParser.java b/patchfinder/src/main/java/fixes/parsers/GenericParser.java index 3f4d79459..0b2ea8fb7 100644 --- a/patchfinder/src/main/java/fixes/parsers/GenericParser.java +++ b/patchfinder/src/main/java/fixes/parsers/GenericParser.java @@ -29,6 +29,7 @@ import org.jsoup.select.Elements; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; @@ -50,6 +51,7 @@ private enum FIX_WORDS { MITIGATION, RESOLVE, RESOLUTION; +// COUNTERMEASURE; /** * Determines if given word is a valid member of this enum (case-insensitive). @@ -58,10 +60,27 @@ private enum FIX_WORDS { * @return whether the word is a valid member of this enum */ public static boolean hasWord(String word) { + word = word.toUpperCase(); try { - FIX_WORDS.valueOf(word.toUpperCase()); + FIX_WORDS.valueOf(word); return true; - } catch (Exception ignored) { return false; } + } catch (Exception ignored) { + // If no direct match, check if word is plural and try singular form + final boolean endsWithES = word.endsWith("ES"); + final boolean endsWithS = endsWithES || word.endsWith("S"); + + if(endsWithES || endsWithS) { + final int endIndex = endsWithES ? 2 : 1; + final String trimmedWord = word.substring(0, endIndex); + try { + FIX_WORDS.valueOf(trimmedWord); + return true; + } catch (Exception ignored1) { } + } + + // Return false if no match + return false; + } } } @@ -84,8 +103,11 @@ protected List parseWebPage() { // Iterate over header objects for (Element e : headerElements) { + // Check text and id of header for keywords + final List words = new ArrayList<>(Arrays.asList(e.text().split(" "))); + words.add(e.id()); // Split text on spaces and check each word. - for (String headerWord : e.text().split(" ")) { + for (String headerWord : words) { // Check if word is a member of FIX_WORDS (case-insensitive) if(FIX_WORDS.hasWord(headerWord)) { // Find and store description elements related to the current header diff --git a/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java b/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java index 929e1e542..c4323e408 100644 --- a/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java +++ b/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java @@ -23,7 +23,7 @@ protected CXSecurityParser getNewParser(String cveId, String url) { @Override //zero fixes are found - public void testParseWebpage() throws IOException { + public void testParseWebpage() { // TODO: Test parseWebpage } From 56d475cadb644255f1c5668df302fb4ff4888132 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Thu, 2 Nov 2023 15:31:35 -0400 Subject: [PATCH 09/19] Structural changes and TODOs for reclassification of several Parsers. This should straighten out the structure and make testing much more effective --- patchfinder/pom.xml | 4 +- .../src/main/java/fixes/FixFinderThread.java | 13 +--- .../src/main/java/fixes/FixProcessor.java | 19 ++++++ .../main/java/fixes/parsers/CISAParser.java | 2 +- .../java/fixes/parsers/CXSecurityParser.java | 3 +- .../main/java/fixes/parsers/FixParser.java | 17 +++-- .../main/java/fixes/parsers/NVDParser.java | 3 +- .../fixes/parsers/RedhatBugzillaParser.java | 2 +- .../main/java/fixes/parsers/RedhatParser.java | 4 +- .../fixes/parsers/RedhatSecurityParser.java | 2 +- .../java/fixes/urlfinders/FixUrlFinder.java | 32 +++++++--- .../fixes/urlfinders/NvdFixUrlFinder.java | 64 +++++++++++++++---- .../urlfinders/VulnerabilityFixUrlFinder.java | 48 +++++++------- 13 files changed, 142 insertions(+), 71 deletions(-) create mode 100644 patchfinder/src/main/java/fixes/FixProcessor.java diff --git a/patchfinder/pom.xml b/patchfinder/pom.xml index b389a77a4..fbd42aff0 100644 --- a/patchfinder/pom.xml +++ b/patchfinder/pom.xml @@ -9,8 +9,8 @@ 1.0 - 1.8 - 1.8 + 17 + 17 UTF-8 diff --git a/patchfinder/src/main/java/fixes/FixFinderThread.java b/patchfinder/src/main/java/fixes/FixFinderThread.java index 0b3ad03af..47b95f25b 100644 --- a/patchfinder/src/main/java/fixes/FixFinderThread.java +++ b/patchfinder/src/main/java/fixes/FixFinderThread.java @@ -83,17 +83,8 @@ public void run() { for (String url : urls) { CompletableFuture> future = CompletableFuture.supplyAsync(() -> { - - - try{ - FixParser parser = FixParser.getParser(cveId, url); - return parser.parse(); - } catch(IOException e){ - logger.error("Error occurred while parsing url {} for CVE {}: {}", url, cveId, e.toString()); - e.printStackTrace(); - return null; - } - + FixParser parser = FixParser.getParser(cveId, url); + return parser.parse(); }); futures.add(future); diff --git a/patchfinder/src/main/java/fixes/FixProcessor.java b/patchfinder/src/main/java/fixes/FixProcessor.java new file mode 100644 index 000000000..cf1a68bac --- /dev/null +++ b/patchfinder/src/main/java/fixes/FixProcessor.java @@ -0,0 +1,19 @@ +package fixes; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + +import java.io.IOException; +import java.net.URL; + +public abstract class FixProcessor { + // Logger instance for FixProcessors + protected static final Logger logger = LogManager.getLogger(); + + // Utility method for getting DOM from string URL, throws IOException in case of an error + protected Document getDOM(String url) throws IOException { + return Jsoup.parse(new URL(url), 10000); + } +} diff --git a/patchfinder/src/main/java/fixes/parsers/CISAParser.java b/patchfinder/src/main/java/fixes/parsers/CISAParser.java index 6ae767423..eea70ca3f 100644 --- a/patchfinder/src/main/java/fixes/parsers/CISAParser.java +++ b/patchfinder/src/main/java/fixes/parsers/CISAParser.java @@ -47,7 +47,7 @@ protected CISAParser(String cveId, String url){ } @Override - protected List parseWebPage() throws IOException { + protected List parseWebPage() { Elements headers = this.DOM.select("div[id=1-full__main]").first().select("h"); return this.fixes; diff --git a/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java b/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java index 02fb48108..5ba090fe6 100644 --- a/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java +++ b/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java @@ -15,7 +15,7 @@ protected CXSecurityParser(String cveId, String url) { } @Override - protected List parseWebPage() throws IOException { + protected List parseWebPage() { List fixSources = new ArrayList<>(); // Retrieve description @@ -28,6 +28,7 @@ protected List parseWebPage() throws IOException { } + // TODO: Remove when class is migrated to type UrlParser // For each URL, find the correct parser for it and add the fixes found for that URL for(String fixSource : fixSources){ FixParser parser = FixParser.getParser(cveId, fixSource); diff --git a/patchfinder/src/main/java/fixes/parsers/FixParser.java b/patchfinder/src/main/java/fixes/parsers/FixParser.java index d16abc442..8c4e529ad 100644 --- a/patchfinder/src/main/java/fixes/parsers/FixParser.java +++ b/patchfinder/src/main/java/fixes/parsers/FixParser.java @@ -25,6 +25,7 @@ */ import fixes.Fix; +import fixes.FixProcessor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jsoup.Jsoup; @@ -42,8 +43,7 @@ * @author Paul Vickers * @author Dylan Mulligan */ -public abstract class FixParser { - protected final static Logger logger = LogManager.getLogger(); +public abstract class FixParser extends FixProcessor { protected final String cveId; protected final String url; @@ -62,7 +62,7 @@ public List parse() { // Attempt to parse page and store returned Document object try { logger.info("{} is parsing url '{}'...", getClass().getSimpleName(), url); - this.DOM = Jsoup.parse(new URL(url), 10000); + this.DOM = this.getDOM(this.url); // Call abstract method implementation based on instance this.parseWebPage(); } @@ -81,7 +81,7 @@ public List parse() { //TODO: Remove this throws unless we really need it, as URL interaction has been // moved to parse() and the IOExceptions are handled there - protected abstract List parseWebPage() throws IOException; + protected abstract List parseWebPage(); /** * Delegation method to determine which parser should be used to find fixes from the given url. @@ -91,9 +91,14 @@ public List parse() { * @return Correct parser to be used * */ - public static FixParser getParser(String cveId, String url) throws MalformedURLException { + public static FixParser getParser(String cveId, String url) { // Objectify url for domain extraction - final URL urlObj = new URL(url); + URL urlObj = null; + try { urlObj = new URL(url); } + catch (Exception e) { + // This should not happen, as URL has already been validated + logger.error("Fatal error occurred: {}", e.toString()); + } // Extract domain final String domain = urlObj.getHost(); diff --git a/patchfinder/src/main/java/fixes/parsers/NVDParser.java b/patchfinder/src/main/java/fixes/parsers/NVDParser.java index f4d2b60f5..624b07993 100644 --- a/patchfinder/src/main/java/fixes/parsers/NVDParser.java +++ b/patchfinder/src/main/java/fixes/parsers/NVDParser.java @@ -78,10 +78,9 @@ protected NVDParser(String cveId, String url){ * scrape for the references table and then delegate to other parsers for those sources. * * @return List of fixes for the CVE - * @throws IOException if an error occurs during scraping */ @Override - public List parseWebPage() throws IOException{ + public List parseWebPage() { // Isolate the HTML for the references table Elements rows = this.DOM.select("div[id=vulnHyperlinksPanel]").first().select("table").first().select("tbody").select("tr"); diff --git a/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java b/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java index a86988e62..c7336c601 100644 --- a/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java +++ b/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java @@ -13,7 +13,7 @@ protected RedhatBugzillaParser(String cveId, String url){ @Override - protected List parseWebPage() throws IOException { + protected List parseWebPage() { List newFixes = new ArrayList<>(); // TODO: Add Bugzilla specific implementation diff --git a/patchfinder/src/main/java/fixes/parsers/RedhatParser.java b/patchfinder/src/main/java/fixes/parsers/RedhatParser.java index 80ee0f993..6a8952bf8 100644 --- a/patchfinder/src/main/java/fixes/parsers/RedhatParser.java +++ b/patchfinder/src/main/java/fixes/parsers/RedhatParser.java @@ -40,7 +40,7 @@ protected RedhatParser(String cveId, String url){ super(cveId, url); } - protected List parseWebPage() throws IOException{ + protected List parseWebPage() { throw new UnsupportedOperationException(); } @@ -48,7 +48,7 @@ protected List parseWebPage() throws IOException{ * Delegates and parses the specified webpage using the RedHat Sub classes * @return list of all found fixes */ - @Override + @Override // TODO: Migrate to UrlFinder and make use of the new methods in FixProcessor/FixUrlFinder public List parse(){ // Init fixes list this.fixes = new ArrayList<>(); diff --git a/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java b/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java index c32298859..c73ba0b60 100644 --- a/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java +++ b/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java @@ -35,7 +35,7 @@ protected RedhatSecurityParser(String cveId, String url){ } @Override - protected List parseWebPage() throws IOException { + protected List parseWebPage() { return null; } } diff --git a/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java index 74ff8d1f1..8c3ff31e9 100644 --- a/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java +++ b/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java @@ -24,6 +24,7 @@ * SOFTWARE. */ +import fixes.FixProcessor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -31,6 +32,7 @@ import java.net.HttpURLConnection; import java.net.URL; import java.util.ArrayList; +import java.util.List; /** * Abstract class responsible for finding possible fix source URLs for the FixFinder. @@ -38,21 +40,31 @@ * @author Dylan Mulligan */ -public abstract class FixUrlFinder { +public abstract class FixUrlFinder extends FixProcessor { + // To be implemented in child classes, houses the actual logic that selects source urls + public abstract List getUrls(String cveId) throws IOException; - protected static final Logger logger = LogManager.getLogger(FixUrlFinder.class.getName()); - - public abstract ArrayList run(String cveId) throws IOException; + //Called for all child instances, makes use of their specific implementation of + // getUrls(), then tests and filters out any urls that can't be connected to + public List run(String cveId) { + try { + final List urls = this.getUrls(cveId); + // Test each source for a valid connection and filter failed connections + return urls.stream().filter(FixUrlFinder::testConnection).toList(); + } catch (IOException e) { + logger.error("Failed to get urls for CVE '{}': {}", cveId, e.toString()); + return new ArrayList<>(); + } + } - protected static boolean testConnection(String address) throws IOException { + // Tests the connection of a given address and returns the boolean result of the test + protected static boolean testConnection(String address) { logger.info("Testing Connection for address: " + address); - URL url = new URL(address); - HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); - int response; - try { - response = urlConnection.getResponseCode(); + URL url = new URL(address); + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + final int response = urlConnection.getResponseCode(); // Don't print OK responses, only when this is not the case if(response != 200) logger.info("Response Code: " + response); return true; diff --git a/patchfinder/src/main/java/fixes/urlfinders/NvdFixUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/NvdFixUrlFinder.java index 4bf75c3fa..58316fe8e 100644 --- a/patchfinder/src/main/java/fixes/urlfinders/NvdFixUrlFinder.java +++ b/patchfinder/src/main/java/fixes/urlfinders/NvdFixUrlFinder.java @@ -25,9 +25,13 @@ */ import fixes.FixFinder; +import fixes.parsers.NVDParser; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; import java.io.IOException; import java.util.ArrayList; +import java.util.List; /** * Implementation of FixUrlFinder for CVEs collected from NVD @@ -36,30 +40,66 @@ */ public class NvdFixUrlFinder extends FixUrlFinder { + private enum RESOURCE_TAGS { + PATCH("Patch"), // Hyperlink relates directly to patch information + VENDOR_ADVISORY("Vendor Advisory"), // Hyperlink relates to an advisory host + THIRD_PARTY_ADVISORY("Third Party Advisory"), // Hyperlink relates to a third-party advisory host + EXPLOIT("Exploit"), // Hyperlink relates to exploit information + ISSUE_TRACKING("Issue Tracking"); // Hyperlink relates to an issue tracking host + + private final String name; + RESOURCE_TAGS(String name) { + this.name = name; + } + + /** + * Safe valueOf method that relates tag name (i.e. "Vendor Advisory") to the correct member + * @param name name of resource tag + * @return correlated tag object, or null if not found + */ + public static RESOURCE_TAGS fromString(String name) { + for(RESOURCE_TAGS tag : RESOURCE_TAGS.values()) { + if(tag.name.equalsIgnoreCase(name)) return tag; + } + return null; + } + } + public NvdFixUrlFinder() { } @Override - public ArrayList run(String cveId) throws IOException { + public ArrayList getUrls(String cveId) throws IOException { logger.info("Getting fixes for CVE: {}", cveId); - ArrayList urlList = new ArrayList<>(); // Get all sources for the cve - ArrayList sources = FixFinder.getDatabaseHelper().getCveSourcesNVD(cveId); - - // Test each source for a valid connection - for (String source : sources) { - // Test reported source - if (testConnection(source)) { - urlList.add(source); - } - } + ArrayList urlList = FixFinder.getDatabaseHelper().getCveSourcesNVD(cveId); // Test NVD direct cve page final String directSource = "https://nvd.nist.gov/vuln/detail/" + cveId; if(testConnection(directSource)) { - urlList.add(directSource); + try { urlList.addAll(this.scrapeReferences(directSource)); } + catch (IOException e) { logger.warn("Failed to scrape references from NVD page: {}", e.toString()); } } return urlList; } + + private List scrapeReferences(String url) throws IOException { + // Isolate the HTML for the references table + Elements rows = this.getDOM(url).select("div[id=vulnHyperlinksPanel]").first().select("table").first().select("tbody").select("tr"); + + // For each URL stored in the table, if it has a "Patch" badge associated with it, add it to fixSources + List fixSources = new ArrayList<>(); + for(Element row : rows){ + String refUrl = row.select("a").text(); + Elements spans = row.select("span.badge"); + // Check all resource tags + for(Element span: spans){ + // Add url if the tag matches any whitelisted tag + if(RESOURCE_TAGS.fromString(span.text()) != null) fixSources.add(refUrl); + } + } + + return fixSources; + } } \ No newline at end of file diff --git a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java index fabc13e6a..c91cc82f9 100644 --- a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java +++ b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java @@ -3,10 +3,7 @@ import fixes.FixFinder; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Objects; -import java.util.Set; +import java.util.*; /** * Implementation of FixUrlFinder for CVEs collected from the NVIP Crawler @@ -18,24 +15,31 @@ public class VulnerabilityFixUrlFinder extends FixUrlFinder { public VulnerabilityFixUrlFinder() { } @Override - public ArrayList run(String cveId) throws IOException { - logger.info("Getting fixes for CVE: {}", cveId); - ArrayList urlList = new ArrayList<>(); - + public List getUrls(String cveId) throws IOException { // Get all sources for the cve - Set sources = new HashSet<>(FixFinder.getDatabaseHelper().getSpecificCveSources(cveId)); - - // Test each source for a valid connection - for (String source : sources) { - //case for cxsecurityFinder - if(Objects.equals(source, "https://cxsecurity.com/cvemap")){ - source = "https://cxsecurity.com/cveshow/".concat(cveId) ; - } - - if (testConnection(source)) { - urlList.add(source); - } - } - return urlList; + return new HashSet<>(FixFinder.getDatabaseHelper().getSpecificCveSources(cveId)).stream().toList(); } + +// @Override +// public ArrayList run(String cveId) { +// logger.info("Getting fixes for CVE: {}", cveId); +// ArrayList urlList = new ArrayList<>(); +// +// // Get all sources for the cve +// Set sources = new HashSet<>(FixFinder.getDatabaseHelper().getSpecificCveSources(cveId)); +// +// // TODO: Move to CXSecurityUrlFinder +// // Test each source for a valid connection +// for (String source : sources) { +// //case for cxsecurityFinder +// if(Objects.equals(source, "https://cxsecurity.com/cvemap")){ +// source = "https://cxsecurity.com/cveshow/".concat(cveId) ; +// } +// +// if (testConnection(source)) { +// urlList.add(source); +// } +// } +// return urlList; +// } } \ No newline at end of file From eceaf10a7134f6e99b17739f41dd864f3b99c1f5 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Thu, 2 Nov 2023 15:39:55 -0400 Subject: [PATCH 10/19] Disabled deprecated NVDParser --- patchfinder/src/main/java/fixes/parsers/FixParser.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/patchfinder/src/main/java/fixes/parsers/FixParser.java b/patchfinder/src/main/java/fixes/parsers/FixParser.java index 8c4e529ad..c59eef72a 100644 --- a/patchfinder/src/main/java/fixes/parsers/FixParser.java +++ b/patchfinder/src/main/java/fixes/parsers/FixParser.java @@ -107,9 +107,9 @@ public static FixParser getParser(String cveId, String url) { // Choose parser based on domain switch (domain) { - case "nvd.nist.gov": - parser = new NVDParser(cveId, url); - break; +// case "nvd.nist.gov": +// parser = new NVDParser(cveId, url); +// break; case "cisa.gov": parser = new CISAParser(cveId, url); break; From 91241323b4646f18ad617077abac3c929fee1e2f Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Thu, 2 Nov 2023 15:40:42 -0400 Subject: [PATCH 11/19] Fixed naming structure --- patchfinder/src/main/java/fixes/FixFinder.java | 8 ++++---- .../{NvdFixUrlFinder.java => NvdUrlFinder.java} | 5 ++--- ...ilityFixUrlFinder.java => VulnerabilityUrlFinder.java} | 4 ++-- .../test/java/fixes/urlfinders/NvdFixUrlFinderTest.java | 4 ++-- 4 files changed, 10 insertions(+), 11 deletions(-) rename patchfinder/src/main/java/fixes/urlfinders/{NvdFixUrlFinder.java => NvdUrlFinder.java} (97%) rename patchfinder/src/main/java/fixes/urlfinders/{VulnerabilityFixUrlFinder.java => VulnerabilityUrlFinder.java} (92%) diff --git a/patchfinder/src/main/java/fixes/FixFinder.java b/patchfinder/src/main/java/fixes/FixFinder.java index baba7dbd9..66d995a94 100644 --- a/patchfinder/src/main/java/fixes/FixFinder.java +++ b/patchfinder/src/main/java/fixes/FixFinder.java @@ -28,8 +28,8 @@ import env.FixFinderEnvVars; import db.DatabaseHelper; import fixes.urlfinders.FixUrlFinder; -import fixes.urlfinders.NvdFixUrlFinder; -import fixes.urlfinders.VulnerabilityFixUrlFinder; +import fixes.urlfinders.NvdUrlFinder; +import fixes.urlfinders.VulnerabilityUrlFinder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -76,8 +76,8 @@ public static void init() { logger.info("Initializing FixUrlFinders..."); // Add the instances to the fixURLFinders list - fixURLFinders.add(new VulnerabilityFixUrlFinder()); - fixURLFinders.add(new NvdFixUrlFinder()); + fixURLFinders.add(new VulnerabilityUrlFinder()); + fixURLFinders.add(new NvdUrlFinder()); logger.info("Done initializing {} FixUrlFinders: {}", fixURLFinders.size(), fixURLFinders); } diff --git a/patchfinder/src/main/java/fixes/urlfinders/NvdFixUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java similarity index 97% rename from patchfinder/src/main/java/fixes/urlfinders/NvdFixUrlFinder.java rename to patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java index 58316fe8e..e03c2a43d 100644 --- a/patchfinder/src/main/java/fixes/urlfinders/NvdFixUrlFinder.java +++ b/patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java @@ -25,7 +25,6 @@ */ import fixes.FixFinder; -import fixes.parsers.NVDParser; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; @@ -38,7 +37,7 @@ * * @author Richard Sawh */ -public class NvdFixUrlFinder extends FixUrlFinder { +public class NvdUrlFinder extends FixUrlFinder { private enum RESOURCE_TAGS { PATCH("Patch"), // Hyperlink relates directly to patch information @@ -65,7 +64,7 @@ public static RESOURCE_TAGS fromString(String name) { } } - public NvdFixUrlFinder() { } + public NvdUrlFinder() { } @Override public ArrayList getUrls(String cveId) throws IOException { diff --git a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityUrlFinder.java similarity index 92% rename from patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java rename to patchfinder/src/main/java/fixes/urlfinders/VulnerabilityUrlFinder.java index c91cc82f9..e3d7e6ef1 100644 --- a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java +++ b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityUrlFinder.java @@ -10,9 +10,9 @@ * * @author Richard Sawh */ -public class VulnerabilityFixUrlFinder extends FixUrlFinder { +public class VulnerabilityUrlFinder extends FixUrlFinder { - public VulnerabilityFixUrlFinder() { } + public VulnerabilityUrlFinder() { } @Override public List getUrls(String cveId) throws IOException { diff --git a/patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java b/patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java index fd4c061cc..f0a820010 100644 --- a/patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java +++ b/patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java @@ -1,8 +1,8 @@ package fixes.urlfinders; -public class NvdFixUrlFinderTest extends FixUrlFinderTest { +public class NvdFixUrlFinderTest extends FixUrlFinderTest { public NvdFixUrlFinderTest() { - super(new NvdFixUrlFinder()); + super(new NvdUrlFinder()); } @Override From e22d36b7f0672cef7831f05c74eed81914d0ef64 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Thu, 2 Nov 2023 15:53:27 -0400 Subject: [PATCH 12/19] Removed unused imports --- patchfinder/src/main/java/fixes/FixFinderThread.java | 4 ---- patchfinder/src/main/java/fixes/parsers/CISAParser.java | 4 ---- .../src/main/java/fixes/parsers/CXSecurityParser.java | 6 ++---- patchfinder/src/main/java/fixes/parsers/FixParser.java | 4 ---- patchfinder/src/main/java/fixes/parsers/GenericParser.java | 2 +- patchfinder/src/main/java/fixes/parsers/NVDParser.java | 5 ----- .../src/main/java/fixes/parsers/RedhatBugzillaParser.java | 1 - .../src/main/java/fixes/parsers/RedhatSecurityParser.java | 1 - .../src/main/java/fixes/urlfinders/FixUrlFinder.java | 2 -- 9 files changed, 3 insertions(+), 26 deletions(-) diff --git a/patchfinder/src/main/java/fixes/FixFinderThread.java b/patchfinder/src/main/java/fixes/FixFinderThread.java index 47b95f25b..8a8847050 100644 --- a/patchfinder/src/main/java/fixes/FixFinderThread.java +++ b/patchfinder/src/main/java/fixes/FixFinderThread.java @@ -25,13 +25,9 @@ */ import fixes.parsers.FixParser; -import fixes.parsers.CISAParser; -import fixes.parsers.GenericParser; -import fixes.parsers.NVDParser; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; diff --git a/patchfinder/src/main/java/fixes/parsers/CISAParser.java b/patchfinder/src/main/java/fixes/parsers/CISAParser.java index eea70ca3f..2dd581c67 100644 --- a/patchfinder/src/main/java/fixes/parsers/CISAParser.java +++ b/patchfinder/src/main/java/fixes/parsers/CISAParser.java @@ -25,12 +25,8 @@ */ import fixes.Fix; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; import org.jsoup.select.Elements; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; /** diff --git a/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java b/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java index 5ba090fe6..4d8b546c1 100644 --- a/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java +++ b/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java @@ -2,11 +2,9 @@ import fixes.Fix; -import java.io.IOException; import java.util.ArrayList; import java.util.List; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; + import org.jsoup.nodes.Element; import org.jsoup.select.Elements; public class CXSecurityParser extends FixParser { @@ -19,7 +17,7 @@ protected List parseWebPage() { List fixSources = new ArrayList<>(); // Retrieve description - String description = String.valueOf(this.DOM.select("h6").first().text()); + String description = this.DOM.select("h6").first().text(); Elements references = this.DOM.select("table").last().select("td").select("div"); for(Element row : references){ diff --git a/patchfinder/src/main/java/fixes/parsers/FixParser.java b/patchfinder/src/main/java/fixes/parsers/FixParser.java index c59eef72a..cb1b73a10 100644 --- a/patchfinder/src/main/java/fixes/parsers/FixParser.java +++ b/patchfinder/src/main/java/fixes/parsers/FixParser.java @@ -26,13 +26,9 @@ import fixes.Fix; import fixes.FixProcessor; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.List; diff --git a/patchfinder/src/main/java/fixes/parsers/GenericParser.java b/patchfinder/src/main/java/fixes/parsers/GenericParser.java index 0b2ea8fb7..f15cb2cae 100644 --- a/patchfinder/src/main/java/fixes/parsers/GenericParser.java +++ b/patchfinder/src/main/java/fixes/parsers/GenericParser.java @@ -121,7 +121,7 @@ protected List parseWebPage() { // If data was found, store in a new Fix object and add to list of found fixes if(fixDescription.length() > 0) - this.fixes.add(new Fix(cveId, fixDescription.toString(), url)); + this.fixes.add(new Fix(cveId, fixDescription, url)); // Skip to next header break; diff --git a/patchfinder/src/main/java/fixes/parsers/NVDParser.java b/patchfinder/src/main/java/fixes/parsers/NVDParser.java index 624b07993..1f862d508 100644 --- a/patchfinder/src/main/java/fixes/parsers/NVDParser.java +++ b/patchfinder/src/main/java/fixes/parsers/NVDParser.java @@ -25,14 +25,9 @@ */ import fixes.Fix; -import fixes.FixFinderThread; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import java.io.IOException; -import java.net.URL; import java.util.ArrayList; import java.util.List; diff --git a/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java b/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java index c7336c601..b4aefa897 100644 --- a/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java +++ b/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java @@ -2,7 +2,6 @@ import fixes.Fix; -import java.io.IOException; import java.util.ArrayList; import java.util.List; diff --git a/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java b/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java index c73ba0b60..6ee59a21c 100644 --- a/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java +++ b/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java @@ -25,7 +25,6 @@ import fixes.Fix; -import java.io.IOException; import java.util.List; public class RedhatSecurityParser extends RedhatParser { diff --git a/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java index 8c3ff31e9..301ab7810 100644 --- a/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java +++ b/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java @@ -25,8 +25,6 @@ */ import fixes.FixProcessor; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import java.io.IOException; import java.net.HttpURLConnection; From 4d2316a8fcdbcf04b0f98012513445216f7fb131 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Thu, 2 Nov 2023 16:31:45 -0400 Subject: [PATCH 13/19] Code cleanup --- patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java index e03c2a43d..82658434b 100644 --- a/patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java +++ b/patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java @@ -76,8 +76,7 @@ public ArrayList getUrls(String cveId) throws IOException { // Test NVD direct cve page final String directSource = "https://nvd.nist.gov/vuln/detail/" + cveId; if(testConnection(directSource)) { - try { urlList.addAll(this.scrapeReferences(directSource)); } - catch (IOException e) { logger.warn("Failed to scrape references from NVD page: {}", e.toString()); } + urlList.addAll(this.scrapeReferences(directSource)); } return urlList; From 4ecc5784efec38b6980e2badfcbcaa46277a6a21 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Thu, 2 Nov 2023 16:32:16 -0400 Subject: [PATCH 14/19] Added blacklist enum to improve filtering/DQ of GenericParser results. --- .../java/fixes/parsers/GenericParser.java | 41 ++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/patchfinder/src/main/java/fixes/parsers/GenericParser.java b/patchfinder/src/main/java/fixes/parsers/GenericParser.java index f15cb2cae..b69f3b4ea 100644 --- a/patchfinder/src/main/java/fixes/parsers/GenericParser.java +++ b/patchfinder/src/main/java/fixes/parsers/GenericParser.java @@ -82,6 +82,37 @@ public static boolean hasWord(String word) { return false; } } + + private static List stringValues() { + return Arrays.stream(values()).map(Enum::toString).toList(); + } + } + + private enum FIX_WORDS_BLACKLIST { + NO, NOT; + + public static int containsKeywords(String text) { + text = text.toUpperCase(); + int numKeywords = 0; + + // Get string values for all blacklist keywords + for (String keyword : FIX_WORDS_BLACKLIST.stringValues()) { + // Append all fix words (looking for things like "no fix" or "not resolved") + // Check past tense forms of keywords if needed + for (String fixWord : FIX_WORDS.stringValues()) { + keyword += " " + fixWord; + if(text.contains(keyword)) numKeywords++; + else if(!keyword.endsWith("D") && text.contains(keyword + "D")) numKeywords++; + else if(!keyword.endsWith("ED") && text.contains(keyword + "ED")) numKeywords++; + } + } + + return numKeywords; + } + + private static List stringValues() { + return Arrays.stream(values()).map(Enum::toString).toList(); + } } protected GenericParser(String cveId, String url) { @@ -120,7 +151,7 @@ protected List parseWebPage() { final String fixDescription = String.join(" ", descriptionElements.eachText()); // If data was found, store in a new Fix object and add to list of found fixes - if(fixDescription.length() > 0) + if(fixDescription.length() > 0 && isFix(fixDescription)) this.fixes.add(new Fix(cveId, fixDescription, url)); // Skip to next header @@ -132,6 +163,14 @@ protected List parseWebPage() { return this.fixes; } + private boolean isFix(String fixDescription) { + // Get number of words that are blacklisted (blacklist words imply not fixed) + final int numBlacklistWords = FIX_WORDS_BLACKLIST.containsKeywords(fixDescription); + + // If we find none, is likely a fix, 1 or more would imply is not a fix + return numBlacklistWords < 1; + } + private Elements findDescriptionElements(Element e) { final Elements elements = new Elements(); // Attempt to get next sibling, store if found From 02b99abe9650be1c2ca3cdcdd8a9e58410ebd2a2 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Thu, 2 Nov 2023 16:35:53 -0400 Subject: [PATCH 15/19] Redundant check --- patchfinder/src/main/java/fixes/parsers/GenericParser.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/patchfinder/src/main/java/fixes/parsers/GenericParser.java b/patchfinder/src/main/java/fixes/parsers/GenericParser.java index b69f3b4ea..27ffba572 100644 --- a/patchfinder/src/main/java/fixes/parsers/GenericParser.java +++ b/patchfinder/src/main/java/fixes/parsers/GenericParser.java @@ -98,12 +98,12 @@ public static int containsKeywords(String text) { // Get string values for all blacklist keywords for (String keyword : FIX_WORDS_BLACKLIST.stringValues()) { // Append all fix words (looking for things like "no fix" or "not resolved") - // Check past tense forms of keywords if needed for (String fixWord : FIX_WORDS.stringValues()) { keyword += " " + fixWord; if(text.contains(keyword)) numKeywords++; - else if(!keyword.endsWith("D") && text.contains(keyword + "D")) numKeywords++; - else if(!keyword.endsWith("ED") && text.contains(keyword + "ED")) numKeywords++; + // Check past tense/plural forms +// else if(!keyword.endsWith("D") && text.contains(keyword + "D")) numKeywords++; +// else if(!keyword.endsWith("ED") && text.contains(keyword + "ED")) numKeywords++; } } From f92d917f46f6932baf09ccfb006bfddc19c81fb0 Mon Sep 17 00:00:00 2001 From: Dylan Mulligan Date: Thu, 2 Nov 2023 17:32:44 -0400 Subject: [PATCH 16/19] Rename test --- .../{NvdFixUrlFinderTest.java => NvdUrlFinderTest.java} | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename patchfinder/src/test/java/fixes/urlfinders/{NvdFixUrlFinderTest.java => NvdUrlFinderTest.java} (57%) diff --git a/patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java b/patchfinder/src/test/java/fixes/urlfinders/NvdUrlFinderTest.java similarity index 57% rename from patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java rename to patchfinder/src/test/java/fixes/urlfinders/NvdUrlFinderTest.java index f0a820010..1350b787b 100644 --- a/patchfinder/src/test/java/fixes/urlfinders/NvdFixUrlFinderTest.java +++ b/patchfinder/src/test/java/fixes/urlfinders/NvdUrlFinderTest.java @@ -1,7 +1,7 @@ package fixes.urlfinders; -public class NvdFixUrlFinderTest extends FixUrlFinderTest { - public NvdFixUrlFinderTest() { +public class NvdUrlFinderTest extends FixUrlFinderTest { + public NvdUrlFinderTest() { super(new NvdUrlFinder()); } From 1f91d689d2a1b91f745c21f33c16c0eb1ed48ed4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 8 Nov 2023 14:37:05 -0500 Subject: [PATCH 17/19] bug --- .../src/main/java/db/DatabaseHelper.java | 2 + .../src/main/java/fixes/FixFinder.java | 2 + .../java/fixes/parsers/CXSecurityParser.java | 38 -------------- .../main/java/fixes/parsers/FixParser.java | 3 -- .../fixes/urlfinders/CXSecurityUrlFinder.java | 49 +++++++++++++++++++ .../urlfinders/VulnerabilityUrlFinder.java | 2 +- .../fixes/parsers/CXSecurityParserTest.java | 42 ---------------- .../urlfinders/CXSecurityUrlFinderTest.java | 30 ++++++++++++ .../fixes/urlfinders/NvdUrlFinderTest.java | 8 ++- 9 files changed, 91 insertions(+), 85 deletions(-) delete mode 100644 patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java create mode 100644 patchfinder/src/main/java/fixes/urlfinders/CXSecurityUrlFinder.java delete mode 100644 patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java create mode 100644 patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java diff --git a/patchfinder/src/main/java/db/DatabaseHelper.java b/patchfinder/src/main/java/db/DatabaseHelper.java index 0aee3f9d8..90dbb956b 100644 --- a/patchfinder/src/main/java/db/DatabaseHelper.java +++ b/patchfinder/src/main/java/db/DatabaseHelper.java @@ -467,4 +467,6 @@ public ArrayList getCveSourcesNVD(String cve_id) { } return sourceURL; } + + } \ No newline at end of file diff --git a/patchfinder/src/main/java/fixes/FixFinder.java b/patchfinder/src/main/java/fixes/FixFinder.java index 66d995a94..b83f97a8b 100644 --- a/patchfinder/src/main/java/fixes/FixFinder.java +++ b/patchfinder/src/main/java/fixes/FixFinder.java @@ -27,6 +27,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import env.FixFinderEnvVars; import db.DatabaseHelper; +import fixes.urlfinders.CXSecurityUrlFinder; import fixes.urlfinders.FixUrlFinder; import fixes.urlfinders.NvdUrlFinder; import fixes.urlfinders.VulnerabilityUrlFinder; @@ -78,6 +79,7 @@ public static void init() { // Add the instances to the fixURLFinders list fixURLFinders.add(new VulnerabilityUrlFinder()); fixURLFinders.add(new NvdUrlFinder()); + fixURLFinders.add(new CXSecurityUrlFinder()); logger.info("Done initializing {} FixUrlFinders: {}", fixURLFinders.size(), fixURLFinders); } diff --git a/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java b/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java deleted file mode 100644 index 4d8b546c1..000000000 --- a/patchfinder/src/main/java/fixes/parsers/CXSecurityParser.java +++ /dev/null @@ -1,38 +0,0 @@ -package fixes.parsers; - -import fixes.Fix; - -import java.util.ArrayList; -import java.util.List; - -import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -public class CXSecurityParser extends FixParser { - protected CXSecurityParser(String cveId, String url) { - super(cveId, url); - } - - @Override - protected List parseWebPage() { - List fixSources = new ArrayList<>(); - - // Retrieve description - String description = this.DOM.select("h6").first().text(); - - Elements references = this.DOM.select("table").last().select("td").select("div"); - for(Element row : references){ - String url = row.text(); - fixSources.add(url); - - } - - // TODO: Remove when class is migrated to type UrlParser - // For each URL, find the correct parser for it and add the fixes found for that URL - for(String fixSource : fixSources){ - FixParser parser = FixParser.getParser(cveId, fixSource); - this.fixes.addAll(parser.parse()); - } - return this.fixes; - } - -} diff --git a/patchfinder/src/main/java/fixes/parsers/FixParser.java b/patchfinder/src/main/java/fixes/parsers/FixParser.java index cb1b73a10..1ad6b2ba2 100644 --- a/patchfinder/src/main/java/fixes/parsers/FixParser.java +++ b/patchfinder/src/main/java/fixes/parsers/FixParser.java @@ -112,9 +112,6 @@ public static FixParser getParser(String cveId, String url) { case "access.redhat.com": parser = new RedhatParser(cveId, url); break; - case "cxsecurity.com": - parser = new CXSecurityParser(cveId, url); - break; default: parser = new GenericParser(cveId, url); break; diff --git a/patchfinder/src/main/java/fixes/urlfinders/CXSecurityUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/CXSecurityUrlFinder.java new file mode 100644 index 000000000..2689849c9 --- /dev/null +++ b/patchfinder/src/main/java/fixes/urlfinders/CXSecurityUrlFinder.java @@ -0,0 +1,49 @@ +package fixes.urlfinders; + +import fixes.FixFinder; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; + +public class CXSecurityUrlFinder extends FixUrlFinder{ + public CXSecurityUrlFinder() { } + + @Override + public ArrayList getUrls(String cveId) throws IOException { + + logger.info("Getting fixes for CVE: {}", cveId); + + // Get all sources for the cve + ArrayList urlList = FixFinder.getDatabaseHelper().getSpecificCveSources(cveId); + + // Test NVD direct cve page + final String directSource = "https://cxsecurity.com/cveshow/" + cveId; + if(testConnection(directSource)) { + urlList.addAll(this.scrapeReferences(directSource)); + } + + return urlList; + } + + private List scrapeReferences(String url) throws IOException { + // Isolate the HTML for the references table + Elements rows = this.getDOM(url).select("table").last().select("td").select("div"); + + // For each URL stored in the table, if it has a "Patch" badge associated with it, add it to fixSources + List fixSources = new ArrayList<>(); + for(Element row : rows){ + String refUrl = row.text(); + fixSources.add(url); + } + + return fixSources; + } + + +} diff --git a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityUrlFinder.java index e3d7e6ef1..472954509 100644 --- a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityUrlFinder.java +++ b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityUrlFinder.java @@ -27,7 +27,7 @@ public List getUrls(String cveId) throws IOException { // // // Get all sources for the cve // Set sources = new HashSet<>(FixFinder.getDatabaseHelper().getSpecificCveSources(cveId)); -// +// //copy over!!!!! // // TODO: Move to CXSecurityUrlFinder // // Test each source for a valid connection // for (String source : sources) { diff --git a/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java b/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java deleted file mode 100644 index c4323e408..000000000 --- a/patchfinder/src/test/java/fixes/parsers/CXSecurityParserTest.java +++ /dev/null @@ -1,42 +0,0 @@ -package fixes.parsers; - -import fixes.Fix; -import org.jsoup.Jsoup; -import org.junit.Test; - -import java.io.IOException; -import java.net.URL; -import java.util.ArrayList; -import java.util.List; -import static org.junit.Assert.assertEquals; - -public class CXSecurityParserTest extends FixParserTest { - public CXSecurityParserTest() { - // TODO: Initialize with test values -// this.setFixParser(getNewParser("", "")); - } - - @Override - protected CXSecurityParser getNewParser(String cveId, String url) { - return new CXSecurityParser(cveId, url); - } - - @Override - //zero fixes are found - public void testParseWebpage() { - // TODO: Test parseWebpage - } - - @Test - public void testParseWebpageNoFixes() { - // TODO: Test parseWebpage with second cve/url - String cveId ="CVE-2023-3990"; - String url ="https://cxsecurity.com/cveshow/CVE-2023-3990"; - this.setFixParser(getNewParser(cveId, url)); - - List actual = this.fixParser().parse(); - List expected = new ArrayList<>(); - - assertEquals(expected, actual); - } -} diff --git a/patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java b/patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java new file mode 100644 index 000000000..b22800e76 --- /dev/null +++ b/patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java @@ -0,0 +1,30 @@ +package fixes.urlfinders; + +import fixes.Fix; +import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class CXSecurityUrlFinderTest extends FixUrlFinderTest{ + public CXSecurityUrlFinderTest() { + super(new CXSecurityUrlFinder()); + } + + //zero urls are found + @Override + public void testRun() { + // TODO: Test parseWebpage with second cve/url + String cveId ="CVE-2023-3990"; + + + List actual = this.fixUrlFinder.run(cveId); + List expected = new ArrayList<>(); + + assertEquals(expected, actual); + } + + + +} diff --git a/patchfinder/src/test/java/fixes/urlfinders/NvdUrlFinderTest.java b/patchfinder/src/test/java/fixes/urlfinders/NvdUrlFinderTest.java index 1350b787b..e54554d6f 100644 --- a/patchfinder/src/test/java/fixes/urlfinders/NvdUrlFinderTest.java +++ b/patchfinder/src/test/java/fixes/urlfinders/NvdUrlFinderTest.java @@ -1,5 +1,10 @@ package fixes.urlfinders; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + public class NvdUrlFinderTest extends FixUrlFinderTest { public NvdUrlFinderTest() { super(new NvdUrlFinder()); @@ -7,6 +12,7 @@ public NvdUrlFinderTest() { @Override public void testRun() { - // TODO: Test run + // TODO: Test parseWebpage with second cve/url + } } From 9f31b02911d8c84eabdcd9c1827ab329808a2821 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 14 Nov 2023 09:49:28 -0500 Subject: [PATCH 18/19] Test suite completed for CXSecurityParser --- patchfinder/src/test/java/fixes/FixFinderTest.java | 2 +- .../java/fixes/urlfinders/CXSecurityUrlFinderTest.java | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/patchfinder/src/test/java/fixes/FixFinderTest.java b/patchfinder/src/test/java/fixes/FixFinderTest.java index f28f59db8..2bf119814 100644 --- a/patchfinder/src/test/java/fixes/FixFinderTest.java +++ b/patchfinder/src/test/java/fixes/FixFinderTest.java @@ -39,7 +39,7 @@ * @author Richard Sawh */ public class FixFinderTest { - + static {FixFinder.init();} @Before public void setUp() { FixFinderEnvVars.initializeEnvVars(true); diff --git a/patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java b/patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java index b22800e76..6cbc222cc 100644 --- a/patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java +++ b/patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java @@ -1,6 +1,7 @@ package fixes.urlfinders; import fixes.Fix; +import fixes.FixFinder; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.List; @@ -8,10 +9,11 @@ import static org.junit.Assert.assertEquals; public class CXSecurityUrlFinderTest extends FixUrlFinderTest{ + public CXSecurityUrlFinderTest() { super(new CXSecurityUrlFinder()); } - + static {FixFinder.init();} //zero urls are found @Override public void testRun() { @@ -20,7 +22,8 @@ public void testRun() { List actual = this.fixUrlFinder.run(cveId); - List expected = new ArrayList<>(); + List expected = List.of("https://cxsecurity.com/cvemap", "https://cxsecurity.com/cveshow/CVE-2023-3990", "https://cxsecurity.com/cveshow/CVE-2023-3990","https://cxsecurity.com/cveshow/CVE-2023-3990"); + assertEquals(expected, actual); } From 87468d852facbece73f677d03a8536d04ecb755f Mon Sep 17 00:00:00 2001 From: Dylan Mulligan <60295839+dylan-mulligan@users.noreply.github.com> Date: Tue, 21 Nov 2023 01:41:24 -0800 Subject: [PATCH 19/19] Update env.list --- patchfinder/env.list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/patchfinder/env.list b/patchfinder/env.list index 1bca8eec6..dc9495ef5 100644 --- a/patchfinder/env.list +++ b/patchfinder/env.list @@ -23,4 +23,4 @@ CLONE_PATH=nvip_data/patch-repos PATCH_SRC_URL_PATH=nvip_data/source_dict.json # --- FIX FINDER VARS --- -FF_INPUT_MODE=dev +FF_INPUT_MODE=db