diff --git a/patchfinder/env.list b/patchfinder/env.list index 8db20f8eb..dc9495ef5 100644 --- a/patchfinder/env.list +++ b/patchfinder/env.list @@ -23,4 +23,4 @@ CLONE_PATH=nvip_data/patch-repos PATCH_SRC_URL_PATH=nvip_data/source_dict.json # --- FIX FINDER VARS --- -FF_INPUT_MODE=db \ No newline at end of file +FF_INPUT_MODE=db diff --git a/patchfinder/pom.xml b/patchfinder/pom.xml index b389a77a4..fbd42aff0 100644 --- a/patchfinder/pom.xml +++ b/patchfinder/pom.xml @@ -9,8 +9,8 @@ 1.0 - 1.8 - 1.8 + 17 + 17 UTF-8 diff --git a/patchfinder/src/main/java/FixFinderMain.java b/patchfinder/src/main/java/FixFinderMain.java index 6b27af1db..bf2ac9517 100644 --- a/patchfinder/src/main/java/FixFinderMain.java +++ b/patchfinder/src/main/java/FixFinderMain.java @@ -85,9 +85,7 @@ private void runRabbit() { private void runDev() { // Manually enter CVEs for development List cveIds = new ArrayList<>(); - cveIds.add("CVE-2022-27911"); - cveIds.add("CVE-2023-30367"); - cveIds.add("CVE-2022-0847"); + cveIds.add("CVE-2023-38571"); try { FixFinder.run(cveIds); @@ -97,6 +95,7 @@ private void runDev() { } public static void main(String[] args) { -// run(); + FixFinderMain finder = new FixFinderMain(); + finder.start(); } } diff --git a/patchfinder/src/main/java/db/DatabaseHelper.java b/patchfinder/src/main/java/db/DatabaseHelper.java index 0aee3f9d8..90dbb956b 100644 --- a/patchfinder/src/main/java/db/DatabaseHelper.java +++ b/patchfinder/src/main/java/db/DatabaseHelper.java @@ -467,4 +467,6 @@ public ArrayList getCveSourcesNVD(String cve_id) { } return sourceURL; } + + } \ No newline at end of file diff --git a/patchfinder/src/main/java/fixes/FixFinder.java b/patchfinder/src/main/java/fixes/FixFinder.java index baba7dbd9..b83f97a8b 100644 --- a/patchfinder/src/main/java/fixes/FixFinder.java +++ b/patchfinder/src/main/java/fixes/FixFinder.java @@ -27,9 +27,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import env.FixFinderEnvVars; import db.DatabaseHelper; +import fixes.urlfinders.CXSecurityUrlFinder; import fixes.urlfinders.FixUrlFinder; -import fixes.urlfinders.NvdFixUrlFinder; -import fixes.urlfinders.VulnerabilityFixUrlFinder; +import fixes.urlfinders.NvdUrlFinder; +import fixes.urlfinders.VulnerabilityUrlFinder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -76,8 +77,9 @@ public static void init() { logger.info("Initializing FixUrlFinders..."); // Add the instances to the fixURLFinders list - fixURLFinders.add(new VulnerabilityFixUrlFinder()); - fixURLFinders.add(new NvdFixUrlFinder()); + fixURLFinders.add(new VulnerabilityUrlFinder()); + fixURLFinders.add(new NvdUrlFinder()); + fixURLFinders.add(new CXSecurityUrlFinder()); logger.info("Done initializing {} FixUrlFinders: {}", fixURLFinders.size(), fixURLFinders); } diff --git a/patchfinder/src/main/java/fixes/FixFinderThread.java b/patchfinder/src/main/java/fixes/FixFinderThread.java index 0b3ad03af..8a8847050 100644 --- a/patchfinder/src/main/java/fixes/FixFinderThread.java +++ b/patchfinder/src/main/java/fixes/FixFinderThread.java @@ -25,13 +25,9 @@ */ import fixes.parsers.FixParser; -import fixes.parsers.CISAParser; -import fixes.parsers.GenericParser; -import fixes.parsers.NVDParser; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.CompletableFuture; @@ -83,17 +79,8 @@ public void run() { for (String url : urls) { CompletableFuture> future = CompletableFuture.supplyAsync(() -> { - - - try{ - FixParser parser = FixParser.getParser(cveId, url); - return parser.parse(); - } catch(IOException e){ - logger.error("Error occurred while parsing url {} for CVE {}: {}", url, cveId, e.toString()); - e.printStackTrace(); - return null; - } - + FixParser parser = FixParser.getParser(cveId, url); + return parser.parse(); }); futures.add(future); diff --git a/patchfinder/src/main/java/fixes/FixProcessor.java b/patchfinder/src/main/java/fixes/FixProcessor.java new file mode 100644 index 000000000..cf1a68bac --- /dev/null +++ b/patchfinder/src/main/java/fixes/FixProcessor.java @@ -0,0 +1,19 @@ +package fixes; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; + +import java.io.IOException; +import java.net.URL; + +public abstract class FixProcessor { + // Logger instance for FixProcessors + protected static final Logger logger = LogManager.getLogger(); + + // Utility method for getting DOM from string URL, throws IOException in case of an error + protected Document getDOM(String url) throws IOException { + return Jsoup.parse(new URL(url), 10000); + } +} diff --git a/patchfinder/src/main/java/fixes/parsers/CISAParser.java b/patchfinder/src/main/java/fixes/parsers/CISAParser.java index 6ae767423..2dd581c67 100644 --- a/patchfinder/src/main/java/fixes/parsers/CISAParser.java +++ b/patchfinder/src/main/java/fixes/parsers/CISAParser.java @@ -25,12 +25,8 @@ */ import fixes.Fix; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; import org.jsoup.select.Elements; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; /** @@ -47,7 +43,7 @@ protected CISAParser(String cveId, String url){ } @Override - protected List parseWebPage() throws IOException { + protected List parseWebPage() { Elements headers = this.DOM.select("div[id=1-full__main]").first().select("h"); return this.fixes; diff --git a/patchfinder/src/main/java/fixes/parsers/FixParser.java b/patchfinder/src/main/java/fixes/parsers/FixParser.java index d9fe64067..1ad6b2ba2 100644 --- a/patchfinder/src/main/java/fixes/parsers/FixParser.java +++ b/patchfinder/src/main/java/fixes/parsers/FixParser.java @@ -25,13 +25,10 @@ */ import fixes.Fix; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jsoup.Jsoup; +import fixes.FixProcessor; import org.jsoup.nodes.Document; import java.io.IOException; -import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -42,8 +39,7 @@ * @author Paul Vickers * @author Dylan Mulligan */ -public abstract class FixParser { - protected final static Logger logger = LogManager.getLogger(); +public abstract class FixParser extends FixProcessor { protected final String cveId; protected final String url; @@ -62,7 +58,7 @@ public List parse() { // Attempt to parse page and store returned Document object try { logger.info("{} is parsing url '{}'...", getClass().getSimpleName(), url); - this.DOM = Jsoup.parse(new URL(url), 10000); + this.DOM = this.getDOM(this.url); // Call abstract method implementation based on instance this.parseWebPage(); } @@ -81,7 +77,7 @@ public List parse() { //TODO: Remove this throws unless we really need it, as URL interaction has been // moved to parse() and the IOExceptions are handled there - protected abstract List parseWebPage() throws IOException; + protected abstract List parseWebPage(); /** * Delegation method to determine which parser should be used to find fixes from the given url. @@ -91,9 +87,14 @@ public List parse() { * @return Correct parser to be used * */ - public static FixParser getParser(String cveId, String url) throws MalformedURLException { + public static FixParser getParser(String cveId, String url) { // Objectify url for domain extraction - final URL urlObj = new URL(url); + URL urlObj = null; + try { urlObj = new URL(url); } + catch (Exception e) { + // This should not happen, as URL has already been validated + logger.error("Fatal error occurred: {}", e.toString()); + } // Extract domain final String domain = urlObj.getHost(); @@ -102,9 +103,9 @@ public static FixParser getParser(String cveId, String url) throws MalformedURLE // Choose parser based on domain switch (domain) { - case "nvd.nist.gov": - parser = new NVDParser(cveId, url); - break; +// case "nvd.nist.gov": +// parser = new NVDParser(cveId, url); +// break; case "cisa.gov": parser = new CISAParser(cveId, url); break; diff --git a/patchfinder/src/main/java/fixes/parsers/GenericParser.java b/patchfinder/src/main/java/fixes/parsers/GenericParser.java index 3f4d79459..27ffba572 100644 --- a/patchfinder/src/main/java/fixes/parsers/GenericParser.java +++ b/patchfinder/src/main/java/fixes/parsers/GenericParser.java @@ -29,6 +29,7 @@ import org.jsoup.select.Elements; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; @@ -50,6 +51,7 @@ private enum FIX_WORDS { MITIGATION, RESOLVE, RESOLUTION; +// COUNTERMEASURE; /** * Determines if given word is a valid member of this enum (case-insensitive). @@ -58,10 +60,58 @@ private enum FIX_WORDS { * @return whether the word is a valid member of this enum */ public static boolean hasWord(String word) { + word = word.toUpperCase(); try { - FIX_WORDS.valueOf(word.toUpperCase()); + FIX_WORDS.valueOf(word); return true; - } catch (Exception ignored) { return false; } + } catch (Exception ignored) { + // If no direct match, check if word is plural and try singular form + final boolean endsWithES = word.endsWith("ES"); + final boolean endsWithS = endsWithES || word.endsWith("S"); + + if(endsWithES || endsWithS) { + final int endIndex = endsWithES ? 2 : 1; + final String trimmedWord = word.substring(0, endIndex); + try { + FIX_WORDS.valueOf(trimmedWord); + return true; + } catch (Exception ignored1) { } + } + + // Return false if no match + return false; + } + } + + private static List stringValues() { + return Arrays.stream(values()).map(Enum::toString).toList(); + } + } + + private enum FIX_WORDS_BLACKLIST { + NO, NOT; + + public static int containsKeywords(String text) { + text = text.toUpperCase(); + int numKeywords = 0; + + // Get string values for all blacklist keywords + for (String keyword : FIX_WORDS_BLACKLIST.stringValues()) { + // Append all fix words (looking for things like "no fix" or "not resolved") + for (String fixWord : FIX_WORDS.stringValues()) { + keyword += " " + fixWord; + if(text.contains(keyword)) numKeywords++; + // Check past tense/plural forms +// else if(!keyword.endsWith("D") && text.contains(keyword + "D")) numKeywords++; +// else if(!keyword.endsWith("ED") && text.contains(keyword + "ED")) numKeywords++; + } + } + + return numKeywords; + } + + private static List stringValues() { + return Arrays.stream(values()).map(Enum::toString).toList(); } } @@ -84,8 +134,11 @@ protected List parseWebPage() { // Iterate over header objects for (Element e : headerElements) { + // Check text and id of header for keywords + final List words = new ArrayList<>(Arrays.asList(e.text().split(" "))); + words.add(e.id()); // Split text on spaces and check each word. - for (String headerWord : e.text().split(" ")) { + for (String headerWord : words) { // Check if word is a member of FIX_WORDS (case-insensitive) if(FIX_WORDS.hasWord(headerWord)) { // Find and store description elements related to the current header @@ -98,8 +151,8 @@ protected List parseWebPage() { final String fixDescription = String.join(" ", descriptionElements.eachText()); // If data was found, store in a new Fix object and add to list of found fixes - if(fixDescription.length() > 0) - this.fixes.add(new Fix(cveId, fixDescription.toString(), url)); + if(fixDescription.length() > 0 && isFix(fixDescription)) + this.fixes.add(new Fix(cveId, fixDescription, url)); // Skip to next header break; @@ -110,6 +163,14 @@ protected List parseWebPage() { return this.fixes; } + private boolean isFix(String fixDescription) { + // Get number of words that are blacklisted (blacklist words imply not fixed) + final int numBlacklistWords = FIX_WORDS_BLACKLIST.containsKeywords(fixDescription); + + // If we find none, is likely a fix, 1 or more would imply is not a fix + return numBlacklistWords < 1; + } + private Elements findDescriptionElements(Element e) { final Elements elements = new Elements(); // Attempt to get next sibling, store if found diff --git a/patchfinder/src/main/java/fixes/parsers/NVDParser.java b/patchfinder/src/main/java/fixes/parsers/NVDParser.java index f4d2b60f5..1f862d508 100644 --- a/patchfinder/src/main/java/fixes/parsers/NVDParser.java +++ b/patchfinder/src/main/java/fixes/parsers/NVDParser.java @@ -25,14 +25,9 @@ */ import fixes.Fix; -import fixes.FixFinderThread; -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; import org.jsoup.select.Elements; -import java.io.IOException; -import java.net.URL; import java.util.ArrayList; import java.util.List; @@ -78,10 +73,9 @@ protected NVDParser(String cveId, String url){ * scrape for the references table and then delegate to other parsers for those sources. * * @return List of fixes for the CVE - * @throws IOException if an error occurs during scraping */ @Override - public List parseWebPage() throws IOException{ + public List parseWebPage() { // Isolate the HTML for the references table Elements rows = this.DOM.select("div[id=vulnHyperlinksPanel]").first().select("table").first().select("tbody").select("tr"); diff --git a/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java b/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java index a86988e62..b4aefa897 100644 --- a/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java +++ b/patchfinder/src/main/java/fixes/parsers/RedhatBugzillaParser.java @@ -2,7 +2,6 @@ import fixes.Fix; -import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -13,7 +12,7 @@ protected RedhatBugzillaParser(String cveId, String url){ @Override - protected List parseWebPage() throws IOException { + protected List parseWebPage() { List newFixes = new ArrayList<>(); // TODO: Add Bugzilla specific implementation diff --git a/patchfinder/src/main/java/fixes/parsers/RedhatParser.java b/patchfinder/src/main/java/fixes/parsers/RedhatParser.java index 80ee0f993..6a8952bf8 100644 --- a/patchfinder/src/main/java/fixes/parsers/RedhatParser.java +++ b/patchfinder/src/main/java/fixes/parsers/RedhatParser.java @@ -40,7 +40,7 @@ protected RedhatParser(String cveId, String url){ super(cveId, url); } - protected List parseWebPage() throws IOException{ + protected List parseWebPage() { throw new UnsupportedOperationException(); } @@ -48,7 +48,7 @@ protected List parseWebPage() throws IOException{ * Delegates and parses the specified webpage using the RedHat Sub classes * @return list of all found fixes */ - @Override + @Override // TODO: Migrate to UrlFinder and make use of the new methods in FixProcessor/FixUrlFinder public List parse(){ // Init fixes list this.fixes = new ArrayList<>(); diff --git a/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java b/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java index c32298859..6ee59a21c 100644 --- a/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java +++ b/patchfinder/src/main/java/fixes/parsers/RedhatSecurityParser.java @@ -25,7 +25,6 @@ import fixes.Fix; -import java.io.IOException; import java.util.List; public class RedhatSecurityParser extends RedhatParser { @@ -35,7 +34,7 @@ protected RedhatSecurityParser(String cveId, String url){ } @Override - protected List parseWebPage() throws IOException { + protected List parseWebPage() { return null; } } diff --git a/patchfinder/src/main/java/fixes/urlfinders/CXSecurityUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/CXSecurityUrlFinder.java new file mode 100644 index 000000000..2689849c9 --- /dev/null +++ b/patchfinder/src/main/java/fixes/urlfinders/CXSecurityUrlFinder.java @@ -0,0 +1,49 @@ +package fixes.urlfinders; + +import fixes.FixFinder; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.List; + +public class CXSecurityUrlFinder extends FixUrlFinder{ + public CXSecurityUrlFinder() { } + + @Override + public ArrayList getUrls(String cveId) throws IOException { + + logger.info("Getting fixes for CVE: {}", cveId); + + // Get all sources for the cve + ArrayList urlList = FixFinder.getDatabaseHelper().getSpecificCveSources(cveId); + + // Test NVD direct cve page + final String directSource = "https://cxsecurity.com/cveshow/" + cveId; + if(testConnection(directSource)) { + urlList.addAll(this.scrapeReferences(directSource)); + } + + return urlList; + } + + private List scrapeReferences(String url) throws IOException { + // Isolate the HTML for the references table + Elements rows = this.getDOM(url).select("table").last().select("td").select("div"); + + // For each URL stored in the table, if it has a "Patch" badge associated with it, add it to fixSources + List fixSources = new ArrayList<>(); + for(Element row : rows){ + String refUrl = row.text(); + fixSources.add(url); + } + + return fixSources; + } + + +} diff --git a/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java index 74ff8d1f1..301ab7810 100644 --- a/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java +++ b/patchfinder/src/main/java/fixes/urlfinders/FixUrlFinder.java @@ -24,13 +24,13 @@ * SOFTWARE. */ -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import fixes.FixProcessor; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URL; import java.util.ArrayList; +import java.util.List; /** * Abstract class responsible for finding possible fix source URLs for the FixFinder. @@ -38,21 +38,31 @@ * @author Dylan Mulligan */ -public abstract class FixUrlFinder { +public abstract class FixUrlFinder extends FixProcessor { + // To be implemented in child classes, houses the actual logic that selects source urls + public abstract List getUrls(String cveId) throws IOException; - protected static final Logger logger = LogManager.getLogger(FixUrlFinder.class.getName()); - - public abstract ArrayList run(String cveId) throws IOException; + //Called for all child instances, makes use of their specific implementation of + // getUrls(), then tests and filters out any urls that can't be connected to + public List run(String cveId) { + try { + final List urls = this.getUrls(cveId); + // Test each source for a valid connection and filter failed connections + return urls.stream().filter(FixUrlFinder::testConnection).toList(); + } catch (IOException e) { + logger.error("Failed to get urls for CVE '{}': {}", cveId, e.toString()); + return new ArrayList<>(); + } + } - protected static boolean testConnection(String address) throws IOException { + // Tests the connection of a given address and returns the boolean result of the test + protected static boolean testConnection(String address) { logger.info("Testing Connection for address: " + address); - URL url = new URL(address); - HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); - int response; - try { - response = urlConnection.getResponseCode(); + URL url = new URL(address); + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + final int response = urlConnection.getResponseCode(); // Don't print OK responses, only when this is not the case if(response != 200) logger.info("Response Code: " + response); return true; diff --git a/patchfinder/src/main/java/fixes/urlfinders/NvdFixUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/NvdFixUrlFinder.java deleted file mode 100644 index 4bf75c3fa..000000000 --- a/patchfinder/src/main/java/fixes/urlfinders/NvdFixUrlFinder.java +++ /dev/null @@ -1,65 +0,0 @@ -package fixes.urlfinders; - -/** - * Copyright 2023 Rochester Institute of Technology (RIT). Developed with - * government support under contract 70RSAT19CB0000020 awarded by the United - * States Department of Homeland Security. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -import fixes.FixFinder; - -import java.io.IOException; -import java.util.ArrayList; - -/** - * Implementation of FixUrlFinder for CVEs collected from NVD - * - * @author Richard Sawh - */ -public class NvdFixUrlFinder extends FixUrlFinder { - - public NvdFixUrlFinder() { } - - @Override - public ArrayList run(String cveId) throws IOException { - logger.info("Getting fixes for CVE: {}", cveId); - ArrayList urlList = new ArrayList<>(); - - // Get all sources for the cve - ArrayList sources = FixFinder.getDatabaseHelper().getCveSourcesNVD(cveId); - - // Test each source for a valid connection - for (String source : sources) { - // Test reported source - if (testConnection(source)) { - urlList.add(source); - } - } - - // Test NVD direct cve page - final String directSource = "https://nvd.nist.gov/vuln/detail/" + cveId; - if(testConnection(directSource)) { - urlList.add(directSource); - } - - return urlList; - } -} \ No newline at end of file diff --git a/patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java new file mode 100644 index 000000000..82658434b --- /dev/null +++ b/patchfinder/src/main/java/fixes/urlfinders/NvdUrlFinder.java @@ -0,0 +1,103 @@ +package fixes.urlfinders; + +/** + * Copyright 2023 Rochester Institute of Technology (RIT). Developed with + * government support under contract 70RSAT19CB0000020 awarded by the United + * States Department of Homeland Security. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +import fixes.FixFinder; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * Implementation of FixUrlFinder for CVEs collected from NVD + * + * @author Richard Sawh + */ +public class NvdUrlFinder extends FixUrlFinder { + + private enum RESOURCE_TAGS { + PATCH("Patch"), // Hyperlink relates directly to patch information + VENDOR_ADVISORY("Vendor Advisory"), // Hyperlink relates to an advisory host + THIRD_PARTY_ADVISORY("Third Party Advisory"), // Hyperlink relates to a third-party advisory host + EXPLOIT("Exploit"), // Hyperlink relates to exploit information + ISSUE_TRACKING("Issue Tracking"); // Hyperlink relates to an issue tracking host + + private final String name; + RESOURCE_TAGS(String name) { + this.name = name; + } + + /** + * Safe valueOf method that relates tag name (i.e. "Vendor Advisory") to the correct member + * @param name name of resource tag + * @return correlated tag object, or null if not found + */ + public static RESOURCE_TAGS fromString(String name) { + for(RESOURCE_TAGS tag : RESOURCE_TAGS.values()) { + if(tag.name.equalsIgnoreCase(name)) return tag; + } + return null; + } + } + + public NvdUrlFinder() { } + + @Override + public ArrayList getUrls(String cveId) throws IOException { + logger.info("Getting fixes for CVE: {}", cveId); + + // Get all sources for the cve + ArrayList urlList = FixFinder.getDatabaseHelper().getCveSourcesNVD(cveId); + + // Test NVD direct cve page + final String directSource = "https://nvd.nist.gov/vuln/detail/" + cveId; + if(testConnection(directSource)) { + urlList.addAll(this.scrapeReferences(directSource)); + } + + return urlList; + } + + private List scrapeReferences(String url) throws IOException { + // Isolate the HTML for the references table + Elements rows = this.getDOM(url).select("div[id=vulnHyperlinksPanel]").first().select("table").first().select("tbody").select("tr"); + + // For each URL stored in the table, if it has a "Patch" badge associated with it, add it to fixSources + List fixSources = new ArrayList<>(); + for(Element row : rows){ + String refUrl = row.select("a").text(); + Elements spans = row.select("span.badge"); + // Check all resource tags + for(Element span: spans){ + // Add url if the tag matches any whitelisted tag + if(RESOURCE_TAGS.fromString(span.text()) != null) fixSources.add(refUrl); + } + } + + return fixSources; + } +} \ No newline at end of file diff --git a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java deleted file mode 100644 index 091f9a095..000000000 --- a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityFixUrlFinder.java +++ /dev/null @@ -1,35 +0,0 @@ -package fixes.urlfinders; - -import fixes.FixFinder; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Set; - -/** - * Implementation of FixUrlFinder for CVEs collected from the NVIP Crawler - * - * @author Richard Sawh - */ -public class VulnerabilityFixUrlFinder extends FixUrlFinder { - - public VulnerabilityFixUrlFinder() { } - - @Override - public ArrayList run(String cveId) throws IOException { - logger.info("Getting fixes for CVE: {}", cveId); - ArrayList urlList = new ArrayList<>(); - - // Get all sources for the cve - Set sources = new HashSet<>(FixFinder.getDatabaseHelper().getSpecificCveSources(cveId)); - - // Test each source for a valid connection - for (String source : sources) { - if (testConnection(source)) { - urlList.add(source); - } - } - return urlList; - } -} \ No newline at end of file diff --git a/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityUrlFinder.java b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityUrlFinder.java new file mode 100644 index 000000000..472954509 --- /dev/null +++ b/patchfinder/src/main/java/fixes/urlfinders/VulnerabilityUrlFinder.java @@ -0,0 +1,45 @@ +package fixes.urlfinders; + +import fixes.FixFinder; + +import java.io.IOException; +import java.util.*; + +/** + * Implementation of FixUrlFinder for CVEs collected from the NVIP Crawler + * + * @author Richard Sawh + */ +public class VulnerabilityUrlFinder extends FixUrlFinder { + + public VulnerabilityUrlFinder() { } + + @Override + public List getUrls(String cveId) throws IOException { + // Get all sources for the cve + return new HashSet<>(FixFinder.getDatabaseHelper().getSpecificCveSources(cveId)).stream().toList(); + } + +// @Override +// public ArrayList run(String cveId) { +// logger.info("Getting fixes for CVE: {}", cveId); +// ArrayList urlList = new ArrayList<>(); +// +// // Get all sources for the cve +// Set sources = new HashSet<>(FixFinder.getDatabaseHelper().getSpecificCveSources(cveId)); +// //copy over!!!!! +// // TODO: Move to CXSecurityUrlFinder +// // Test each source for a valid connection +// for (String source : sources) { +// //case for cxsecurityFinder +// if(Objects.equals(source, "https://cxsecurity.com/cvemap")){ +// source = "https://cxsecurity.com/cveshow/".concat(cveId) ; +// } +// +// if (testConnection(source)) { +// urlList.add(source); +// } +// } +// return urlList; +// } +} \ No newline at end of file diff --git a/patchfinder/src/test/java/fixes/FixFinderTest.java b/patchfinder/src/test/java/fixes/FixFinderTest.java index f12434aa7..2bf119814 100644 --- a/patchfinder/src/test/java/fixes/FixFinderTest.java +++ b/patchfinder/src/test/java/fixes/FixFinderTest.java @@ -28,11 +28,6 @@ import org.junit.Before; import org.junit.Test; -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import static org.junit.jupiter.api.Assertions.*; @@ -44,10 +39,19 @@ * @author Richard Sawh */ public class FixFinderTest { - + static {FixFinder.init();} @Before public void setUp() { FixFinderEnvVars.initializeEnvVars(true); } + @Test + public void testInit() { + // TODO: Test init + } + + @Test + public void testRun() { + // TODO: Test init + } } diff --git a/patchfinder/src/test/java/fixes/parsers/CISAParserTest.java b/patchfinder/src/test/java/fixes/parsers/CISAParserTest.java new file mode 100644 index 000000000..db50f6440 --- /dev/null +++ b/patchfinder/src/test/java/fixes/parsers/CISAParserTest.java @@ -0,0 +1,20 @@ +package fixes.parsers; + +import org.junit.Test; + +public class CISAParserTest extends FixParserTest { + public CISAParserTest() { + // TODO: Initialize with test values + this.setFixParser(getNewParser("", "")); + } + + @Override + protected CISAParser getNewParser(String cveId, String url) { + return new CISAParser(cveId, url); + } + + @Override + public void testParseWebpage() { + // TODO: Test parseWebpage + } +} diff --git a/patchfinder/src/test/java/fixes/parsers/FixParserTest.java b/patchfinder/src/test/java/fixes/parsers/FixParserTest.java new file mode 100644 index 000000000..b3237934f --- /dev/null +++ b/patchfinder/src/test/java/fixes/parsers/FixParserTest.java @@ -0,0 +1,33 @@ +package fixes.parsers; + +import env.FixFinderEnvVars; +import org.junit.Test; + +import java.io.IOException; +public abstract class FixParserTest { + private T fixParser; + + protected FixParserTest() { +// this.fixParser = fixParser; + FixFinderEnvVars.initializeEnvVars(true); + } + + public T fixParser() { return fixParser; } + + public void setFixParser(T fixParser) { this.fixParser = fixParser; } + + protected abstract T getNewParser(String cveId, String url); + + @Test + public abstract void testParseWebpage() throws IOException; + + @Test + public void testParse() { + // TODO: Test parse + } + + @Test + public void testGetParser() { + // TODO: Test getParser + } +} diff --git a/patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java b/patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java new file mode 100644 index 000000000..6cbc222cc --- /dev/null +++ b/patchfinder/src/test/java/fixes/urlfinders/CXSecurityUrlFinderTest.java @@ -0,0 +1,33 @@ +package fixes.urlfinders; + +import fixes.Fix; +import fixes.FixFinder; +import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class CXSecurityUrlFinderTest extends FixUrlFinderTest{ + + public CXSecurityUrlFinderTest() { + super(new CXSecurityUrlFinder()); + } + static {FixFinder.init();} + //zero urls are found + @Override + public void testRun() { + // TODO: Test parseWebpage with second cve/url + String cveId ="CVE-2023-3990"; + + + List actual = this.fixUrlFinder.run(cveId); + List expected = List.of("https://cxsecurity.com/cvemap", "https://cxsecurity.com/cveshow/CVE-2023-3990", "https://cxsecurity.com/cveshow/CVE-2023-3990","https://cxsecurity.com/cveshow/CVE-2023-3990"); + + + assertEquals(expected, actual); + } + + + +} diff --git a/patchfinder/src/test/java/fixes/urlfinders/FixUrlFinderTest.java b/patchfinder/src/test/java/fixes/urlfinders/FixUrlFinderTest.java new file mode 100644 index 000000000..d860ad3f5 --- /dev/null +++ b/patchfinder/src/test/java/fixes/urlfinders/FixUrlFinderTest.java @@ -0,0 +1,16 @@ +package fixes.urlfinders; + +import env.FixFinderEnvVars; +import org.junit.Test; + +public abstract class FixUrlFinderTest { + final protected T fixUrlFinder; + + protected FixUrlFinderTest(T fixUrlFinder) { + this.fixUrlFinder = fixUrlFinder; + FixFinderEnvVars.initializeEnvVars(true); + } + + @Test + public abstract void testRun(); +} diff --git a/patchfinder/src/test/java/fixes/urlfinders/NvdUrlFinderTest.java b/patchfinder/src/test/java/fixes/urlfinders/NvdUrlFinderTest.java new file mode 100644 index 000000000..e54554d6f --- /dev/null +++ b/patchfinder/src/test/java/fixes/urlfinders/NvdUrlFinderTest.java @@ -0,0 +1,18 @@ +package fixes.urlfinders; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class NvdUrlFinderTest extends FixUrlFinderTest { + public NvdUrlFinderTest() { + super(new NvdUrlFinder()); + } + + @Override + public void testRun() { + // TODO: Test parseWebpage with second cve/url + + } +}