diff --git a/.gitignore b/.gitignore index bbac347..27d2d8c 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ venv/ __pycache__/ _old/ fuzzing*/ -java/ jasmin/ work/ evaluation/test/ diff --git a/README.md b/README.md index ed3c238..197abb2 100644 --- a/README.md +++ b/README.md @@ -22,3 +22,31 @@ pip install -e . - Edit the template `run_template.sh' to point to the relevant filepaths on your machine - Run the shell script + + +# UROP +Things I have worked on: +- Switching paths after hotspot JIT compiled with one path to trigger deoptimization +- Created a fuzzer that produces Java code where each function represents nodes, to prevent everything being jammed in one method like in Java bytecode +- Tested the fuzzer on OpenJDK and GraalVM +- Running multiple paths concurrently to test the Thread-Safeness of the JIT compiler +- Created a hotspot log analyzer that summarizes compilation patterns, inlines, number of invocations etc. + +Potential areas to look into: +- GraalVM - Less mature JVM compared to OpenJDK so higher chance of finding bugs. Uncommon traps are not being set unlike OpenJDK probably because they use different optimizations so creating a fuzzer to target GraalVM optimizations could be interesting. + - Javabc - the compilation fails completely because of an irreducible loop when the graph size is too big. It does compile on small graphs and setting the MaxDuplicationFactor can increase that bounds. + - Java - all the inlining is failing for small and large graphs; it could be causing an error and is getting aborted(?) +- Generating semi-complex graphs - javabc is producing an irreducible loop that is not optimizable and in the Java fuzzer, each function is too simple to cause any bugs. Creating a fuzzer that does something in between the two might exercise the JIT compiler more. +- Concurrency - I've only tested running multiple paths concurrently. If the deoptimization is dealing with values in the registers real-time during deoptimization instead of only falling back to lower tiered code, it may be complex enough to be a source of bugs +- Dynamic dispatch - having an interface and calling different child class methods should trigger optimization/deoptimization which isn't tested in the current fuzzer. + + +Notes: +- Running the fuzzer with switching paths can be specified with ACTION='fuzz_with_changing_paths' in the argument +- I've renewed the wrapper to accept single or multiple paths. The normal one and multithread one are both in the root directory so copy-pasting it to the wrapper folder should switch the wrapper. +- JIT-related flags that I've used are in the java/javabc runner which is commented/uncommented to toggle +- Analyzer + - The logs can be specified in the setup_logs function under analyze.py + - It parses the xml and looks for tags task_queued, nmethod and task with method names that start with "TestCase" + - The tags are linked by compile_id to gather the statistics + - Occasionally, the hotspot log emits data in the under fragment tag and if important information is emitted there, the parser fails to find it and crashes sometimes. The information inside these tags is sometimes incomplete as well so I didn't know how to handle it so I just reran the fuzzer in case that happened. diff --git a/WrapperMultiThread.java b/WrapperMultiThread.java new file mode 100644 index 0000000..963c272 --- /dev/null +++ b/WrapperMultiThread.java @@ -0,0 +1,267 @@ +import java.io.File; +import java.util.Scanner; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.lang.Math; +import java.lang.Thread; +import java.lang.reflect.*; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +class Path { + String inputFilename; + int dir[]; + int actualOutput[]; + int expectedOutput[]; + int outputSize; + + Path(String inputFilename) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, ParseException { + this.inputFilename = inputFilename; + + // get direction size and expected output size from file + JSONParser parser = new JSONParser(); + JSONObject jsonObject = (JSONObject) parser.parse(new FileReader(inputFilename)); + + JSONArray jsonDir = (JSONArray) jsonObject.get("dirs"); + JSONArray jsonOutput = (JSONArray) jsonObject.get("expected_output"); + + outputSize = jsonOutput.size(); + int dirSize = jsonDir.size(); + + // create arrays + dir = new int[dirSize]; + actualOutput = new int[2*outputSize]; + expectedOutput = new int[outputSize]; + + // fill arrays + int counter = 0; + for (Object o : jsonDir) { + dir[counter] = (int) (long) o; + counter++; + } + counter = 0; + for (Object o : jsonOutput) { + expectedOutput[counter] = (int) (long) o; + counter++; + } + for (int i = outputSize; i < 2*outputSize; i++) { + actualOutput[i] = -1; + } + } + + boolean compare() { + // check actual and expected path are the same + for(int i = 0; i < outputSize; i++){ + if(expectedOutput[i] != actualOutput[i]){ + return false; + } + } + + // check no overflow occurred + for(int i = outputSize; i < 2*outputSize; i++){ + if(actualOutput[i] != -1){ + return false; + } + } + + return true; + } + + void recordOutput(String fileName, boolean result) { + if (fileName == null) { + System.out.print("Expected and actual output are"); + + if(result) { + System.out.print(""); + } + else { + System.out.print(" not"); + } + + System.out.print(" the same\n"); + + System.out.print("Expected output:"); + + for(int i = 0; i < outputSize; i++){ + System.out.print(" " + expectedOutput[i]); + } + + System.out.print("\nActual output: "); + + for(int i = 0; i < 2*outputSize; i++){ + System.out.print(" " + actualOutput[i]); + } + + System.out.print("\n"); + return; + } + + try{ + FileWriter fw = new FileWriter(fileName, true); + + fw.write("Test name: " + inputFilename + "\n"); + + fw.write("Expected and actual output are"); + + if(result) { + fw.write(""); + } + else { + fw.write(" not"); + } + + fw.write(" the same\n"); + + fw.write("Expected output:"); + + for(int i = 0; i < outputSize; i++){ + fw.write(" " + expectedOutput[i]); + } + + fw.write("\n"); + + + fw.write("Actual output: "); + + for(int i = 0; i < 2*outputSize; i++){ + fw.write(" " + actualOutput[i]); + } + + fw.write("\n"); + + fw.close(); + + }catch(IOException e){ + System.out.println("Error writing results to file"); + } + } + + void resetActualOutput() { + for (int i = 0; i < actualOutput.length; ++i) { + actualOutput[i] = -1; + } + } +} + +class Wrapper{ + public static void main(String[] args) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, ParseException, InterruptedException { + // parse args + String inputFilenames[] = args[0].split(":"); + String outputFilename = args[1]; + String badOutputFilename = args[2]; + int nFunctionRepeats = Integer.parseInt(args[3]); + + Path paths[] = new Path[inputFilenames.length]; + + for (int i = 0; i < inputFilenames.length; i++) { + paths[i] = new Path(inputFilenames[i]); + } + + // To keep track if any of them failed + AtomicBoolean success = new AtomicBoolean(true); + + Thread threads[] = new Thread[paths.length]; + + // Initializing threads + for (int i = 0; i < threads.length; ++i) { + final Path path = paths[i]; + final String tempBadOutputFilename = badOutputFilename + i; + + threads[i] = new Thread(() -> { + boolean threadSuccess = true; + boolean result = true; + TestCase test = new TestCase(); + + for(int j = 0; j < nFunctionRepeats; j++){ + path.resetActualOutput(); + test.testCase(path.dir, path.actualOutput); + + // compare each expected and actual and write out if any are inconsistent + result = path.compare(); + threadSuccess = threadSuccess && result; + + // record bad output in separate file + if (!result){ + path.recordOutput(tempBadOutputFilename, result); + } + + } + + // Update success if any of paths failed in this thread + success.set(success.get() && threadSuccess); + + // Print to console + path.recordOutput(null, result); + }); + + // Catch exceptions occured in threads + threads[i].setUncaughtExceptionHandler((thread, exception) -> { + boolean result = path.compare(); + path.recordOutput(outputFilename, result); + path.recordOutput(badOutputFilename, result); + path.recordOutput(null, result); + + exception.printStackTrace(); + System.exit(1); + }); + } + + // create class + TestCase test = new TestCase(); + + // Running one path to let it JIT compile + for(int i = 0; i < nFunctionRepeats; i++){ + try { + paths[0].resetActualOutput(); + test.testCase(paths[0].dir, paths[0].actualOutput); + } catch (Exception e) { + boolean result = paths[0].compare(); + paths[0].recordOutput(outputFilename, result); + paths[0].recordOutput(badOutputFilename, result); + paths[0].recordOutput(null, result); + + e.printStackTrace(); + System.exit(1); + } + + // compare each expected and actual and write out if any are inconsistent + boolean result = paths[0].compare(); + success.set(success.get() && result); + + // record bad output in separate file + if (!result){ + paths[0].recordOutput(badOutputFilename, result); + } + + } + + // Launching all threads + for (Thread thread : threads) { + thread.start(); + } + + for (Thread thread : threads) { + thread.join(); + } + + for (Path path : paths) { + boolean result = path.compare(); + path.recordOutput(outputFilename, result); + if (!result) paths[0].recordOutput(badOutputFilename, result); + } + + + if(success.get()) { + System.exit(0); + } + + System.exit(1); + + } +} diff --git a/WrapperNoReflection.java b/WrapperNoReflection.java new file mode 100644 index 0000000..c236ebf --- /dev/null +++ b/WrapperNoReflection.java @@ -0,0 +1,255 @@ +import java.io.File; +import java.util.Scanner; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.lang.Math; +import java.lang.reflect.*; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; + +class Path { + String inputFilename; + int dir[]; + int actualOutput[]; + int expectedOutput[]; + int outputSize; + + Path(String inputFilename) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, ParseException { + this.inputFilename = inputFilename; + + // get direction size and expected output size from file + JSONParser parser = new JSONParser(); + JSONObject jsonObject = (JSONObject) parser.parse(new FileReader(inputFilename)); + + JSONArray jsonDir = (JSONArray) jsonObject.get("dirs"); + JSONArray jsonOutput = (JSONArray) jsonObject.get("expected_output"); + + outputSize = jsonOutput.size(); + int dirSize = jsonDir.size(); + + // create arrays + dir = new int[dirSize]; + actualOutput = new int[2*outputSize]; + expectedOutput = new int[outputSize]; + + // fill arrays + int counter = 0; + for (Object o : jsonDir) { + dir[counter] = (int) (long) o; + counter++; + } + counter = 0; + for (Object o : jsonOutput) { + expectedOutput[counter] = (int) (long) o; + counter++; + } + for (int i = outputSize; i < 2*outputSize; i++) { + actualOutput[i] = -1; + } + } + + boolean compare() { + // check actual and expected path are the same + for(int i = 0; i < outputSize; i++){ + if(expectedOutput[i] != actualOutput[i]){ + return false; + } + } + + // check no overflow occurred + for(int i = outputSize; i < 2*outputSize; i++){ + if(actualOutput[i] != -1){ + return false; + } + } + + return true; + } + + void recordOutput(String fileName, boolean result) { + if (fileName == null) { + System.out.print("Expected and actual output are"); + + if(result) { + System.out.print(""); + } + else { + System.out.print(" not"); + } + + System.out.print(" the same\n"); + + System.out.print("Expected output:"); + + for(int i = 0; i < outputSize; i++){ + System.out.print(" " + expectedOutput[i]); + } + + System.out.print("\nActual output: "); + + for(int i = 0; i < 2*outputSize; i++){ + System.out.print(" " + actualOutput[i]); + } + + System.out.print("\n"); + return; + } + + try{ + FileWriter fw = new FileWriter(fileName, true); + + fw.write("Test name: " + inputFilename + "\n"); + + fw.write("Expected and actual output are"); + + if(result) { + fw.write(""); + } + else { + fw.write(" not"); + } + + fw.write(" the same\n"); + + fw.write("Expected output:"); + + for(int i = 0; i < outputSize; i++){ + fw.write(" " + expectedOutput[i]); + } + + fw.write("\n"); + + + fw.write("Actual output: "); + + for(int i = 0; i < 2*outputSize; i++){ + fw.write(" " + actualOutput[i]); + } + + fw.write("\n"); + + fw.close(); + + }catch(IOException e){ + System.out.println("Error writing results to file"); + } + } + + void resetActualOutput() { + for (int i = 0; i < actualOutput.length; ++i) { + actualOutput[i] = -1; + } + } +} + +class Wrapper{ + public static void main(String[] args) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, ParseException { + + // parse args + String inputFilenames[] = args[0].split(":"); + String outputFilename = args[1]; + String badOutputFilename = args[2]; + int nFunctionRepeats = Integer.parseInt(args[3]); + + Path paths[] = new Path[inputFilenames.length]; + + for (int i = 0; i < inputFilenames.length; i++) { + paths[i] = new Path(inputFilenames[i]); + } + + boolean result = true; + boolean success = true; + + // create class + TestCase test = new TestCase(); + + int index = (int) (Math.random() * paths.length); + + // repeat function to induce JIT + for(int i = 0; i < nFunctionRepeats; i++){ + try { + paths[index].resetActualOutput(); + test.testCase(paths[index].dir, paths[index].actualOutput); + } catch (Exception e) { + result = paths[index].compare(); + paths[index].recordOutput(outputFilename, result); + paths[index].recordOutput(badOutputFilename, result); + paths[index].recordOutput(null, result); + + e.printStackTrace(); + System.exit(1); + } + + // compare each expected and actual and write out if any are inconsistent + result = paths[index].compare(); + success = success && result; + + // record bad output in separate file + if (!result){ + paths[index].recordOutput(badOutputFilename, result); + } + + } + + // record all output + paths[index].recordOutput(outputFilename, result); + + // print outcomes + paths[index].recordOutput(null, result); + + // if multiple paths provided, change the path to trigger deoptimization + if (paths.length >= 2) { + int repeat = (int) (Math.random() * paths.length * 1.5) + 1; + + for (int i = 0; i < repeat; ++i) { + System.out.println("Switching paths"); + index = (int) (Math.random() * paths.length); + + // repeat function to induce JIT + for(int j = 0; j < nFunctionRepeats; j++){ + try { + paths[index].resetActualOutput(); + test.testCase(paths[index].dir, paths[index].actualOutput); + } catch (Exception e) { + result = paths[index].compare(); + paths[index].recordOutput(outputFilename, result); + paths[index].recordOutput(badOutputFilename, result); + paths[index].recordOutput(null, result); + + e.printStackTrace(); + System.exit(1); + } + + // compare each expected and actual and write out if any are inconsistent + result = paths[index].compare(); + success = success && result; + + // record bad output in separate file + if (!result){ + paths[index].recordOutput(badOutputFilename, result); + } + + } + + // record all output + paths[index].recordOutput(outputFilename, result); + + // print outcomes + paths[index].recordOutput(null, result); + } + } + + + if(success) { + System.exit(0); + } + + System.exit(1); + + } +} diff --git a/java_change_dir/run_template.sh b/java_change_dir/run_template.sh new file mode 100755 index 0000000..e92cf09 --- /dev/null +++ b/java_change_dir/run_template.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +ACTION='fuzz_with_changing_paths' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=. +LANG='java' +COMPILER='hotspot' +JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +# JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' + +. $SRC/venv/bin/activate + +PYTHONPATH=$SRC/fuzzflesh + +mkdir -p $OUTPUT + +python3 -m fuzzflesh \ + $ACTION \ + -base=$OUTPUT \ + -graphs=1 \ + -paths=3 \ + -min_size=3 \ + -max_size=10000 \ + $LANG \ + $COMPILER \ + $JVM \ + $JSON \ + # --reflection + diff --git a/java_change_dir_concurrent/run_template.sh b/java_change_dir_concurrent/run_template.sh new file mode 100755 index 0000000..e92cf09 --- /dev/null +++ b/java_change_dir_concurrent/run_template.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +ACTION='fuzz_with_changing_paths' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=. +LANG='java' +COMPILER='hotspot' +JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +# JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' + +. $SRC/venv/bin/activate + +PYTHONPATH=$SRC/fuzzflesh + +mkdir -p $OUTPUT + +python3 -m fuzzflesh \ + $ACTION \ + -base=$OUTPUT \ + -graphs=1 \ + -paths=3 \ + -min_size=3 \ + -max_size=10000 \ + $LANG \ + $COMPILER \ + $JVM \ + $JSON \ + # --reflection + diff --git a/java_change_dir_graal/run_template.sh b/java_change_dir_graal/run_template.sh new file mode 100755 index 0000000..a187d61 --- /dev/null +++ b/java_change_dir_graal/run_template.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +ACTION='fuzz_with_changing_paths' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=. +LANG='java' +COMPILER='hotspot' +# JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' + +. $SRC/venv/bin/activate + +PYTHONPATH=$SRC/fuzzflesh + +mkdir -p $OUTPUT + +python3 -m fuzzflesh \ + $ACTION \ + -base=$OUTPUT \ + -graphs=1 \ + -paths=3 \ + -min_size=3 \ + -max_size=10000 \ + $LANG \ + $COMPILER \ + $JVM \ + $JSON \ + # --reflection + diff --git a/java_change_dir_graal_concurrent/run_template.sh b/java_change_dir_graal_concurrent/run_template.sh new file mode 100755 index 0000000..a187d61 --- /dev/null +++ b/java_change_dir_graal_concurrent/run_template.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +ACTION='fuzz_with_changing_paths' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=. +LANG='java' +COMPILER='hotspot' +# JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' + +. $SRC/venv/bin/activate + +PYTHONPATH=$SRC/fuzzflesh + +mkdir -p $OUTPUT + +python3 -m fuzzflesh \ + $ACTION \ + -base=$OUTPUT \ + -graphs=1 \ + -paths=3 \ + -min_size=3 \ + -max_size=10000 \ + $LANG \ + $COMPILER \ + $JVM \ + $JSON \ + # --reflection + diff --git a/java_change_dir_graal_small/run_template.sh b/java_change_dir_graal_small/run_template.sh new file mode 100755 index 0000000..bcdca04 --- /dev/null +++ b/java_change_dir_graal_small/run_template.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +ACTION='fuzz_with_changing_paths' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=. +LANG='java' +COMPILER='hotspot' +# JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' + +. $SRC/venv/bin/activate + +PYTHONPATH=$SRC/fuzzflesh + +mkdir -p $OUTPUT + +python3 -m fuzzflesh \ + $ACTION \ + -base=$OUTPUT \ + -graphs=1 \ + -paths=3 \ + -min_size=3 \ + -max_size=10 \ + $LANG \ + $COMPILER \ + $JVM \ + $JSON \ + # --reflection + diff --git a/javabc_change_dir/run_template.sh b/javabc_change_dir/run_template.sh new file mode 100755 index 0000000..e1de706 --- /dev/null +++ b/javabc_change_dir/run_template.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +ACTION='fuzz_with_changing_paths' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=. +LANG='javabc' +COMPILER='hotspot' +JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +# JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' + +. $SRC/venv/bin/activate + +PYTHONPATH=$SRC/fuzzflesh + +mkdir -p $OUTPUT + +python3 -m fuzzflesh \ + $ACTION \ + -base=$OUTPUT \ + -graphs=1 \ + -paths=3 \ + -min_size=3 \ + -max_size=100 \ + $LANG \ + $COMPILER \ + $JVM \ + $JASMIN \ + $JSON \ + # --reflection + diff --git a/javabc_change_dir_graal/run_template.sh b/javabc_change_dir_graal/run_template.sh new file mode 100755 index 0000000..7d7b17d --- /dev/null +++ b/javabc_change_dir_graal/run_template.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +ACTION='fuzz_with_changing_paths' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=. +LANG='javabc' +COMPILER='hotspot' +# JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' + +. $SRC/venv/bin/activate + +PYTHONPATH=$SRC/fuzzflesh + +mkdir -p $OUTPUT + +python3 -m fuzzflesh \ + $ACTION \ + -base=$OUTPUT \ + -graphs=1 \ + -paths=3 \ + -min_size=3 \ + -max_size=100 \ + $LANG \ + $COMPILER \ + $JVM \ + $JASMIN \ + $JSON \ + # --reflection + diff --git a/javabc_change_dir_graal_small/run_template.sh b/javabc_change_dir_graal_small/run_template.sh new file mode 100755 index 0000000..0a26995 --- /dev/null +++ b/javabc_change_dir_graal_small/run_template.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +ACTION='fuzz_with_changing_paths' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=. +LANG='javabc' +COMPILER='hotspot' +# JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' + +. $SRC/venv/bin/activate + +PYTHONPATH=$SRC/fuzzflesh + +mkdir -p $OUTPUT + +python3 -m fuzzflesh \ + $ACTION \ + -base=$OUTPUT \ + -graphs=1 \ + -paths=3 \ + -min_size=10 \ + -max_size=11 \ + $LANG \ + $COMPILER \ + $JVM \ + $JASMIN \ + $JSON \ + # --reflection + diff --git a/javabc_dir/run_template.sh b/javabc_dir/run_template.sh new file mode 100755 index 0000000..84d5c13 --- /dev/null +++ b/javabc_dir/run_template.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +ACTION='fuzz' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=. +LANG='javabc' +COMPILER='hotspot' +JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +# JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' + +. $SRC/venv/bin/activate + +PYTHONPATH=$SRC/fuzzflesh + +mkdir -p $OUTPUT + +python3 -m fuzzflesh \ + $ACTION \ + -base=$OUTPUT \ + -graphs=1 \ + -paths=2 \ + -min_size=3 \ + -max_size=100 \ + --dirs \ + $LANG \ + $COMPILER \ + $JVM \ + $JASMIN \ + $JSON \ + # --reflection + diff --git a/javabc_no_dir/run_template.sh b/javabc_no_dir/run_template.sh new file mode 100755 index 0000000..de31f3a --- /dev/null +++ b/javabc_no_dir/run_template.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +ACTION='fuzz' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=. +LANG='javabc' +COMPILER='hotspot' +JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +# JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' + +. $SRC/venv/bin/activate + +PYTHONPATH=$SRC/fuzzflesh + +mkdir -p $OUTPUT + +python3 -m fuzzflesh \ + $ACTION \ + -base=$OUTPUT \ + -graphs=1 \ + -paths=2 \ + -min_size=3 \ + -max_size=100 \ + $LANG \ + $COMPILER \ + $JVM \ + $JASMIN \ + $JSON \ + # --reflection + diff --git a/run_template.sh b/run_template.sh index 8795fd2..48c6bca 100755 --- a/run_template.sh +++ b/run_template.sh @@ -1,27 +1,37 @@ #!/bin/sh -SRC='/data/dev/fuzzflesh/src' -ACTION='fuzz' -OUTPUT='/data/work/fuzzflesh/dev_testing/output' -LANG='javabc' -COMPILER='cfr' -JVM='/usr/lib/jvm/java-19-openjdk-amd64/bin' -JASMIN='/homes/agg22/dev/jasmin-2.4' -JSON='/data/dev/java/json-simple-1.1.1.jar' -DECOMPILER_PATH='/vol/bitbucket/agg22/cfr-0.152.jar' +TIMESTAMP=$(date +%s) +NAME=$(hostname) +MACHINE=${NAME%%.*} + +SRC='/homes/rk1923/control_flow_fleshing/src' +# ACTION='fuzz' +ACTION='fuzz_with_changing_paths' +# OUTPUT=/vol/bitbucket/rk1923/UROP/output/${MACHINE}_${TIMESTAMP} +OUTPUT=./output +LANG='java' +COMPILER='hotspot' +JVM='/usr/lib/jvm/java-17-openjdk-amd64/bin' +# JVM='/vol/bitbucket/rk1923/UROP/graalvm-jdk-22.0.2+9.1/bin' +JASMIN='/vol/bitbucket/rk1923/UROP/jasmin-2.4' +JSON='/vol/bitbucket/rk1923/UROP/json-simple-1.1.1.jar' . $SRC/venv/bin/activate PYTHONPATH=$SRC/fuzzflesh +mkdir -p $OUTPUT + python3 -m fuzzflesh \ $ACTION \ -base=$OUTPUT \ - --dirs \ + -graphs=1 \ + -paths=3 \ + -min_size=3 \ + -max_size=10000 \ $LANG \ $COMPILER \ $JVM \ - $JASMIN \ $JSON \ - $DECOMPILER_PATH \ - --reflection + # --reflection + diff --git a/src/fuzzflesh/__main__.py b/src/fuzzflesh/__main__.py index c9c0ac1..64aa0b2 100644 --- a/src/fuzzflesh/__main__.py +++ b/src/fuzzflesh/__main__.py @@ -8,19 +8,21 @@ from fuzzflesh.common.utils import Lang, Compiler, Program, RunnerReturn from fuzzflesh.graph_generator.generator import generate_graph from fuzzflesh.program_generator.flesher import ProgramFlesher +from fuzzflesh.program_generator.java.java_generator import JavaProgramGenerator from fuzzflesh.program_generator.javabc.javabc_generator import JavaBCProgramGenerator from fuzzflesh.program_generator.c.c_generator import CProgramGenerator from fuzzflesh.harness.runner import Runner +from fuzzflesh.harness.java.java_runner import JavaRunner from fuzzflesh.harness.javabc.javabc_runner import JavaBCRunner from fuzzflesh.harness.c.c_runner import CRunner from fuzzflesh.cfg.CFG import CFG, Route -def main(): +def main(): parser = argparse.ArgumentParser() # Common parser args - parser.add_argument("action", choices=['gen', 'run', 'fuzz'], + parser.add_argument("action", choices=['gen', 'run', 'fuzz', 'fuzz_with_changing_paths'], help='''Gen produces graphs and paths. Run runs existing graphs and paths. Fuzz combines gen and run.''') @@ -112,6 +114,26 @@ def main(): default = None, help = 'Path to decompiler under test.') + # Java parser args + java_parser = subparsers.add_parser('java', + help='Java bytecode help') + java_parser.add_argument("compiler", + choices=['cfr','fernflower','procyon','hotspot','graalvm'], + help='Compiler / decompiler toolchain under test.') + java_parser.add_argument("jvm", + type=str, + help="Path to JVM to be used") + java_parser.add_argument("json", + type=str, + help="Path to json simple jar") + java_parser.add_argument("-compiler_path", + default = None, + type=str, + help="Path to the (de)compiler under test. Not required for testing JIT compiler e.g. HotSpot where the software under test is in the JVM path") + java_parser.add_argument("--reflection", + action=argparse.BooleanOptionalAction, + help='Use reflection instead of static compilation.') + args = parser.parse_args() language : Lang = Lang[args.language.upper()] @@ -143,6 +165,15 @@ def main(): elif args.action == 'fuzz': (graph, programs, paths) = gen(args, language, graph_dir, graph_id) run(args, language, compiler, programs, paths, base_dir, graph) + + elif args.action == 'fuzz_with_changing_paths': + assert(language == Lang.JAVABC or language == Lang.JAVA) + assert(compiler == Compiler.HOTSPOT) + assert(args.paths >= 2) + + args.dirs = False + (graph, programs, paths) = gen(args, language, graph_dir, graph_id) + run_with_changing_paths(args, language, compiler, programs, paths, base_dir, graph) def gen(args, language : Lang, graph_dir : Path, graph_id : int,) -> tuple[Path, Path, list[Path]]: @@ -150,7 +181,7 @@ def gen(args, language : Lang, graph_dir : Path, graph_id : int,) -> tuple[Path, print(f'Generating graph {graph_id}...') filepath = Path(graph_dir, f'graph_{graph_id}') - + filepath.mkdir(exist_ok=True) graph = generate_graph(min_graph_size = args.min_size, @@ -252,8 +283,81 @@ def run(args, language : Lang, compiler : Compiler, programs : list[Path], paths delete_path(path) else: - delete_program(prog, language) graph_has_failed = True + + if not graph_has_failed: + delete_program(prog, language) + + + # If no programs associated with this graph fail, then tidy up by removing the graph + if not graph_has_failed: + delete_graph(graph_path) + + +def run_with_changing_paths(args, language : Lang, compiler : Compiler, programs : list[Path], paths : list[Path], base_dir : Path, graph_path : Path): + assert(not args.dirs) + + if (language == Lang.JAVABC): + runner = JavaBCRunner(compiler, + Path(args.jvm), + Path(args.jasmin), + Path(args.json), + base_dir, + None, + args.reflection) + else: + runner = JavaRunner(compiler, + Path(args.jvm), + Path(args.json), + base_dir, + None, + args.reflection) + + # Indicator for whether any program derived from this particular graph has failed + graph_has_failed : bool = False + + if args.dirs: + assert(len(programs)==len(paths)) + + for (i, prog) in enumerate(programs): + + # Compile + compile_result = runner.compile(program=prog) + print(f'Result: {compile_result}') + + if compile_result == RunnerReturn.COMPILATION_FAIL: + log(compile_result) + return + + # Execute a single path with a single program + # We pass the path so that the runner can compare the expected and actual result + if args.dirs: + exe_result = runner.execute(program=prog, path=paths[i]) + + print(f'Result: {exe_result}') + + if exe_result == RunnerReturn.SUCCESS and args.tidy: + delete_program(prog, language) + delete_path(path=paths[i]) + + else: + graph_has_failed = True + + # Execute multiple paths with a single program + else: + exe_result = runner.execute_with_changing_paths(program=prog, paths=paths) + + print(f'Result: {exe_result}') + if exe_result != RunnerReturn.SUCCESS: + log(exe_result) + + if exe_result == RunnerReturn.SUCCESS and args.tidy: + for path in paths: + delete_path(path) + delete_program(prog, language) + + else: + graph_has_failed = True # If no programs associated with this graph fail, then tidy up by removing the graph if not graph_has_failed: @@ -275,6 +379,13 @@ def delete_program(program : Path, language : Lang): case Lang.C: cmd = ['rm', '-f', f'{str(program.parent)}/{str(program.stem)}*'] + case Lang.JAVA: + cmd = ['rm', '-rf', f'{str(program.parent)}/{str(program.stem)}'] + result = subprocess.run(cmd) + + cmd = ['rm', '-rf', f'{str(program.parent)}/{str(program.stem)}.java'] + result = subprocess.run(cmd) + def delete_graph(graph : Path): ''' Deletes graph and graph folder @@ -295,6 +406,8 @@ def create_folders(args, base_dir : Path, language : Lang, wrapper_dir : Path) - return create_javabc_folders(base_dir, args.reflection, wrapper_dir) case Lang.C: return create_c_folders(base_dir, wrapper_dir) + case Lang.JAVA: + return create_javabc_folders(base_dir, args.reflection, wrapper_dir) return False @@ -350,6 +463,9 @@ def get_flesher(args, language : Lang, cfg : CFG) -> ProgramFlesher: case Lang.C: return CProgramGenerator(cfg, args.dirs) + + case Lang.JAVA: + return JavaProgramGenerator(cfg, args.dirs, args.reflection) return None @@ -373,6 +489,14 @@ def get_runner(args, language : Lang, compiler : Compiler, base_dir : Path) -> R base_dir, Path(args.include_path), args.dirs) + case Lang.JAVA: + path = Path(args.compiler_path) if compiler != Compiler.HOTSPOT else None + return JavaRunner(compiler, + Path(args.jvm), + Path(args.json), + base_dir, + path, + args.reflection) return None def paths_to_dict(all_paths : list[Route]) -> dict: @@ -396,5 +520,10 @@ def get_compiler(compiler : str) -> Compiler: else: return Compiler[compiler.upper()] +def log(result: RunnerReturn): + with open('/vol/bitbucket/rk1923/UROP/outputlog.txt', 'a') as file: + file.write(f"Result: {result}\n") + + if __name__ == "__main__": main() diff --git a/src/fuzzflesh/analyzer/analyze.py b/src/fuzzflesh/analyzer/analyze.py new file mode 100644 index 0000000..dffce19 --- /dev/null +++ b/src/fuzzflesh/analyzer/analyze.py @@ -0,0 +1,315 @@ +import xml.etree.ElementTree as ET +from xml.etree.ElementTree import ElementTree, Element +import json +import re +import numpy as np + +METHOD = 'method' +TASK_QUEUED = 'task_queued' +TASK = 'task' +MAKE_NOT_ENTRANT = 'make_not_entrant' +NMETHOD = 'nmethod' +COMPILE_ID = 'compile_id' +LEVEL = 'level' +IICOUNT = 'iicount' +STAMP = 'stamp' +UNCOMMON_TRAP = 'uncommon_trap' +REASON = 'reason' +FAILURE = 'failure' +COMPILATION_PATTERN = 'compilation pattern' +COMPILATION = 'compilation' +INVOCATION_QUEUED = 'invocations till queued' +TIME_QUEUED = 'time stamp queued' +INVOCATION_COMPILED = 'invocations till compiled' +TIME_COMPILED = 'time stamp compiled' +UNCOMMON_TRAPS = 'uncommon traps' +COMPILATION_DETAILS = 'compilation details' +FAILURE_REASON = 'failure reason' +TIME_STAMP = 'time stamp' +NAME = 'name' +INLINE_SUCCESS = 'inline_success' +INLINE_FAILURE = 'inline_fail' +INLINE = 'inline' +FAILED_INLINE = 'failed inline' +PARSE = 'parse' +FRAGMENT = 'fragment' + +MEAN = 'mean' +MEDIAN = 'median' +MIN = 'min' +MAX = 'max' +STDEV = 'standard deviation' + + +class Data: + def __init__(self): + self.filepath: str = '' + self.compiler: str = '' + self.single_threaded: bool = True + self.jit_compiled: bool = False + self.deoptimized: bool = False + self.number_of_failed_compilation: int = 0 + self.inline_count: int = 0 + self.failed_inline_count: int = 0 + self.compilation_count: dict = {} + self.compilation_patterns: dict = {} + self.uncommon_traps_count: dict = {} + self.invocation_queued_level: dict = {} + self.invocation_compiled_level: dict = {} + + def to_dict(self) -> dict: + return { + 'file path': self.filepath, + 'compiler': self.compiler, + 'single threaded': self.single_threaded, + 'deoptimized': self.deoptimized, + 'number of failed compilation': self.number_of_failed_compilation, + 'number of successful inline': self.inline_count, + 'number of failed inline': self.failed_inline_count, + 'frequency of compilation at each level': self.compilation_count, + 'frequency of each compilation pattern': self.compilation_patterns, + 'frequency of each uncommon trap': self.uncommon_traps_count, + 'stats on number of invocation until method gets added to compile queue': self.invocation_queued_level, + 'stats on number of invocation until method gets compiled': self.invocation_compiled_level, + + } + +class Compilation: + def __init__(self): + self.method_name: str = '' + self.queued_task: Element = None + self.compiled_task: Element = None + self.nmethod: Element = None + self.make_not_entrant: Element = None + + + +class Log: + def __init__(self, + _filepath: str, + _compiler_name: str, + _single_threaded: bool): + + self.filepath: str = _filepath + self.compiler_name: str = _compiler_name + self.single_threaded: bool = _single_threaded + + def analyze(self, index: int, output_dir: str) -> dict: + tree = ET.parse(self.filepath) + + is_method_in_test_case = lambda task: task.attrib[METHOD].startswith('TestCase') + + # Get a list of elements with tag task_queued and is a meber function of TestCase class + queued_tasks = list(filter(is_method_in_test_case, tree.iter(TASK_QUEUED))) + + # Get a list of elements with tag task and is a meber function of TestCase class + compiled_tasks = list(filter(is_method_in_test_case, tree.iter(TASK))) + + # Get a list of elements with tag nmethod and is a meber function of TestCase class + nmethods = list(filter(is_method_in_test_case, tree.iter(NMETHOD))) + + # Dictionary of compile_id to compilation object which stores queued_task, compiled_task, etc. + compilation_dict = {} + + # Set up compile id to method dictionary + for task in queued_tasks: + match = re.search(r"TestCase (.*?) .*", task.attrib[METHOD]) + method_name = match.group(1) + compile_id = task.attrib[COMPILE_ID] + + level = Compilation() + level.method_name = method_name + level.queued_task = task + + compilation_dict[compile_id] = level + + for task in compiled_tasks: + compile_id = task.attrib[COMPILE_ID] + compilation_dict[compile_id].compiled_task = task + + for task in nmethods: + compile_id = task.attrib[COMPILE_ID] + compilation_dict[compile_id].nmethod = task + + make_not_entrants = list(filter(lambda task: task.attrib[COMPILE_ID] in compilation_dict, tree.iter(MAKE_NOT_ENTRANT))) + + for task in make_not_entrants: + compile_id = task.attrib[COMPILE_ID] + compilation_dict[compile_id].make_not_entrant = task + + + # Store processed log + intermediate = {} + + for id, compilation in compilation_dict.items(): + method = intermediate.setdefault(compilation.method_name, {}) + if compilation.nmethod is not None: + level = compilation.nmethod.attrib[LEVEL] + method.setdefault(COMPILATION_PATTERN, []).append(level) + + compilation_details = { + COMPILATION: level, + INVOCATION_QUEUED: int(compilation.queued_task.attrib[IICOUNT]), + TIME_QUEUED: float(compilation.queued_task.attrib[STAMP]), + INVOCATION_COMPILED: int(compilation.compiled_task.attrib[IICOUNT]), + TIME_COMPILED: float(compilation.compiled_task.attrib[STAMP]) + } + + # Handle uncommon traps + traps = list(compilation.compiled_task.iter(UNCOMMON_TRAP)) + + if traps: + traps_dict = {} + + for trap in traps: + trap_name = trap.attrib[REASON] + traps_dict.setdefault(trap_name, 0) + traps_dict[trap_name] += 1 + + compilation_details[UNCOMMON_TRAPS] = traps_dict + + # Handle inlining + iter = compilation.compiled_task.iter() + + for element in iter: + if element.tag == PARSE: + break + + for element in iter: + if element.tag == METHOD: + method_name = element.attrib[NAME] + while element.tag not in [INLINE_SUCCESS, INLINE_FAILURE]: + element = next(iter) + + if element.tag == INLINE_SUCCESS: + compilation_details.setdefault(INLINE, []).append(method_name) + else: + compilation_details.setdefault(FAILED_INLINE, []).append(method_name) + + + method.setdefault(COMPILATION_DETAILS, []).append(compilation_details) + else: + level = (compilation.compiled_task.attrib[LEVEL] if LEVEL in compilation.compiled_task.attrib else '4') + ' (failed)' + method.setdefault(COMPILATION_PATTERN, []).append(level) + + compilation_details = { + COMPILATION: level + } + compilation_details[FAILURE_REASON] = compilation.compiled_task.find(FAILURE).attrib[REASON] + compilation_details[TIME_STAMP] = float(compilation.compiled_task.attrib[STAMP]) + + method.setdefault(COMPILATION_DETAILS, []).append(compilation_details) + + if compilation.make_not_entrant is not None: + level = compilation.make_not_entrant.attrib[LEVEL] + " (made not entrant)" + method.setdefault(COMPILATION_PATTERN, []).append(level) + + compilation_details = { + COMPILATION: level + } + compilation_details[TIME_STAMP] = float(compilation.make_not_entrant.attrib[STAMP]) + + method.setdefault(COMPILATION_DETAILS, []).append(compilation_details) + + + + # for key, val in data.items(): + # print(key + ": " +str(val)) + + intermediate_file = output_dir + f'/intermediate_{index}.json' + with open(intermediate_file, 'w') as file: + json.dump(intermediate, file, indent = 4) + + + + # Calculate statistics based on the processed log + data = Data() + data.filepath = self.filepath + data.compiler = self.compiler_name + data.single_threaded = self.single_threaded + data.jit_compiled = len(queued_tasks) != 0 + data.deoptimized = len(make_not_entrants) != 0 + data.number_of_failed_compilation = 0 + + invocation_queued_dict = {} + invocation_compiled_dict = {} + + for _, method in intermediate.items(): + for compilation_details in method[COMPILATION_DETAILS]: + compilation = compilation_details[COMPILATION] + + data.compilation_count.setdefault(compilation, 0) + data.compilation_count[compilation] += 1 + + if compilation.endswith("(failed)"): + data.number_of_failed_compilation += 1 + + if INVOCATION_QUEUED in compilation_details: + invocation_queued_dict.setdefault(compilation, []).append(compilation_details[INVOCATION_QUEUED]) + if INVOCATION_COMPILED in compilation_details: + invocation_compiled_dict.setdefault(compilation, []).append(compilation_details[INVOCATION_COMPILED]) + + if UNCOMMON_TRAPS in compilation_details: + for uncommon_trap, count in compilation_details[UNCOMMON_TRAPS].items(): + data.uncommon_traps_count.setdefault(uncommon_trap, 0) + data.uncommon_traps_count[uncommon_trap] += count + + if INLINE in compilation_details: + data.inline_count += len(compilation_details[INLINE]) + if FAILED_INLINE in compilation_details: + data.failed_inline_count += len(compilation_details[FAILED_INLINE]) + + compilation_pattern = ', '.join(method[COMPILATION_PATTERN]) + data.compilation_patterns.setdefault(compilation_pattern, 0) + data.compilation_patterns[compilation_pattern] += 1 + + for compilation, invocation_queued in invocation_queued_dict.items(): + stats = { + MEAN: np.average(invocation_queued), + MEDIAN: np.median(invocation_queued), + MIN: int(np.min(invocation_queued)), + MAX: int(np.max(invocation_queued)), + STDEV: np.std(invocation_queued) + } + + data.invocation_queued_level[compilation] = stats + + for compilation, invocation_compiled in invocation_compiled_dict.items(): + stats = { + MEAN: np.average(invocation_compiled), + MEDIAN: np.median(invocation_compiled), + MIN: int(np.min(invocation_compiled)), + MAX: int(np.max(invocation_compiled)), + STDEV: np.std(invocation_compiled) + } + + data.invocation_compiled_level[compilation] = stats + + return data.to_dict() + + +def setup_logs(): + logs: list[Log] = [ + Log('/homes/rk1923/control_flow_fleshing/java_change_dir/out/graph_0/prog_0/hotspot.log', 'OpenJDK', True), + Log('/homes/rk1923/control_flow_fleshing/java_change_dir_concurrent/out/graph_0/prog_0/hotspot.log', 'OpenJDK', False), + Log('/homes/rk1923/control_flow_fleshing/java_change_dir_graal/out/graph_0/prog_0/hotspot.log', 'graalVM', True), + Log('/homes/rk1923/control_flow_fleshing/java_change_dir_graal_concurrent/out/graph_0/prog_0/hotspot.log', 'graalVM', False), + Log('/homes/rk1923/control_flow_fleshing/java_change_dir_graal_small/out/graph_0/prog_0/hotspot.log', 'graalVM', True), + Log('/homes/rk1923/control_flow_fleshing/javabc_change_dir/out/graph_0/prog_0/hotspot.log', 'OpenJDK', True), + Log('/homes/rk1923/control_flow_fleshing/javabc_change_dir_graal/out/graph_0/prog_0/hotspot.log', 'graalVM', True), + Log('/homes/rk1923/control_flow_fleshing/javabc_change_dir_graal_small/out/graph_0/prog_0/hotspot.log', 'graalVM', True), + ] + + + output_dir = '/homes/rk1923/control_flow_fleshing/analysis' + + analyses = [] + for index, log in enumerate(logs): + analyses.append(log.analyze(index, output_dir)) + + output_file = output_dir + '/analysis.json' + with open(output_file, 'w') as file: + json.dump(analyses, file, indent = 4) + +if __name__ == "__main__": + setup_logs() \ No newline at end of file diff --git a/src/fuzzflesh/common/utils.py b/src/fuzzflesh/common/utils.py index 6f02faa..261ebd7 100644 --- a/src/fuzzflesh/common/utils.py +++ b/src/fuzzflesh/common/utils.py @@ -4,6 +4,7 @@ class Lang(Enum): JAVABC = 1 C = 2 + JAVA = 3 class Compiler(Enum): CFR = 1 @@ -54,6 +55,8 @@ def get_extension(self) -> str: return '.j' case Lang.C: return '.c' + case Lang.JAVA: + return '.java' case _ : return '.txt' diff --git a/src/fuzzflesh/harness/java/__init__.py b/src/fuzzflesh/harness/java/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/fuzzflesh/harness/java/java_runner.py b/src/fuzzflesh/harness/java/java_runner.py new file mode 100644 index 0000000..230b60a --- /dev/null +++ b/src/fuzzflesh/harness/java/java_runner.py @@ -0,0 +1,292 @@ +import subprocess +from pathlib import Path + +from fuzzflesh.harness.runner import Runner +from fuzzflesh.common.utils import Compiler, Lang, RunnerReturn + +def get_class_location(program : Path) -> str: + return f'{str(program.parent)}/{str(program.stem)}' + +class JavaRunner(Runner): + + def __init__(self, + _toolchain : Compiler, + _jvm : Path, + _json : Path, + _output : Path, + _compiler_path : Path, + _reflection : bool): + super(Runner, self).__init__() + self.compiler_name : Compiler = _toolchain + self.compiler_path : Path = None + self.output : Path = _output + self.jvm : Path = Path(_jvm, 'java') + self.javac : Path = Path(_jvm, 'javac') + self.json_jar : Path = Path(_json) + self.wrapper : Path = Path(_output, 'testing/Wrapper.java') if _reflection else Path(_output,'Wrapper.java') + self.interface : Path = Path(_output, 'testing/TestCaseInterface.java') + self.n_function_repeats : int = 300000 + self.reflection : bool = _reflection + + @property + def language(self): + return Lang.JAVA + + @property + def toolchain(self): + return self.compiler_name + + def compile(self, program : Path) -> RunnerReturn: + + class_location = get_class_location(program) + + if self.is_decompiler(): + + # We compile, decompile, and re-compile the program + print('Compiling...') + self.compile_test(program) + print('Decompiling...') + self.decompile_test(Path(class_location,'TestCase.class')) + print('Recompiling...') + self.recompile_test(Path(class_location,'TestCase.java')) + + else: + print('Compiling...') + return self.compile_test(program) + + def execute(self, program :Path, path : Path) -> RunnerReturn: + class_location = get_class_location(program) + + return self.execute_test(program, path, class_location) + + def execute_with_changing_paths(self, program : Path, paths : list[Path]) -> RunnerReturn: + class_location = get_class_location(program) + + path = ":".join(map(str, paths)) + + if self.reflection: + exe_cmd = [f'{self.jvm}', + '-XX:+UnlockDiagnosticVMOptions', + # '-XX:CompileCommand=print,testing.TestCase::testCase', + # '-XX:+PrintCompilation', + # '-XX:+LogCompilation', + # f'-XX:LogFile={class_location}/hotspot.log', + '-cp', + f':{self.output}:{class_location}:{self.json_jar}', + f'testing/Wrapper', + f'testing.TestCase', + path, + f'{class_location}/output.txt', + f'{class_location}/bad_output.txt', + f'{self.n_function_repeats}', + '-XX:CompileThreshold=100'] + + else: + exe_cmd = [f'{self.jvm}', + # '-Dgraal.PrintCompilation=true', + # '-Djdk.graal.Dump=:1', + # f'-Dgraal.LogFile={class_location}/graal.log', + + # '-Dgraal.MaxDuplicationFactor=100000.0', + '-XX:+UnlockDiagnosticVMOptions', + # # '-XX:CompileCommand=print,TestCase::testCase', + # '-XX:+PrintAssembly', + # '-XX:+PrintCompilation', + '-XX:+LogCompilation', + f'-XX:LogFile={class_location}/hotspot.log', + '-cp', + f':{class_location}:{self.json_jar}', + 'Wrapper', + path, + f'{class_location}/output.txt', + f'{class_location}/bad_output.txt', + f'{self.n_function_repeats}', + # '-XX:CompileThreshold=100' + ] + + result = subprocess.run(exe_cmd) + + return RunnerReturn.EXECUTION_FAIL if result.returncode != 0 else RunnerReturn.SUCCESS + + + def is_decompiler(self): + if self.toolchain in [Compiler.CFR,Compiler.PROCYON,Compiler.FERNFLOWER]: + return True + + return False + + def compile_test(self, program : Path) -> int: + + class_location = f'{str(program.parent)}/{str(program.stem)}' + + if self.reflection: + return self.compile_test_with_reflection(program, class_location) + + return self.compile_test_without_reflection(program, class_location) + + def compile_test_without_reflection(self, program : Path, class_location : Path) -> RunnerReturn: + + compile_test_result = self.compile_testcase(program, class_location) + + if compile_test_result != 0: + return RunnerReturn.COMPILATION_FAIL + + compile_wrapper_cmd = [str(self.javac), + '-cp', + f':{class_location}:{self.json_jar}', + str(self.wrapper), + '-d', + class_location] + + compile_wrapper_result = subprocess.run(compile_wrapper_cmd) + + if compile_wrapper_result.returncode != 0: + return RunnerReturn.COMPILATION_FAIL + + return RunnerReturn.SUCCESS + + def compile_test_with_reflection(self, program: Path, class_location : Path) -> RunnerReturn: + + interface_cmd = [f'{self.javac}', + f'{self.interface}'] + + interface_result = subprocess.run(interface_cmd) + + compile_test_result = self.compile_testcase(program, class_location) + + if compile_test_result != 0: + return RunnerReturn.COMPILATION_FAIL + + if interface_result.returncode != 0: + print('Interface compilation failed!') + return interface_result.returncode + + wrapper_cmd = [f'{self.javac}', + '-cp', + f':{self.output}:{self.json_jar}', + f'{self.wrapper}'] + + wrapper_result = subprocess.run(wrapper_cmd) + + if wrapper_result.returncode != 0: + print('Wrapper compilation failed!') + return wrapper_result.returncode + + return wrapper_result.returncode + + def compile_testcase(self, program : Path, testcase_location : Path) -> int: + + class_location = f'{str(program.parent)}/{str(program.stem)}' + + compile_test_cmd = [f'{self.javac}', + '-cp', + f'{self.output}', + str(program), + '-d', + str(testcase_location)] + + compile_test_result = subprocess.run(compile_test_cmd) + + return compile_test_result.returncode + + def execute_test(self, program : Path, path : Path, class_location : Path) -> RunnerReturn: + + if self.reflection: + exe_cmd = [f'{self.jvm}', + '-XX:+UnlockDiagnosticVMOptions', + # '-XX:CompileCommand=print,testing.TestCase::testCase', + # '-XX:+PrintCompilation', + # '-XX:+LogCompilation', + # f'-XX:LogFile={class_location}/hotspot.log', + '-cp', + f':{self.output}:{class_location}:{self.json_jar}', + f'testing/Wrapper', + f'testing.TestCase', + str(path), + f'{class_location}/output.txt', + f'{class_location}/bad_output.txt', + f'{self.n_function_repeats}', + '-XX:CompileThreshold=100'] + else: + exe_cmd = [f'{self.jvm}', + '-XX:+UnlockDiagnosticVMOptions', + # '-XX:CompileCommand=print,TestCase.testCase', + '-XX:+PrintCompilation', + # '-XX:+LogCompilation', + # f'-XX:LogFile={class_location}/hotspot.log', + '-cp', + f':{class_location}:{self.json_jar}', + 'Wrapper', + str(path), + f'{class_location}/output.txt', + f'{class_location}/bad_output.txt', + f'{self.n_function_repeats}', + '-XX:CompileThreshold=100'] + + result = subprocess.run(exe_cmd) + + return RunnerReturn.EXECUTION_FAIL if result.returncode != 0 else RunnerReturn.SUCCESS + + def decompile_test(self, class_file : Path) -> RunnerReturn: + + outputdir = str(class_file.parent) + + # decompilation syntax varies depending on which decompiler toolchain is used + + if self.compiler_name == Compiler.CFR: + + decompile_cmd = [f'{self.jvm}', + '-jar', + str(self.compiler_path), + '--outputdir', + outputdir, + str(class_file)] + + elif self.compiler_name == Compiler.FERNFLOWER: + + decompile_cmd = [str(self.jvm), + '-jar', + str(self.compiler_path), + str(class_file), + outputdir] + + #TODO: syntax for other decompilers + """ + elif self.params.decompiler.value == Decompiler.PROCYON.value: + decompile_cmd = [f'''./javabc/decompile_test_procyon.sh {self.filepaths.src_filepath} {test_name} {self.filepaths.jvm} {self.filepaths.decompiler_path}'''] + """ + decompile_result = subprocess.run(decompile_cmd) + + if decompile_result.returncode != 0: + return RunnerReturn.DECOMPILATION_FAIL + + return RunnerReturn.SUCCESS + + + def recompile_test(self, program : Path) -> RunnerReturn: + + # Recompile test case + compile_cmd = [str(self.javac), + str(program), + '-d', + str(program.parent)] + + compile_result = subprocess.run(compile_cmd) + + if compile_result.returncode != 0: + return RunnerReturn.RECOMPILATION_FAIL + + # Compile wrapper with recompiled test case class + compile_wrapper_cmd = [str(self.javac), + '-cp', + f':{str(program.parent)}:{self.json_jar}', + str(self.wrapper), + '-d', + str(program.parent)] + + wrapper_result = subprocess.run(compile_wrapper_cmd) + + return RunnerReturn.SUCCESS if wrapper_result.returncode == 0 else RunnerReturn.RECOMPILATION_FAIL + + + diff --git a/src/fuzzflesh/harness/javabc/javabc_runner.py b/src/fuzzflesh/harness/javabc/javabc_runner.py index f8271fe..6d601ee 100644 --- a/src/fuzzflesh/harness/javabc/javabc_runner.py +++ b/src/fuzzflesh/harness/javabc/javabc_runner.py @@ -19,7 +19,7 @@ def __init__(self, _reflection : bool): super(Runner, self).__init__() self.compiler_name : Compiler = _toolchain - self.compiler_path : Path = _compiler_path + self.compiler_path : Path = None self.output : Path = _output self.jvm : Path = Path(_jvm, 'java') self.javac : Path = Path(_jvm, 'javac') @@ -27,7 +27,7 @@ def __init__(self, self.json_jar : Path = Path(_json) self.wrapper : Path = Path(_output, 'testing/Wrapper.java') if _reflection else Path(_output,'Wrapper.java') self.interface : Path = Path(_output, 'testing/TestCaseInterface.java') - self.n_function_repeats : int = 1000 + self.n_function_repeats : int = 300000 self.reflection : bool = _reflection @property @@ -58,8 +58,56 @@ def compile(self, program : Path) -> RunnerReturn: def execute(self, program :Path, path : Path) -> RunnerReturn: class_location = get_class_location(program) + print(str(class_location)) + print(str(program)) + return self.execute_test(program, path, class_location) + + def execute_with_changing_paths(self, program : Path, paths : list[Path]) -> RunnerReturn: + class_location = get_class_location(program) + + path = ":".join(map(str, paths)) + + if self.reflection: + exe_cmd = [f'{self.jvm}', + '-XX:+UnlockDiagnosticVMOptions', + # '-XX:CompileCommand=print,testing.TestCase::testCase', + # '-XX:+PrintCompilation', + '-XX:+LogCompilation', + f'-XX:LogFile={class_location}/hotspot.log', + '-cp', + f':{self.output}:{class_location}:{self.json_jar}', + f'testing/Wrapper', + f'testing.TestCase', + path, + f'{class_location}/output.txt', + f'{class_location}/bad_output.txt', + f'{self.n_function_repeats}', + '-XX:CompileThreshold=100'] + + else: + exe_cmd = [f'{self.jvm}', + '-Dgraal.MaxDuplicationFactor=100000.0', + '-XX:+UnlockDiagnosticVMOptions', + # '-XX:CompileCommand=print,TestCase.testCase', + # '-XX:+PrintCompilation', + # '-XX:+PrintAssembly', + # '-XX:+LogCompilation', + # f'-XX:LogFile={class_location}/hotspot.log', + '-cp', + f':{class_location}:{self.json_jar}', + 'Wrapper', + path, + f'{class_location}/output.txt', + f'{class_location}/bad_output.txt', + f'{self.n_function_repeats}', + '-XX:CompileThreshold=100'] + + result = subprocess.run(exe_cmd) + + return RunnerReturn.EXECUTION_FAIL if result.returncode != 0 else RunnerReturn.SUCCESS + def is_decompiler(self): if self.toolchain in [Compiler.CFR,Compiler.PROCYON,Compiler.FERNFLOWER]: @@ -145,6 +193,12 @@ def execute_test(self, program : Path, path : Path, class_location : Path) -> Ru if self.reflection: exe_cmd = [f'{self.jvm}', + '-XX:+UnlockDiagnosticVMOptions', + # '-XX:CompileCommand=print,testing.TestCase::testCase', + '-XX:+PrintCompilation', + '-XX:+PrintAssembly', + '-XX:+LogCompilation', + f'-XX:LogFile={class_location}/hotspot.log', '-cp', f':{self.output}:{class_location}:{self.json_jar}', f'testing/Wrapper', @@ -156,6 +210,12 @@ def execute_test(self, program : Path, path : Path, class_location : Path) -> Ru '-XX:CompileThreshold=100'] else: exe_cmd = [f'{self.jvm}', + '-XX:+UnlockDiagnosticVMOptions', + # '-XX:CompileCommand=print,TestCase.testCase', + # '-XX:+PrintCompilation', + '-XX:+PrintAssembly', + '-XX:+LogCompilation', + f'-XX:LogFile={class_location}/hotspot.log', '-cp', f':{class_location}:{self.json_jar}', 'Wrapper', diff --git a/src/fuzzflesh/program_generator/java/__init__.py b/src/fuzzflesh/program_generator/java/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/fuzzflesh/program_generator/java/java_generator.py b/src/fuzzflesh/program_generator/java/java_generator.py new file mode 100644 index 0000000..29a4568 --- /dev/null +++ b/src/fuzzflesh/program_generator/java/java_generator.py @@ -0,0 +1,213 @@ +from typing import List +from pathlib import Path + +from fuzzflesh.common.utils import InstructionBlock, Lang +from fuzzflesh.program_generator.flesher import ProgramFlesher +from fuzzflesh.cfg import CFG + +class JavaProgramGenerator(ProgramFlesher): + + def __init__(self, cfg : CFG, dirs_known_at_compile : bool = False, _reflection : bool = False): + super(JavaProgramGenerator, self).__init__(cfg, dirs_known_at_compile) + self.program_number : int = 0 + self.reflection = _reflection + + @property + def language(self): + return Lang.JAVA + + def increment_program_number(self) -> None: + self.program_number += 1 + + def flesh_program_start(self) -> List[InstructionBlock]: + if self.reflection: + code = '''package testing; + +class TestCase implements TestCaseInterface { + int dirs[]; + int dirsIndex; + int actualOutput[]; + int outputIndex; + + // default constructor + public TestCase() {} + + @Override + public void testCase(int dirs[], int actualOutput[]) { + this.dirs = dirs; // direction not known at compile time + this.dirsIndex = 0; + this.actualOutput = actualOutput; + this.outputIndex = 0; + + block_0(); // call starting block + } + +''' + + else: + code = '''class TestCase { + int dirs[]; + int dirsIndex; + int actualOutput[]; + int outputIndex; + + // default constructor + public TestCase() {} + + public void testCase(int dirs[], int actualOutput[]) { + this.dirs = dirs; // direction not known at compile time + this.dirsIndex = 0; + this.actualOutput = actualOutput; + this.outputIndex = 0; + + block_0(); // call starting block + } + +''' + + return [InstructionBlock(code)] + + def flesh_program_start_with_dirs(self, directions : list[int]) -> List[InstructionBlock]: + dirs = '''// direction known at compile time + this.dirs = new int []{''' + + # fill out directions array + for i, d in enumerate(directions): + dirs += ''' + {direction},'''.format(direction = d) + + dirs += ''' + };''' + + if self.reflection: + code = '''package testing; + +class TestCase implements TestCaseInterface {{ + int dirs[]; + int dirsIndex; + int actualOutput[]; + int outputIndex; + + // default constructor + public TestCase() {{}} + + @Override + public void testCase(int dirs[], int actualOutput[]) {{ + {dirs} + + this.dirsIndex = 0; + this.actualOutput = actualOutput; + this.outputIndex = 0; + + block_0(); // call starting block + }} + +'''.format(dirs = dirs) + + else: + code = '''class TestCase {{ + int dirs[]; + int dirsIndex; + int actualOutput[]; + int outputIndex; + + // default constructor + public TestCase() {{}} + + public void testCase(int dirs[], int actualOutput[]) {{ + {dirs} + + this.dirsIndex = 0; + this.actualOutput = actualOutput; + this.outputIndex = 0; + + block_0(); // call starting block + }} + +'''.format(dirs = dirs) + return [InstructionBlock(code)] + + def get_program_number(self) -> int: + num = self.program_number + self.increment_program_number + return num + + def flesh_start_of_node(self, n : int) -> InstructionBlock: + # Double {{ for escaping + code = ''' void block_{i}() {{ + actualOutput[outputIndex++] = {i}; +'''.format(i = n) + + return InstructionBlock(code) + + def flesh_exit_node(self, n : int) -> str: + ''' + returns code for node n with no successors + (exit node). + ''' + + code = ''' return; // exit node + } +''' + + return InstructionBlock(code) + + def flesh_unconditional_node(self, n : int) -> InstructionBlock: + ''' + returns code for node n with single successor + ''' + + code = ''' block_{successor}(); // unconditional node + }} +'''.format(successor = list(self.cfg.graph.adj[n])[0]) + + return InstructionBlock(code) + + def flesh_conditional_node(self, n : int) -> InstructionBlock: + ''' + returns code for node n with two successors, one of + which may be self (e.g. in case of loop) + note this does not deal with switch statements where + there are > 2 successor nodes + ''' + + code = ''' int dir = dirs[dirsIndex++]; + + // conditional node + if (dir == 0) {{ + block_{successor_true}(); + }} else {{ + block_{successor_false}(); + }} + }} +'''.format(successor_false = list(self.cfg.graph.adj[n])[1], successor_true = list(self.cfg.graph.adj[n])[0]) + + return InstructionBlock(code) + + def flesh_switch_node(self, n : int, n_successors : int) -> InstructionBlock: + ''' + returns code for node with > 2 successors + e.g. a switch statement + ''' + + code = ''' int dir = dirs[dirsIndex++]; + + // switch node + switch (dir) { +''' + + for j in range(n_successors): + code += ''' case {i}: block_{successor}(); break; +'''.format(i = j, successor = list(self.cfg.graph.adj[n])[j]) + + + code += ''' default: block_{default}(); + }} + }} +'''.format(default = list(self.cfg.graph.adj[n])[0]) + + return InstructionBlock(code) + + def flesh_program_end(self) -> InstructionBlock: + return InstructionBlock('''} +''') diff --git a/src/fuzzflesh/program_generator/javabc/javabc_generator.py b/src/fuzzflesh/program_generator/javabc/javabc_generator.py index 9165675..1077eef 100644 --- a/src/fuzzflesh/program_generator/javabc/javabc_generator.py +++ b/src/fuzzflesh/program_generator/javabc/javabc_generator.py @@ -168,7 +168,7 @@ def flesh_conditional_node(self, n : int) -> InstructionBlock: # depending on whether they are known at compile time or not #TODO: switch dir and output in passing function so dirs is always var 2 #TODO: something is broken here with the 1 and 5 - dir_local_var = 2 if self.dirs_known_at_compile else 1 + dir_local_var = 5 if self.dirs_known_at_compile else 1 code = ''' ; get directions for node diff --git a/src/fuzzflesh/wrappers/Wrapper.java b/src/fuzzflesh/wrappers/Wrapper.java index e6534bb..6fae693 100644 --- a/src/fuzzflesh/wrappers/Wrapper.java +++ b/src/fuzzflesh/wrappers/Wrapper.java @@ -12,31 +12,30 @@ import org.json.simple.JSONArray; import org.json.simple.JSONObject; -class Wrapper{ - - public static void main(String[] args) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, ParseException { +class Path { + String inputFilename; + int dir[]; + int actualOutput[]; + int expectedOutput[]; + int outputSize; - // parse args - String className = args[0]; - String inputFilename = args[1]; - String outputFilename = args[2]; - String badOutputFilename = args[3]; - int nFunctionRepeats = Integer.parseInt(args[4]); + Path(String inputFilename) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, ParseException { + this.inputFilename = inputFilename; // get direction size and expected output size from file - JSONParser parser = new JSONParser(); + JSONParser parser = new JSONParser(); JSONObject jsonObject = (JSONObject) parser.parse(new FileReader(inputFilename)); - JSONArray jsonDir = (JSONArray) jsonObject.get("dirs"); - JSONArray jsonOutput = (JSONArray) jsonObject.get("expected_output"); + JSONArray jsonDir = (JSONArray) jsonObject.get("dirs"); + JSONArray jsonOutput = (JSONArray) jsonObject.get("expected_output"); - int outputSize = jsonOutput.size(); - int dirSize = jsonDir.size(); + outputSize = jsonOutput.size(); + int dirSize = jsonDir.size(); // create arrays - int[] dir = new int[dirSize]; - int[] actualOutput = new int[2*outputSize]; - int[] expectedOutput = new int[outputSize]; + dir = new int[dirSize]; + actualOutput = new int[2*outputSize]; + expectedOutput = new int[outputSize]; // fill arrays int counter = 0; @@ -52,71 +51,59 @@ public static void main(String[] args) throws IOException, ClassNotFoundExceptio for (int i = outputSize; i < 2*outputSize; i++) { actualOutput[i] = -1; } + } - // create class - Constructor constructor = Class.forName(className).getConstructor(); - - TestCaseInterface test = (TestCaseInterface) constructor.newInstance(); - - // repeat function to induce JIT - for(int i = 0; i < nFunctionRepeats; i++){ - test.testCase(dir, actualOutput); + boolean compare() { + // check actual and expected path are the same + for(int i = 0; i < outputSize; i++){ + if(expectedOutput[i] != actualOutput[i]){ + return false; + } } - // compare expected and actual - boolean result = compare(expectedOutput, actualOutput, outputSize); - - // record all output - recordOutput(outputFilename, inputFilename, result, expectedOutput, actualOutput, outputSize); - - // record bad output in separate file - if (!result){ - recordOutput(badOutputFilename, inputFilename, result, expectedOutput, actualOutput, outputSize); + // check no overflow occurred + for(int i = outputSize; i < 2*outputSize; i++){ + if(actualOutput[i] != -1){ + return false; + } } - // print outcomes - System.out.print("Expected and actual output are"); - - if(result) { - System.out.print(""); - } - else { - System.out.print(" not"); - } + return true; + } - System.out.print(" the same\n"); + void recordOutput(String fileName, boolean result) { + if (fileName == null) { + System.out.print("Expected and actual output are"); - System.out.print("Expected output:"); - - for(int i = 0; i < outputSize; i++){ - System.out.print(" " + expectedOutput[i]); - } + if(result) { + System.out.print(""); + } + else { + System.out.print(" not"); + } - System.out.print("\n"); + System.out.print(" the same\n"); + System.out.print("Expected output:"); + + for(int i = 0; i < outputSize; i++){ + System.out.print(" " + expectedOutput[i]); + } - System.out.print("Actual output: "); - - for(int i = 0; i < 2*outputSize; i++){ - System.out.print(" " + actualOutput[i]); - } + System.out.print("\nActual output: "); - System.out.print("\n"); + for(int i = 0; i < 2*outputSize; i++){ + System.out.print(" " + actualOutput[i]); + } - if(result) { - System.exit(0); + System.out.print("\n"); + return; } - System.exit(1); - - } - - private static void recordOutput(String fileName, String testName, boolean result, int[] expectedOutput, int[] actualOutput, int outputSize){ - try{ FileWriter fw = new FileWriter(fileName, true); - fw.write("Test name: " + testName + "\n"); + fw.write("Test name: " + inputFilename + "\n"); fw.write("Expected and actual output are"); @@ -151,45 +138,123 @@ private static void recordOutput(String fileName, String testName, boolean resul }catch(IOException e){ System.out.println("Error writing results to file"); } - } - private static void setupArrays(int dirSize, int outputSize, int[] dir, int[] actualOutput, int[] expectedOutput, Scanner reader){ - - // initialise to -1 - for(int i = 0; i < 2*outputSize; i++){ + void resetActualOutput() { + for (int i = 0; i < actualOutput.length; ++i) { actualOutput[i] = -1; } - - // fill arrays - for(int i = 0; i < dirSize; i++){ - dir[i] = reader.nextInt(); - } - - for(int i = 0; i < outputSize; i++){ - expectedOutput[i] = reader.nextInt(); + } +} + +class Wrapper{ + + public static void main(String[] args) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, ParseException { + + // parse args + String className = args[0]; + String inputFilenames[] = args[1].split(":"); + String outputFilename = args[2]; + String badOutputFilename = args[3]; + int nFunctionRepeats = Integer.parseInt(args[4]); + + Path paths[] = new Path[inputFilenames.length]; + + for (int i = 0; i < inputFilenames.length; i++) { + paths[i] = new Path(inputFilenames[i]); } - } + boolean result = true; + boolean success = true; - private static boolean compare(int[] expectedOutput, int[] actualOutput, int outputSize){ + // create class + Constructor constructor = Class.forName(className).getConstructor(); + + TestCaseInterface test = (TestCaseInterface) constructor.newInstance(); - // check actual and expected path are the same - for(int i = 0; i < outputSize; i++){ - if(expectedOutput[i] != actualOutput[i]){ - return false; + int index = (int) (Math.random() * paths.length); + + // repeat function to induce JIT + for(int i = 0; i < nFunctionRepeats; i++){ + try { + paths[index].resetActualOutput(); + test.testCase(paths[index].dir, paths[index].actualOutput); + } catch (Exception e) { + result = paths[index].compare(); + paths[index].recordOutput(outputFilename, result); + paths[index].recordOutput(badOutputFilename, result); + paths[index].recordOutput(null, result); + + e.printStackTrace(); + System.exit(1); } + + // compare each expected and actual and write out if any are inconsistent + result = paths[index].compare(); + success = success && result; + + // record bad output in separate file + if (!result){ + paths[index].recordOutput(badOutputFilename, result); + } + } - // check no overflow occurred - for(int i = outputSize; i < 2*outputSize; i++){ - if(actualOutput[i] != -1){ - return false; + // record all output + paths[index].recordOutput(outputFilename, result); + + // print outcomes + paths[index].recordOutput(null, result); + + // if multiple paths provided, change the path to trigger deoptimization + if (paths.length >= 2) { + int repeat = (int) (Math.random() * paths.length * 1.5) + 1; + + for (int i = 0; i < repeat; ++i) { + System.out.println("Switching paths"); + index = (int) (Math.random() * paths.length); + + // repeat function to induce JIT + for(int j = 0; j < nFunctionRepeats; j++){ + try { + paths[index].resetActualOutput(); + test.testCase(paths[index].dir, paths[index].actualOutput); + } catch (Exception e) { + result = paths[index].compare(); + paths[index].recordOutput(outputFilename, result); + paths[index].recordOutput(badOutputFilename, result); + paths[index].recordOutput(null, result); + + e.printStackTrace(); + System.exit(1); + } + + // compare each expected and actual and write out if any are inconsistent + result = paths[index].compare(); + success = success && result; + + // record bad output in separate file + if (!result){ + paths[index].recordOutput(badOutputFilename, result); + } + + } + + // record all output + paths[index].recordOutput(outputFilename, result); + + // print outcomes + paths[index].recordOutput(null, result); } } + - return true; - } + if(success) { + System.exit(0); + } + System.exit(1); + + } } diff --git a/src/fuzzflesh/wrappers/WrapperNoReflection.java b/src/fuzzflesh/wrappers/WrapperNoReflection.java index a3c4382..c236ebf 100644 --- a/src/fuzzflesh/wrappers/WrapperNoReflection.java +++ b/src/fuzzflesh/wrappers/WrapperNoReflection.java @@ -5,38 +5,37 @@ import java.io.IOException; import java.io.Reader; import java.util.ArrayList; +import java.lang.Math; import java.lang.reflect.*; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; import org.json.simple.JSONArray; import org.json.simple.JSONObject; -class Wrapper{ - - public static void main(String[] args) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, ParseException { +class Path { + String inputFilename; + int dir[]; + int actualOutput[]; + int expectedOutput[]; + int outputSize; - // parse args - String inputFilename = args[0]; - String outputFilename = args[1]; - String badOutputFilename = args[2]; - int nFunctionRepeats = Integer.parseInt(args[3]); + Path(String inputFilename) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, ParseException { + this.inputFilename = inputFilename; - boolean result = true; - // get direction size and expected output size from file - JSONParser parser = new JSONParser(); + JSONParser parser = new JSONParser(); JSONObject jsonObject = (JSONObject) parser.parse(new FileReader(inputFilename)); - JSONArray jsonDir = (JSONArray) jsonObject.get("dirs"); - JSONArray jsonOutput = (JSONArray) jsonObject.get("expected_output"); + JSONArray jsonDir = (JSONArray) jsonObject.get("dirs"); + JSONArray jsonOutput = (JSONArray) jsonObject.get("expected_output"); - int outputSize = jsonOutput.size(); - int dirSize = jsonDir.size(); + outputSize = jsonOutput.size(); + int dirSize = jsonDir.size(); // create arrays - int[] dir = new int[dirSize]; - int[] actualOutput = new int[2*outputSize]; - int[] expectedOutput = new int[outputSize]; + dir = new int[dirSize]; + actualOutput = new int[2*outputSize]; + expectedOutput = new int[outputSize]; // fill arrays int counter = 0; @@ -52,76 +51,59 @@ public static void main(String[] args) throws IOException, ClassNotFoundExceptio for (int i = outputSize; i < 2*outputSize; i++) { actualOutput[i] = -1; } + } - // create class - TestCase test = new TestCase(); - - // repeat function to induce JIT - for(int i = 0; i < nFunctionRepeats; i++){ - - test.testCase(dir, actualOutput); - - // compare each expected and actual and write out if any are inconsistent - result = compare(expectedOutput, actualOutput, outputSize); - - // record bad output in separate file - if (!result){ - recordOutput(badOutputFilename, inputFilename, result, expectedOutput, actualOutput, outputSize); + boolean compare() { + // check actual and expected path are the same + for(int i = 0; i < outputSize; i++){ + if(expectedOutput[i] != actualOutput[i]){ + return false; } - } - // compare final expected and actual - result = compare(expectedOutput, actualOutput, outputSize); - - // record all output - recordOutput(outputFilename, inputFilename, result, expectedOutput, actualOutput, outputSize); - - // record bad output in separate file - if (!result){ - recordOutput(badOutputFilename, inputFilename, result, expectedOutput, actualOutput, outputSize); + // check no overflow occurred + for(int i = outputSize; i < 2*outputSize; i++){ + if(actualOutput[i] != -1){ + return false; + } } - // print outcomes - System.out.print("Expected and actual output are"); + return true; + } - if(result) { - System.out.print(""); - } - else { - System.out.print(" not"); - } + void recordOutput(String fileName, boolean result) { + if (fileName == null) { + System.out.print("Expected and actual output are"); - System.out.print(" the same\n"); + if(result) { + System.out.print(""); + } + else { + System.out.print(" not"); + } - System.out.print("Expected output:"); - - for(int i = 0; i < outputSize; i++){ - System.out.print(" " + expectedOutput[i]); - } + System.out.print(" the same\n"); - System.out.print("\nActual output: "); + System.out.print("Expected output:"); + + for(int i = 0; i < outputSize; i++){ + System.out.print(" " + expectedOutput[i]); + } - for(int i = 0; i < 2*outputSize; i++){ - System.out.print(" " + actualOutput[i]); - } + System.out.print("\nActual output: "); - System.out.print("\n"); + for(int i = 0; i < 2*outputSize; i++){ + System.out.print(" " + actualOutput[i]); + } - if(result) { - System.exit(0); + System.out.print("\n"); + return; } - System.exit(1); - - } - - private static void recordOutput(String fileName, String testName, boolean result, int[] expectedOutput, int[] actualOutput, int outputSize){ - try{ FileWriter fw = new FileWriter(fileName, true); - fw.write("Test name: " + testName + "\n"); + fw.write("Test name: " + inputFilename + "\n"); fw.write("Expected and actual output are"); @@ -156,47 +138,118 @@ private static void recordOutput(String fileName, String testName, boolean resul }catch(IOException e){ System.out.println("Error writing results to file"); } - } - private static void setupArrays(int dirSize, int outputSize, int[] dir, int[] actualOutput, int[] expectedOutput, Scanner reader){ - - - // initialise to -1 - for(int i = 0; i < 2*outputSize; i++){ + void resetActualOutput() { + for (int i = 0; i < actualOutput.length; ++i) { actualOutput[i] = -1; } - - // fill arrays - for(int i = 0; i < dirSize; i++){ - dir[i] = reader.nextInt(); - } - - for(int i = 0; i < outputSize; i++){ - expectedOutput[i] = reader.nextInt(); + } +} + +class Wrapper{ + public static void main(String[] args) throws IOException, ClassNotFoundException, InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException, ParseException { + + // parse args + String inputFilenames[] = args[0].split(":"); + String outputFilename = args[1]; + String badOutputFilename = args[2]; + int nFunctionRepeats = Integer.parseInt(args[3]); + + Path paths[] = new Path[inputFilenames.length]; + + for (int i = 0; i < inputFilenames.length; i++) { + paths[i] = new Path(inputFilenames[i]); } - - } + boolean result = true; + boolean success = true; - private static boolean compare(int[] expectedOutput, int[] actualOutput, int outputSize){ + // create class + TestCase test = new TestCase(); - // check actual and expected path are the same - for(int i = 0; i < outputSize; i++){ - if(expectedOutput[i] != actualOutput[i]){ - return false; + int index = (int) (Math.random() * paths.length); + + // repeat function to induce JIT + for(int i = 0; i < nFunctionRepeats; i++){ + try { + paths[index].resetActualOutput(); + test.testCase(paths[index].dir, paths[index].actualOutput); + } catch (Exception e) { + result = paths[index].compare(); + paths[index].recordOutput(outputFilename, result); + paths[index].recordOutput(badOutputFilename, result); + paths[index].recordOutput(null, result); + + e.printStackTrace(); + System.exit(1); + } + + // compare each expected and actual and write out if any are inconsistent + result = paths[index].compare(); + success = success && result; + + // record bad output in separate file + if (!result){ + paths[index].recordOutput(badOutputFilename, result); } + } - // check no overflow occurred - for(int i = outputSize; i < 2*outputSize; i++){ - if(actualOutput[i] != -1){ - return false; + // record all output + paths[index].recordOutput(outputFilename, result); + + // print outcomes + paths[index].recordOutput(null, result); + + // if multiple paths provided, change the path to trigger deoptimization + if (paths.length >= 2) { + int repeat = (int) (Math.random() * paths.length * 1.5) + 1; + + for (int i = 0; i < repeat; ++i) { + System.out.println("Switching paths"); + index = (int) (Math.random() * paths.length); + + // repeat function to induce JIT + for(int j = 0; j < nFunctionRepeats; j++){ + try { + paths[index].resetActualOutput(); + test.testCase(paths[index].dir, paths[index].actualOutput); + } catch (Exception e) { + result = paths[index].compare(); + paths[index].recordOutput(outputFilename, result); + paths[index].recordOutput(badOutputFilename, result); + paths[index].recordOutput(null, result); + + e.printStackTrace(); + System.exit(1); + } + + // compare each expected and actual and write out if any are inconsistent + result = paths[index].compare(); + success = success && result; + + // record bad output in separate file + if (!result){ + paths[index].recordOutput(badOutputFilename, result); + } + + } + + // record all output + paths[index].recordOutput(outputFilename, result); + + // print outcomes + paths[index].recordOutput(null, result); } } + - return true; - } + if(success) { + System.exit(0); + } + System.exit(1); + } }