diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61751c3..69a177f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,3 +39,16 @@ jobs: with: name: test-logs path: bazel-testlogs/tests/**/test.log + + - name: make pipeline script executable + run: chmod 755 scripts/run_pipeline.sh + - name: run pipeline + run: ./scripts/run_pipeline.sh tests/python/example1.json output/test + - name: upload pipeline results + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: pipeline-results + path: | + ./output/test/*.json + ./output/test/*.html \ No newline at end of file diff --git a/apps/cpp_cli_tool/BUILD b/apps/cpp_cli_tool/BUILD index 46f7132..10a9cea 100644 --- a/apps/cpp_cli_tool/BUILD +++ b/apps/cpp_cli_tool/BUILD @@ -1,3 +1,14 @@ -cc_library( - -) \ No newline at end of file +cc_binary( + name = "cli", + srcs = ["cli.cc"], + deps = [ + "//libs/parser:parser_factory", + "//libs/math:stats", + "@bazel_tools//tools/cpp/runfiles", + ], + visibility = ["//scripts:__subpackages__"], + + data = ["//tests/python:testdata"], + +) + diff --git a/apps/cpp_cli_tool/cli.cc b/apps/cpp_cli_tool/cli.cc index e69de29..a8d5af5 100644 --- a/apps/cpp_cli_tool/cli.cc +++ b/apps/cpp_cli_tool/cli.cc @@ -0,0 +1,93 @@ +/** + * This should accept file path + type, use factory parser, run statistics, print summary. + * + * DoD: + * bazel run //apps/cpp_cli_tool:cli -- data.csv works. + * Console output includes mean, median, variance, frequencies. + * + */ + +#include +#include +#include + +#include "tools/cpp/runfiles/runfiles.h" +#include "libs/parser/parser_factory.h" +#include "libs/math/stats.h" + + +/** + * main function for CLI tool. Call by running bazel run //apps/cpp_cli_tool:cli -- + * @param argc argument count, @param argv an array of C strings (the arguments themselves) + * @return 0 on success, 1 on failure + */ + +int main(int argc, char* argv[]){ + if(argc < 2){ + std::cout << "Usage: mytool " << std::endl; + return 1; + } + std::string filename = argv[1]; + std::string filetype = filename.substr(filename.find_last_of(".") + 1); + + std::string err; + auto runfiles(bazel::tools::cpp::runfiles::Runfiles::Create(argv[0], &err)); + if (runfiles == nullptr) { + std::cerr << "Error initializing runfiles: " << err << std::endl; + return 1; + } + + std::string filepath = runfiles->Rlocation(filename); //<-- Will cause a seg fault if runfiles==nullptr :( + + + //Create parser w. parserfactory + static shared_ptr parserPtr; + parserPtr = ParserFactory(filetype); //Evaluates to csv or json parser + + parserPtr->parse(filepath); //Parses the file + + Stats stats = Stats(); + + stats.readData(parserPtr->getEntries()); // Should pass in correct datatype + stats.loadScores(); + std::cout << "\n\n\n"; + + std::cout << std::setfill('=') << std::setw(25 + strlen("SCORES-STATISTICS")) << "SCORES-STATISTICS" << std::setw(25) << '\n'; + std::cout << std::setfill(' ') << "Mean: " << stats.calcMean() << std::setw(strlen("Median: ") + 2); + std::cout << "Median: " << stats.calcMedian() << std::setw(strlen("Variance: " + 2)) << " Variance: " << stats.calcVariance(); + std::cout << std::setw(strlen("Standard deviation: ") + 2) << " Standard deviation: " << stats.calcStandardDeviation(); + + auto data = parserPtr->getEntries(); + std::vector scores {}; + + for (size_t i = 0; i < data.size(); i++) + { + auto entry = parserPtr->getEntryById(std::to_string(i+1)); + for (auto &&[k, v]: entry) + { + if(k == "score"){ + scores.push_back(std::stoi(v)); + break; + } + } + } + + if(stats.calcMode().size() == scores.size()){ + std::cout << std::setw(strlen("No identifiable mode") + 2) << "No identifiable mode"; + }else{ + std::cout << std::setw(strlen("Mode: ") + 2) << "Mode: " << stats.calcMode()[0]; + } + std::cout << "\n" << std::endl; + + std::cout << std::setfill('=') << std::setw(25 + strlen("FREQUENCIES-OF-SCORES")) << "FREQUENCIES-OF-SCORES" << std::setw(25) << "\n"; + for (auto &&[k,v] : stats.calcFrequencies()) + { + std::cout << k << " appeared " << v << " time(s) "; + } + std::cout << std::endl; + std::cout << "\n\n"; + + + + return 0; +} \ No newline at end of file diff --git a/libs/math/BUILD b/libs/math/BUILD index dfda20a..99523ec 100644 --- a/libs/math/BUILD +++ b/libs/math/BUILD @@ -27,6 +27,18 @@ cc_binary( ) # usage: bazel build //libs/math:generate_stats_file +gen_stats( + name = "generate_stats_file_pipeline", + # take the cc_binary as exec tool + tool = ":generate_stats", + data = [ + "//output/test:example1_cleaned.json", + ], + visibility = [ + "//:__subpackages__", + ] +) + gen_stats( name = "generate_stats_file", # take the cc_binary as exec tool diff --git a/libs/math/generate_stats.cc b/libs/math/generate_stats.cc index c857ed5..4ff40ad 100644 --- a/libs/math/generate_stats.cc +++ b/libs/math/generate_stats.cc @@ -14,7 +14,6 @@ int main(int argc, char* argv[]) } string filePath = argv[1]; - int totalSize = filePath.size(); int suffixSize = 4; diff --git a/libs/parser/BUILD b/libs/parser/BUILD index 852c1ec..1d14d32 100644 --- a/libs/parser/BUILD +++ b/libs/parser/BUILD @@ -62,6 +62,6 @@ cc_library( ":json_parser", ":csv_parser", ], - visibility = ["//libs/math:__pkg__"], includes = ["."], + visibility = ["//:__subpackages__"], ) \ No newline at end of file diff --git a/libs/parser/csv_parser.cc b/libs/parser/csv_parser.cc index 40d70c3..0da88f0 100644 --- a/libs/parser/csv_parser.cc +++ b/libs/parser/csv_parser.cc @@ -29,6 +29,8 @@ void CsvParser::parse(const string& filename) stringstream ss(line); string header; while(getline(ss, header, ',')){ + header.erase(remove(header.begin(), header.end(), '\r'), header.end()); + header.erase(remove(header.begin(), header.end(), ' '), header.end()); headers.push_back(header); } } @@ -43,6 +45,9 @@ void CsvParser::parse(const string& filename) while(getline(ss, value, ',')){ if(col < headers.size()){ + value.erase(remove(value.begin(), value.end(), '\r'), value.end()); + value.erase(remove(value.begin(), value.end(), ' '), value.end()); + row[headers[col]] = value; } col++; diff --git a/output/BUILD b/output/BUILD new file mode 100644 index 0000000..e69de29 diff --git a/output/test/BUILD b/output/test/BUILD new file mode 100644 index 0000000..2963406 --- /dev/null +++ b/output/test/BUILD @@ -0,0 +1 @@ +exports_files(glob(allow_empty=True, include=["*.csv", "*.json"])) \ No newline at end of file diff --git a/python_tools/data_cleaner/cleaner.py b/python_tools/data_cleaner/cleaner.py index 34537d2..49ddc4d 100644 --- a/python_tools/data_cleaner/cleaner.py +++ b/python_tools/data_cleaner/cleaner.py @@ -9,14 +9,15 @@ class DataCleaner: """ Class to read, clean and write the content of csv or json files. - :atribute str filename: Name of the file being processed. - :atribute str filetype: csv or json depending on filenames ending. - :atribute list filecontent: empty list representeing the content of the file being read. - :atribute runfiles r: Object which enables file reading in bazel environments. - """ - def __init__(self): - self.filename = "" - self.filetype = "" + :attribute str filename: Name of the file being processed. + :attribute str filetype: csv or json depending on filenames ending. + :attribute list filecontent: empty list representeing the content of the file being read. + :attribute runfiles r: Object which enables file reading in bazel environments. + """ + def __init__(self, output_dir=None): + self.filename = None + self.filetype = None + self.output_dir = output_dir self.filecontent = [] self.r = runfiles.Create() @@ -34,25 +35,19 @@ def detect_filetype(self, filepath): else: raise ValueError("Unsupported filetype") - """ - :Method to initialize the terminal arguments to call tests - """ - def initArgs(self): - self.parser.add_argument("filepath", help = "Path to the input file") - self.parser.add_argument("--read-file-only", action="store_true", help="Reading a file without cleaning the data") - """ Method to read content of file depending on the filetype. :param str filepath: filepath to the file which should be read. """ - def readFile(self, filepath): - workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', 'data_processing_pipeline') + def readFile(self, filepath): + workspace = os.environ.get('TEST_WORKSPACE', None) - data_location = self.r.Rlocation(f"{workspace}/{filepath}") - - self.filetype = self.detect_filetype(filepath) - self.output = filepath.split(".") - self.output = f"{self.output[0]}_cleaned.{self.output[1]}" + filepath = os.path.normpath(filepath) + filepath = f"{workspace}/{filepath}" if workspace and not os.path.isabs(filepath) else filepath + + data_location = self.r.Rlocation(filepath) + self.filename = filepath + self.filetype = self.detect_filetype(filepath) self.filecontent.clear() @@ -73,18 +68,30 @@ def readFile(self, filepath): Method to write the content of self.filecontent to a filepath. """ def writeFile(self): - workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', 'data_processing_pipeline') + workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', "./") - data_location = self.r.Rlocation(f"{workspace}/{self.output}") + basename, ext = os.path.splitext(os.path.basename(self.filename)) + cleaned_filename = f"{basename}_cleaned{ext}" + + if self.output_dir is None: + print("No output directory passed \n") + + if self.output_dir: + output_dir = self.output_dir if os.path.isabs(self.output_dir) else os.path.join(workspace, self.output_dir) + + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join(output_dir, cleaned_filename) + + if(self.filetype == "csv"): - with(open(data_location, mode='w')) as file: + with(open(output_path, mode='w')) as file: csv_writer = csv.DictWriter(file, fieldnames=self.filecontent[0].keys()) csv_writer.writeheader() csv_writer.writerows(self.filecontent) elif(self.filetype == "json"): - with(open(data_location, mode='w')) as file: + with(open(output_path, mode='w')) as file: json.dump(self.filecontent, file) """ @@ -135,13 +142,16 @@ def main(): parser = argparse.ArgumentParser(description="Clean CSV/JSON files") parser.add_argument("filepath", help="Path to the input file") parser.add_argument("--read-file-only", action="store_true", help="Only read the file and not clean") + parser.add_argument("--output-dir", default=None, help="Optional directory to write cleaned ouput files to (default: same directory as input)") args = parser.parse_args() - cleaner = DataCleaner() + cleaner = DataCleaner(output_dir=args.output_dir) if args.read_file_only: cleaner.readFile(filepath=args.filepath) else: cleaner.readFileAndClean(filepath=args.filepath) + + return args if __name__ == '__main__': - main() \ No newline at end of file + args = main() \ No newline at end of file diff --git a/python_tools/report_generator/BUILD b/python_tools/report_generator/BUILD index 71fdb5b..89c5432 100644 --- a/python_tools/report_generator/BUILD +++ b/python_tools/report_generator/BUILD @@ -1,5 +1,18 @@ +py_binary( + name = "generator_pipeline", + main = "generator.py", + srcs = ["generator.py"], + # set data as custom generation rule as it produces an output file + data = [ + "@rules_python//python/runfiles", + "//libs/math:generate_stats_file_pipeline", + ], + visibility = ["//:__subpackages__"], +) + py_binary( name = "generator", + main = "generator.py", srcs = ["generator.py"], # set data as custom generation rule as it produces an output file data = [ diff --git a/python_tools/report_generator/generator.py b/python_tools/report_generator/generator.py index c54fb93..d7b46a5 100644 --- a/python_tools/report_generator/generator.py +++ b/python_tools/report_generator/generator.py @@ -111,13 +111,11 @@ def generate_HTML_file(self): def main(): argparser = argparse.ArgumentParser(description="Parses arguments") - argparser.add_argument("output_dir", nargs="?", default=None, help="Path to output directory") + argparser.add_argument("--output_dir", nargs="?", default="", help="Path to output directory") + argparser.add_argument("--input_file", nargs="?", default="output.json", help="Path to input file") args = argparser.parse_args() - if args.output_dir is None: - rg = ReportGenerator() - else: - rg = ReportGenerator(args.output_dir) + rg = ReportGenerator(args.output_dir, args.input_file) rg.read_JSON_file() rg.handle_empty_values() diff --git a/scripts/BUILD b/scripts/BUILD index e69de29..20cdaa4 100644 --- a/scripts/BUILD +++ b/scripts/BUILD @@ -0,0 +1,29 @@ +sh_library( + name = "run_pipeline_lib", + visibility = ["//visibility:public"], + #srcs = ["run_pipeline.sh"], + data = [ + ":run_pipeline", #Sh_binary below + ], + +) + +sh_binary( + name = "run_pipeline", + srcs = ["run_pipeline.sh",], + #data = ["//tests/python:testdata",], + data = [ + "//apps/cpp_cli_tool:cli", + "//python_tools/data_cleaner:cleaner", + "//tests/python:testdata", + ":bash_data" + ], + visibility = ["//visibility:__subpackages__"], + #deps = [":run_pipeline_lib"], +) + +filegroup( + name = "bash_data", + srcs = glob(["*.csv", "*.json"]), + visibility = ["//:__subpackages__"], +) \ No newline at end of file diff --git a/scripts/run_pipeline.sh b/scripts/run_pipeline.sh index e69de29..db61deb 100644 --- a/scripts/run_pipeline.sh +++ b/scripts/run_pipeline.sh @@ -0,0 +1,44 @@ +# TO RUN: ./scripts/run_pipeline.sh tests/python/example1.json output/test + +# Order of running pipeline cpp_cli_tool --> data_cleaner --> report_generator + +set -e + +INPUT_FILE="$1" +OUTPUT_DIR="$2" + +if [-z $INPUT_FILE]; then + echo "Usage: bazel run //scripts:run_pipeline " + exit 1 +fi + +echo "====================RUNNING-PIPELINE-ON-$INPUT_FILE====================" + +echo "Fetching directories..." + +WORKSPACE_DIR="$(bazel info workspace)/" +BIN_DIR="$(bazel info bazel-bin)" + + +CLI_TARGET="//apps/cpp_cli_tool:cli" +PY_TARGET="//python_tools/data_cleaner:cleaner" + +echo "Fetched directories..." + +bazel build $CLI_TARGET $PY_TARGET + +echo "Running C++ CLI tool on $INPUT_FILE..." + +bazel run $CLI_TARGET -- $WORKSPACE_DIR$INPUT_FILE + +echo "Running Python data cleaner tool on $INPUT_FILE..." + +bazel run $PY_TARGET -- $WORKSPACE_DIR$INPUT_FILE "--output-dir=$OUTPUT_DIR" + +echo "Generating report of $INPUT_FILE..." + +bazel run //python_tools/report_generator:generator_pipeline -- --output_dir $OUTPUT_DIR --input_file output_generate_stats_file_pipeline.json + +echo "Pipeline finished, check $(realpath $OUTPUT_DIR) for results" + +echo "====================PIPELINE-SUCCESSFUL====================" \ No newline at end of file diff --git a/tests/python/BUILD b/tests/python/BUILD index ce7f45c..0c2b4f5 100644 --- a/tests/python/BUILD +++ b/tests/python/BUILD @@ -11,6 +11,7 @@ pytest_test( "//tests/python/pytest:pytest_wrapper", ], imports = ["."], + data = [":testdata"], ) pytest_test( diff --git a/tests/python/macros.bzl b/tests/python/macros.bzl index 94650d9..cdf83f9 100644 --- a/tests/python/macros.bzl +++ b/tests/python/macros.bzl @@ -5,7 +5,7 @@ default_args = [ "--disable-warnings", "-v", "--tb=short", - "-color=yes", + "--color=yes", "-rfEP", ] diff --git a/tests/python/test_cleaner.py b/tests/python/test_cleaner.py index ab69e54..0af69ca 100644 --- a/tests/python/test_cleaner.py +++ b/tests/python/test_cleaner.py @@ -2,7 +2,7 @@ def test_handleNaNsCSV(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFile("tests/python/example1.csv") cleaner.handleNaNs() cleaner.writeFile() @@ -14,7 +14,7 @@ def test_handleNaNsCSV(): ] def test_normalizeTextCSV(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFile("tests/python/example2.csv") cleaner.normalizeText() cleaner.writeFile() @@ -30,7 +30,7 @@ def test_normalizeTextCSV(): def test_handleNaNsJSON(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFile("tests/python/example1.json") cleaner.handleNaNs() cleaner.writeFile() @@ -44,7 +44,7 @@ def test_handleNaNsJSON(): def test_normalizeTextJSON(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFile("tests/python/example2.json") cleaner.normalizeText() cleaner.writeFile() @@ -59,7 +59,7 @@ def test_normalizeTextJSON(): def test_completeCleaningCSV(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFileAndClean("tests/python/example3.csv") assert cleaner.filecontent == [ @@ -70,7 +70,7 @@ def test_completeCleaningCSV(): ] def test_completeCleaningJSON(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFileAndClean("tests/python/example3.json") assert cleaner.filecontent == [