From c20c5e6573a1e1a856a5bfa9b2d0ab84bfd011b2 Mon Sep 17 00:00:00 2001 From: Samuel Palmhager Date: Tue, 7 Oct 2025 16:05:29 +0200 Subject: [PATCH 01/18] Cli tool initial commit (#7) Initial commit of CLI tool. Can read input from terminal as specified in ticket DoD. Parses file and begins stats calculation. TBD: Finish formatting the output of statistics. --- apps/cpp_cli_tool/BUILD | 17 ++++++++-- apps/cpp_cli_tool/cli.cc | 70 ++++++++++++++++++++++++++++++++++++++++ libs/parser/BUILD | 2 +- 3 files changed, 85 insertions(+), 4 deletions(-) diff --git a/apps/cpp_cli_tool/BUILD b/apps/cpp_cli_tool/BUILD index 46f7132..f1c9dbc 100644 --- a/apps/cpp_cli_tool/BUILD +++ b/apps/cpp_cli_tool/BUILD @@ -1,3 +1,14 @@ -cc_library( - -) \ No newline at end of file +cc_binary( + name = "cli", + srcs = ["cli.cc"], + deps = [ + "//libs/parser:parser_factory", + "//libs/math:stats", + "@bazel_tools//tools/cpp/runfiles", + ], + + data = [ + "//tests/python:example1.json", + ], +) + diff --git a/apps/cpp_cli_tool/cli.cc b/apps/cpp_cli_tool/cli.cc index e69de29..46c9fd6 100644 --- a/apps/cpp_cli_tool/cli.cc +++ b/apps/cpp_cli_tool/cli.cc @@ -0,0 +1,70 @@ +/** + * This should accept file path + type, use factory parser, run statistics, print summary. + * + * DoD: + * bazel run //apps/cpp_cli_tool:cli -- data.csv works. + * Console output includes mean, median, variance, frequencies. + * + */ + +#include +#include +#include + +#include "tools/cpp/runfiles/runfiles.h" +#include "libs/parser/parser_factory.h" +#include "libs/math/stats.h" + + +/** + * main function for CLI tool. Call by running bazel run //apps/cpp_cli_tool:cli -- + * @param argc argument count, @param argv an array of C strings (the arguments themselves) + * @return 0 on success, 1 on failure + */ + +int main(int argc, char* argv[]){ + if(argc < 2){ + std::cout << "Usage: mytool " << std::endl; + return 1; + } + + std::cout << "argc = " << argc << std::endl; + + for (int i = 0; i < argc; ++i) + std::cout << "argv[" << i << "] = " << argv[i] << std::endl; + + std::string filename = argv[1]; + std::string filetype = filename.substr(filename.find_last_of(".") + 1); + + + + std::string err; + std::unique_ptr runfiles(bazel::tools::cpp::runfiles::Runfiles::Create(argv[0], &err)); + + std::string filepath = runfiles->Rlocation(filename); + + //Create parser w. parserfactory + static shared_ptr parserPtr; + parserPtr = ParserFactory(filetype); //Evaluates to csv or json parser + + parserPtr->parse(filepath); //Parses the file + + Stats stats = Stats(); + + stats.readData(parserPtr->getEntries()); // Should pass in correct datatype + stats.loadScores(); + + + std::cout << std::setfill('=') << std::setw(50) << '\n'; + std::cout << std::setfill(' ') << "Mean: " << stats.calcMean() << std::setw(13); + std::cout << "Median: " << stats.calcMedian(); + + if(stats.calcMode().size() == parserPtr->getEntryById("score").size()){ + std::cout << "No identifiably mode"; + }else{ + std::cout << std::setw(10) << "Mode: " << stats.calcMode()[0]; + //std::cout << "Size of mode " << stats.calcMode().size() << " Size of scores " << stats.scores.size(); + } + + return 0; +} \ No newline at end of file diff --git a/libs/parser/BUILD b/libs/parser/BUILD index 852c1ec..1d14d32 100644 --- a/libs/parser/BUILD +++ b/libs/parser/BUILD @@ -62,6 +62,6 @@ cc_library( ":json_parser", ":csv_parser", ], - visibility = ["//libs/math:__pkg__"], includes = ["."], + visibility = ["//:__subpackages__"], ) \ No newline at end of file From 4603dd76a9a46b689fd1ade04e1751d650fab6dd Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Wed, 8 Oct 2025 16:42:07 +0200 Subject: [PATCH 02/18] Second commit of cli_cpp_tool --- apps/cpp_cli_tool/BUILD | 1 + apps/cpp_cli_tool/cli.cc | 45 +++++++++++++++++++++++++++------------ libs/parser/csv_parser.cc | 5 +++++ tests/python/example1.csv | 4 ++-- 4 files changed, 39 insertions(+), 16 deletions(-) diff --git a/apps/cpp_cli_tool/BUILD b/apps/cpp_cli_tool/BUILD index f1c9dbc..8253c9a 100644 --- a/apps/cpp_cli_tool/BUILD +++ b/apps/cpp_cli_tool/BUILD @@ -9,6 +9,7 @@ cc_binary( data = [ "//tests/python:example1.json", + "//tests/python:example1.csv", ], ) diff --git a/apps/cpp_cli_tool/cli.cc b/apps/cpp_cli_tool/cli.cc index 46c9fd6..e0d01c3 100644 --- a/apps/cpp_cli_tool/cli.cc +++ b/apps/cpp_cli_tool/cli.cc @@ -27,12 +27,6 @@ int main(int argc, char* argv[]){ std::cout << "Usage: mytool " << std::endl; return 1; } - - std::cout << "argc = " << argc << std::endl; - - for (int i = 0; i < argc; ++i) - std::cout << "argv[" << i << "] = " << argv[i] << std::endl; - std::string filename = argv[1]; std::string filetype = filename.substr(filename.find_last_of(".") + 1); @@ -53,18 +47,41 @@ int main(int argc, char* argv[]){ stats.readData(parserPtr->getEntries()); // Should pass in correct datatype stats.loadScores(); - - - std::cout << std::setfill('=') << std::setw(50) << '\n'; - std::cout << std::setfill(' ') << "Mean: " << stats.calcMean() << std::setw(13); + std::cout << "\n\n\n"; + + std::cout << std::setfill('=') << std::setw(25 + strlen("SCORES-STATISTICS")) << "SCORES-STATISTICS" << std::setw(25) << '\n'; + std::cout << std::setfill(' ') << "Mean: " << stats.calcMean() << std::setw(strlen("Median: ") + 2); std::cout << "Median: " << stats.calcMedian(); - if(stats.calcMode().size() == parserPtr->getEntryById("score").size()){ - std::cout << "No identifiably mode"; + auto data = parserPtr->getEntries(); + std::vector scores {}; + + for (size_t i = 0; i < data.size(); i++) + { + auto entry = parserPtr->getEntryById(std::to_string(i+1)); + for (auto &&[k, v]: entry) + { + if(k == "score"){ + scores.push_back(std::stoi(v)); + break; + } + } + } + + if(stats.calcMode().size() == scores.size()){ + std::cout << std::setw(strlen("No identifiable mode") + 2) << "No identifiable mode"; }else{ - std::cout << std::setw(10) << "Mode: " << stats.calcMode()[0]; - //std::cout << "Size of mode " << stats.calcMode().size() << " Size of scores " << stats.scores.size(); + std::cout << std::setw(strlen("Mode: ") + 2) << "Mode: " << stats.calcMode()[0]; } + std::cout << std::endl; + std::cout << "Frequencies of scores: " << "\n"; + for (auto &&[k,v] : stats.calcFrequencies()) + { + std::cout << k << " appeared " << v << " time(s) "; + } + std::cout << std::endl; + + return 0; } \ No newline at end of file diff --git a/libs/parser/csv_parser.cc b/libs/parser/csv_parser.cc index 40d70c3..0da88f0 100644 --- a/libs/parser/csv_parser.cc +++ b/libs/parser/csv_parser.cc @@ -29,6 +29,8 @@ void CsvParser::parse(const string& filename) stringstream ss(line); string header; while(getline(ss, header, ',')){ + header.erase(remove(header.begin(), header.end(), '\r'), header.end()); + header.erase(remove(header.begin(), header.end(), ' '), header.end()); headers.push_back(header); } } @@ -43,6 +45,9 @@ void CsvParser::parse(const string& filename) while(getline(ss, value, ',')){ if(col < headers.size()){ + value.erase(remove(value.begin(), value.end(), '\r'), value.end()); + value.erase(remove(value.begin(), value.end(), ' '), value.end()); + row[headers[col]] = value; } col++; diff --git a/tests/python/example1.csv b/tests/python/example1.csv index 036e515..5bed549 100644 --- a/tests/python/example1.csv +++ b/tests/python/example1.csv @@ -1,5 +1,5 @@ id,name,age,score 1,Alice,30,88 -2,Bob,,72 +2,Bob,1,72 3,Charlie,35,95 -4,Diana,,85 \ No newline at end of file +4,Diana,1,85 \ No newline at end of file From 6ed15112918d5347a9450173237dbbb73942c079 Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Thu, 9 Oct 2025 08:48:34 +0200 Subject: [PATCH 03/18] Complete cpp_cli_tool file as according to DoD --- apps/cpp_cli_tool/cli.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apps/cpp_cli_tool/cli.cc b/apps/cpp_cli_tool/cli.cc index e0d01c3..12d169e 100644 --- a/apps/cpp_cli_tool/cli.cc +++ b/apps/cpp_cli_tool/cli.cc @@ -51,7 +51,8 @@ int main(int argc, char* argv[]){ std::cout << std::setfill('=') << std::setw(25 + strlen("SCORES-STATISTICS")) << "SCORES-STATISTICS" << std::setw(25) << '\n'; std::cout << std::setfill(' ') << "Mean: " << stats.calcMean() << std::setw(strlen("Median: ") + 2); - std::cout << "Median: " << stats.calcMedian(); + std::cout << "Median: " << stats.calcMedian() << std::setw(strlen("Variance: " + 2)) << " Variance: " << stats.calcVariance(); + std::cout << std::setw(strlen("Standard deviation: ") + 2) << " Standard deviation: " << stats.calcStandardDeviation(); auto data = parserPtr->getEntries(); std::vector scores {}; @@ -73,15 +74,15 @@ int main(int argc, char* argv[]){ }else{ std::cout << std::setw(strlen("Mode: ") + 2) << "Mode: " << stats.calcMode()[0]; } - std::cout << std::endl; + std::cout << "\n" << std::endl; - std::cout << "Frequencies of scores: " << "\n"; + std::cout << std::setfill('=') << std::setw(25 + strlen("FREQUENCIES-OF-SCORES")) << "FREQUENCIES-OF-SCORES" << std::setw(25) << "\n"; for (auto &&[k,v] : stats.calcFrequencies()) { std::cout << k << " appeared " << v << " time(s) "; } std::cout << std::endl; - + std::cout << "\n\n"; return 0; } \ No newline at end of file From ca9d6133f0b71e59f7a03ba8fcc44663395eaa99 Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Tue, 14 Oct 2025 16:52:25 +0200 Subject: [PATCH 04/18] Commit for run_pipeline.sh including changes to data_cleaner (#20) --- apps/cpp_cli_tool/BUILD | 7 ++- apps/cpp_cli_tool/cli.cc | 13 +++-- python_tools/data_cleaner/cleaner.py | 43 ++++++++++++---- requirements_lock.txt | 76 ++++++++++++++++++++++++++++ scripts/BUILD | 29 +++++++++++ scripts/run_pipeline.sh | 50 ++++++++++++++++++ tests/python/example1.csv | 4 +- 7 files changed, 201 insertions(+), 21 deletions(-) diff --git a/apps/cpp_cli_tool/BUILD b/apps/cpp_cli_tool/BUILD index 8253c9a..10a9cea 100644 --- a/apps/cpp_cli_tool/BUILD +++ b/apps/cpp_cli_tool/BUILD @@ -6,10 +6,9 @@ cc_binary( "//libs/math:stats", "@bazel_tools//tools/cpp/runfiles", ], + visibility = ["//scripts:__subpackages__"], + + data = ["//tests/python:testdata"], - data = [ - "//tests/python:example1.json", - "//tests/python:example1.csv", - ], ) diff --git a/apps/cpp_cli_tool/cli.cc b/apps/cpp_cli_tool/cli.cc index 12d169e..a8d5af5 100644 --- a/apps/cpp_cli_tool/cli.cc +++ b/apps/cpp_cli_tool/cli.cc @@ -30,12 +30,15 @@ int main(int argc, char* argv[]){ std::string filename = argv[1]; std::string filetype = filename.substr(filename.find_last_of(".") + 1); - - std::string err; - std::unique_ptr runfiles(bazel::tools::cpp::runfiles::Runfiles::Create(argv[0], &err)); + auto runfiles(bazel::tools::cpp::runfiles::Runfiles::Create(argv[0], &err)); + if (runfiles == nullptr) { + std::cerr << "Error initializing runfiles: " << err << std::endl; + return 1; + } + + std::string filepath = runfiles->Rlocation(filename); //<-- Will cause a seg fault if runfiles==nullptr :( - std::string filepath = runfiles->Rlocation(filename); //Create parser w. parserfactory static shared_ptr parserPtr; @@ -84,5 +87,7 @@ int main(int argc, char* argv[]){ std::cout << std::endl; std::cout << "\n\n"; + + return 0; } \ No newline at end of file diff --git a/python_tools/data_cleaner/cleaner.py b/python_tools/data_cleaner/cleaner.py index 34537d2..38c0abd 100644 --- a/python_tools/data_cleaner/cleaner.py +++ b/python_tools/data_cleaner/cleaner.py @@ -9,14 +9,15 @@ class DataCleaner: """ Class to read, clean and write the content of csv or json files. - :atribute str filename: Name of the file being processed. - :atribute str filetype: csv or json depending on filenames ending. - :atribute list filecontent: empty list representeing the content of the file being read. - :atribute runfiles r: Object which enables file reading in bazel environments. + :attribute str filename: Name of the file being processed. + :attribute str filetype: csv or json depending on filenames ending. + :attribute list filecontent: empty list representeing the content of the file being read. + :attribute runfiles r: Object which enables file reading in bazel environments. """ - def __init__(self): + def __init__(self, output_dir=None): self.filename = "" self.filetype = "" + self.output_dir = output_dir self.filecontent = [] self.r = runfiles.Create() @@ -46,13 +47,17 @@ def initArgs(self): :param str filepath: filepath to the file which should be read. """ def readFile(self, filepath): + workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', 'data_processing_pipeline') + #workspace_root = os.environ.get('BUILD_WORKSPACE_DIRECTORY', "data_processing_pipeline") + #filepath = os.path.join(workspace_root, filepath) - data_location = self.r.Rlocation(f"{workspace}/{filepath}") + data_location = self.r.Rlocation(filepath) self.filetype = self.detect_filetype(filepath) + self.output = filepath.split(".") - self.output = f"{self.output[0]}_cleaned.{self.output[1]}" + self.output = f"{os.path.splitext(filepath)[0]}_cleaned{os.path.splitext(filepath)[1]}" #_cleaned self.filecontent.clear() @@ -75,16 +80,28 @@ def readFile(self, filepath): def writeFile(self): workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', 'data_processing_pipeline') - data_location = self.r.Rlocation(f"{workspace}/{self.output}") + #Check if the current output path exists, if not create the directories + + basename, ext = os.path.splitext(os.path.basename(self.filename)) + cleaned_filename = f"{basename}_cleaned.{self.filetype}" + + if self.output_dir: #If outputdir is provided + output_dir = os.path.join(workspace, self.output_dir) + else: + input_dir = os.path.dirname(self.filename) + output_dir = os.path.join(workspace, input_dir) + + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join(output_dir, cleaned_filename) if(self.filetype == "csv"): - with(open(data_location, mode='w')) as file: + with(open(output_path, mode='w')) as file: csv_writer = csv.DictWriter(file, fieldnames=self.filecontent[0].keys()) csv_writer.writeheader() csv_writer.writerows(self.filecontent) elif(self.filetype == "json"): - with(open(data_location, mode='w')) as file: + with(open(output_path, mode='w')) as file: json.dump(self.filecontent, file) """ @@ -135,6 +152,7 @@ def main(): parser = argparse.ArgumentParser(description="Clean CSV/JSON files") parser.add_argument("filepath", help="Path to the input file") parser.add_argument("--read-file-only", action="store_true", help="Only read the file and not clean") + parser.add_argument("--output-dir", default=None, help="Optional directory to write cleaned ouput files to (default: same directory as input)") args = parser.parse_args() cleaner = DataCleaner() @@ -142,6 +160,9 @@ def main(): cleaner.readFile(filepath=args.filepath) else: cleaner.readFileAndClean(filepath=args.filepath) + + return parser.parse_args() if __name__ == '__main__': - main() \ No newline at end of file + args = main() + #cleaner = DataCleaner(args.filepath, output_dir=args.output_dir) \ No newline at end of file diff --git a/requirements_lock.txt b/requirements_lock.txt index e69de29..9248e4f 100644 --- a/requirements_lock.txt +++ b/requirements_lock.txt @@ -0,0 +1,76 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# bazel run //:requirements.update +# +iniconfig==2.1.0 \ + --hash=sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7 \ + --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 + # via pytest +numpy==2.1.0 \ + --hash=sha256:08801848a40aea24ce16c2ecde3b756f9ad756586fb2d13210939eb69b023f5b \ + --hash=sha256:0937e54c09f7a9a68da6889362ddd2ff584c02d015ec92672c099b61555f8911 \ + --hash=sha256:0ab32eb9170bf8ffcbb14f11613f4a0b108d3ffee0832457c5d4808233ba8977 \ + --hash=sha256:0abb3916a35d9090088a748636b2c06dc9a6542f99cd476979fb156a18192b84 \ + --hash=sha256:0af3a5987f59d9c529c022c8c2a64805b339b7ef506509fba7d0556649b9714b \ + --hash=sha256:10e2350aea18d04832319aac0f887d5fcec1b36abd485d14f173e3e900b83e33 \ + --hash=sha256:15ef8b2177eeb7e37dd5ef4016f30b7659c57c2c0b57a779f1d537ff33a72c7b \ + --hash=sha256:1f817c71683fd1bb5cff1529a1d085a57f02ccd2ebc5cd2c566f9a01118e3b7d \ + --hash=sha256:24003ba8ff22ea29a8c306e61d316ac74111cebf942afbf692df65509a05f111 \ + --hash=sha256:30014b234f07b5fec20f4146f69e13cfb1e33ee9a18a1879a0142fbb00d47673 \ + --hash=sha256:343e3e152bf5a087511cd325e3b7ecfd5b92d369e80e74c12cd87826e263ec06 \ + --hash=sha256:378cb4f24c7d93066ee4103204f73ed046eb88f9ad5bb2275bb9fa0f6a02bd36 \ + --hash=sha256:398049e237d1aae53d82a416dade04defed1a47f87d18d5bd615b6e7d7e41d1f \ + --hash=sha256:3a3336fbfa0d38d3deacd3fe7f3d07e13597f29c13abf4d15c3b6dc2291cbbdd \ + --hash=sha256:442596f01913656d579309edcd179a2a2f9977d9a14ff41d042475280fc7f34e \ + --hash=sha256:44e44973262dc3ae79e9063a1284a73e09d01b894b534a769732ccd46c28cc62 \ + --hash=sha256:54139e0eb219f52f60656d163cbe67c31ede51d13236c950145473504fa208cb \ + --hash=sha256:5474dad8c86ee9ba9bb776f4b99ef2d41b3b8f4e0d199d4f7304728ed34d0300 \ + --hash=sha256:54c6a63e9d81efe64bfb7bcb0ec64332a87d0b87575f6009c8ba67ea6374770b \ + --hash=sha256:624884b572dff8ca8f60fab591413f077471de64e376b17d291b19f56504b2bb \ + --hash=sha256:6326ab99b52fafdcdeccf602d6286191a79fe2fda0ae90573c5814cd2b0bc1b8 \ + --hash=sha256:652e92fc409e278abdd61e9505649e3938f6d04ce7ef1953f2ec598a50e7c195 \ + --hash=sha256:6c1de77ded79fef664d5098a66810d4d27ca0224e9051906e634b3f7ead134c2 \ + --hash=sha256:76368c788ccb4f4782cf9c842b316140142b4cbf22ff8db82724e82fe1205dce \ + --hash=sha256:7a894c51fd8c4e834f00ac742abad73fc485df1062f1b875661a3c1e1fb1c2f6 \ + --hash=sha256:7dc90da0081f7e1da49ec4e398ede6a8e9cc4f5ebe5f9e06b443ed889ee9aaa2 \ + --hash=sha256:848c6b5cad9898e4b9ef251b6f934fa34630371f2e916261070a4eb9092ffd33 \ + --hash=sha256:899da829b362ade41e1e7eccad2cf274035e1cb36ba73034946fccd4afd8606b \ + --hash=sha256:8ab81ccd753859ab89e67199b9da62c543850f819993761c1e94a75a814ed667 \ + --hash=sha256:8fb49a0ba4d8f41198ae2d52118b050fd34dace4b8f3fb0ee34e23eb4ae775b1 \ + --hash=sha256:9156ca1f79fc4acc226696e95bfcc2b486f165a6a59ebe22b2c1f82ab190384a \ + --hash=sha256:9523f8b46485db6939bd069b28b642fec86c30909cea90ef550373787f79530e \ + --hash=sha256:a0756a179afa766ad7cb6f036de622e8a8f16ffdd55aa31f296c870b5679d745 \ + --hash=sha256:a0cdef204199278f5c461a0bed6ed2e052998276e6d8ab2963d5b5c39a0500bc \ + --hash=sha256:ab83adc099ec62e044b1fbb3a05499fa1e99f6d53a1dde102b2d85eff66ed324 \ + --hash=sha256:b34fa5e3b5d6dc7e0a4243fa0f81367027cb6f4a7215a17852979634b5544ee0 \ + --hash=sha256:b47c551c6724960479cefd7353656498b86e7232429e3a41ab83be4da1b109e8 \ + --hash=sha256:c4cd94dfefbefec3f8b544f61286584292d740e6e9d4677769bc76b8f41deb02 \ + --hash=sha256:c4f982715e65036c34897eb598d64aef15150c447be2cfc6643ec7a11af06574 \ + --hash=sha256:d8f699a709120b220dfe173f79c73cb2a2cab2c0b88dd59d7b49407d032b8ebd \ + --hash=sha256:dd94ce596bda40a9618324547cfaaf6650b1a24f5390350142499aa4e34e53d1 \ + --hash=sha256:de844aaa4815b78f6023832590d77da0e3b6805c644c33ce94a1e449f16d6ab5 \ + --hash=sha256:e5f0642cdf4636198a4990de7a71b693d824c56a757862230454629cf62e323d \ + --hash=sha256:f07fa2f15dabe91259828ce7d71b5ca9e2eb7c8c26baa822c825ce43552f4883 \ + --hash=sha256:f15976718c004466406342789f31b6673776360f3b1e3c575f25302d7e789575 \ + --hash=sha256:f358ea9e47eb3c2d6eba121ab512dfff38a88db719c38d1e67349af210bc7529 \ + --hash=sha256:f505264735ee074250a9c78247ee8618292091d9d1fcc023290e9ac67e8f1afa \ + --hash=sha256:f5ebbf9fbdabed208d4ecd2e1dfd2c0741af2f876e7ae522c2537d404ca895c3 \ + --hash=sha256:f6b26e6c3b98adb648243670fddc8cab6ae17473f9dc58c51574af3e64d61211 \ + --hash=sha256:f8e93a01a35be08d31ae33021e5268f157a2d60ebd643cfc15de6ab8e4722eb1 \ + --hash=sha256:fe76d75b345dc045acdbc006adcb197cc680754afd6c259de60d358d60c93736 \ + --hash=sha256:ffbd6faeb190aaf2b5e9024bac9622d2ee549b7ec89ef3a9373fa35313d44e0e + # via -r requirements.txt +packaging==25.0 \ + --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ + --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f + # via pytest +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via pytest +pytest==8.3.2 \ + --hash=sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5 \ + --hash=sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce + # via -r requirements.txt diff --git a/scripts/BUILD b/scripts/BUILD index e69de29..20cdaa4 100644 --- a/scripts/BUILD +++ b/scripts/BUILD @@ -0,0 +1,29 @@ +sh_library( + name = "run_pipeline_lib", + visibility = ["//visibility:public"], + #srcs = ["run_pipeline.sh"], + data = [ + ":run_pipeline", #Sh_binary below + ], + +) + +sh_binary( + name = "run_pipeline", + srcs = ["run_pipeline.sh",], + #data = ["//tests/python:testdata",], + data = [ + "//apps/cpp_cli_tool:cli", + "//python_tools/data_cleaner:cleaner", + "//tests/python:testdata", + ":bash_data" + ], + visibility = ["//visibility:__subpackages__"], + #deps = [":run_pipeline_lib"], +) + +filegroup( + name = "bash_data", + srcs = glob(["*.csv", "*.json"]), + visibility = ["//:__subpackages__"], +) \ No newline at end of file diff --git a/scripts/run_pipeline.sh b/scripts/run_pipeline.sh index e69de29..9aea743 100644 --- a/scripts/run_pipeline.sh +++ b/scripts/run_pipeline.sh @@ -0,0 +1,50 @@ +#Order of running pipeline: data_cleaning --> cpp_cli_tool --> report_generator (?) +#Called by running bazel run //scripts:run_pipeline data.csv (arg1) + +set -e + +INPUT_FILE="$1" #$1 is the first argument, in this scenario the filepath + +if [-z $INPUT_FILE]; then + echo "Usage: bazel run //scripts:run_pipeline " + exit 1 +fi + +#Run CLI tool (?) + +echo "====================RUNNING-PIPELINE-ON-$INPUT_FILE====================" + +#bazel run //python_tools/data_cleaner:cleaner $INPUT_FILE +#bazel run //cpp_cli_tool:cli $INPUT_FILE +#RUN REPORT GENERATOR HERE EVENTUALLY + +echo "Fetching directories..." + +WORKSPACE_DIR="$(bazel info workspace)" +BIN_DIR="$(bazel info bazel-bin)" + + +CLI_TARGET="//apps/cpp_cli_tool:cli" +PY_TARGET="//python_tools/data_cleaner:cleaner" + +#CLI_BIN="$BIN_DIR/apps/cpp_cli_tool/cli" #Path to the cli tools BUILD file +#PY_BIN="$BIN_DIR/python_tools/data_cleaner/cleaner" + +echo "Fetched directories..." + +bazel build $CLI_TARGET $PY_TARGET + +echo "Running C++ CLI tool on $INPUT_FILE..." +#Call using bazel instead of direct path to bin +bazel run $CLI_TARGET -- $INPUT_FILE + +#"$CLI_BIN" "$INPUT_FILE" + +echo "Running Python data cleaner tool on $INPUT_FILE..." +bazel run $PY_TARGET -- $INPUT_FILE "--output-dir=outputs/cleaned" + +# bazel-bin/scripts/run_pipeline.sh data_processing_pipeline/tests/python/example1.json + +echo "====================PIPELINE-SUCCESSFUL====================" + +chmod +x scripts/run_pipeline.sh \ No newline at end of file diff --git a/tests/python/example1.csv b/tests/python/example1.csv index 5bed549..036e515 100644 --- a/tests/python/example1.csv +++ b/tests/python/example1.csv @@ -1,5 +1,5 @@ id,name,age,score 1,Alice,30,88 -2,Bob,1,72 +2,Bob,,72 3,Charlie,35,95 -4,Diana,1,85 \ No newline at end of file +4,Diana,,85 \ No newline at end of file From 994bffb943b3ea339ab177bf58df3b27de531958 Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Wed, 15 Oct 2025 14:37:02 +0200 Subject: [PATCH 05/18] Commit before integrating report generator (#20) --- python_tools/data_cleaner/cleaner.py | 65 ++++++++++++++-------------- scripts/run_pipeline.sh | 19 ++++++-- tests/python/BUILD | 1 + tests/python/macros.bzl | 2 +- tests/python/test_cleaner.py | 16 +++---- 5 files changed, 58 insertions(+), 45 deletions(-) diff --git a/python_tools/data_cleaner/cleaner.py b/python_tools/data_cleaner/cleaner.py index 38c0abd..4aed879 100644 --- a/python_tools/data_cleaner/cleaner.py +++ b/python_tools/data_cleaner/cleaner.py @@ -15,8 +15,8 @@ class DataCleaner: :attribute runfiles r: Object which enables file reading in bazel environments. """ def __init__(self, output_dir=None): - self.filename = "" - self.filetype = "" + self.filename = None + self.filetype = None self.output_dir = output_dir self.filecontent = [] self.r = runfiles.Create() @@ -35,29 +35,15 @@ def detect_filetype(self, filepath): else: raise ValueError("Unsupported filetype") - """ - :Method to initialize the terminal arguments to call tests - """ - def initArgs(self): - self.parser.add_argument("filepath", help = "Path to the input file") - self.parser.add_argument("--read-file-only", action="store_true", help="Reading a file without cleaning the data") - """ Method to read content of file depending on the filetype. :param str filepath: filepath to the file which should be read. """ def readFile(self, filepath): - - workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', 'data_processing_pipeline') - #workspace_root = os.environ.get('BUILD_WORKSPACE_DIRECTORY', "data_processing_pipeline") - #filepath = os.path.join(workspace_root, filepath) - + print(f"===============================================================READING-FILEPATH: {filepath}===============================================================") data_location = self.r.Rlocation(filepath) - - self.filetype = self.detect_filetype(filepath) - - self.output = filepath.split(".") - self.output = f"{os.path.splitext(filepath)[0]}_cleaned{os.path.splitext(filepath)[1]}" #_cleaned + self.filename = filepath + self.filetype = self.detect_filetype(filepath) self.filecontent.clear() @@ -78,21 +64,35 @@ def readFile(self, filepath): Method to write the content of self.filecontent to a filepath. """ def writeFile(self): - workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', 'data_processing_pipeline') + workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', "./") - #Check if the current output path exists, if not create the directories + basename, ext = os.path.splitext(os.path.basename(self.filename)) # -> ('tests/python/example1', '.csv') + cleaned_filename = f"{basename}_cleaned{ext}" # -> 'tests/python/example1_cleaned.csv' - basename, ext = os.path.splitext(os.path.basename(self.filename)) - cleaned_filename = f"{basename}_cleaned.{self.filetype}" + if self.output_dir is None: + print("No output directory passed \n") + + if self.output_dir: + #If output_dir is absolute, use it as is, else, join with workspace + output_dir = self.output_dir if os.path.isabs(self.output_dir) else os.path.join(workspace, self.output_dir) # This creates a output like: cwd/output/cleaned/ - if self.output_dir: #If outputdir is provided - output_dir = os.path.join(workspace, self.output_dir) + """ else: - input_dir = os.path.dirname(self.filename) - output_dir = os.path.join(workspace, input_dir) + input_dir = os.path.dirname(self.filename) # -> Returns a path like data_processing_pipeline/tests/python + output_dir = os.path.join(workspace, input_dir) # Right now creates output_dir like: /data_processing_pipeline/, need to remove one data_processing_pipeline before joining + """ + print("Workspace: " + workspace + '\n' + "Cleaned filename: " + cleaned_filename + '\n' + "Output dir: " + output_dir + '\n' + "self.filename " + self.filename + '\n' + "basename: " + basename + '\n' + "ext: " + ext + '\n' + ) - os.makedirs(output_dir, exist_ok=True) - output_path = os.path.join(output_dir, cleaned_filename) + os.makedirs(output_dir, exist_ok=True) #Makes dir as specified above + output_path = os.path.join(output_dir, cleaned_filename) # -> '/output/cleaned/tests/python/example1_cleaned.csv, Hmm still results in -> /output/cleaned/tests/python/_cleaned + + if(self.filetype == "csv"): with(open(output_path, mode='w')) as file: @@ -154,15 +154,14 @@ def main(): parser.add_argument("--read-file-only", action="store_true", help="Only read the file and not clean") parser.add_argument("--output-dir", default=None, help="Optional directory to write cleaned ouput files to (default: same directory as input)") args = parser.parse_args() - cleaner = DataCleaner() + cleaner = DataCleaner(output_dir=args.output_dir) # Pass the output_dir here if args.read_file_only: cleaner.readFile(filepath=args.filepath) else: cleaner.readFileAndClean(filepath=args.filepath) - return parser.parse_args() + return args if __name__ == '__main__': - args = main() - #cleaner = DataCleaner(args.filepath, output_dir=args.output_dir) \ No newline at end of file + args = main() \ No newline at end of file diff --git a/scripts/run_pipeline.sh b/scripts/run_pipeline.sh index 9aea743..c1cbd35 100644 --- a/scripts/run_pipeline.sh +++ b/scripts/run_pipeline.sh @@ -4,6 +4,7 @@ set -e INPUT_FILE="$1" #$1 is the first argument, in this scenario the filepath +OUTPUT_DIR="$2" if [-z $INPUT_FILE]; then echo "Usage: bazel run //scripts:run_pipeline " @@ -41,10 +42,22 @@ bazel run $CLI_TARGET -- $INPUT_FILE #"$CLI_BIN" "$INPUT_FILE" echo "Running Python data cleaner tool on $INPUT_FILE..." -bazel run $PY_TARGET -- $INPUT_FILE "--output-dir=outputs/cleaned" +bazel run $PY_TARGET -- $INPUT_FILE "--output-dir=$OUTPUT_DIR" -# bazel-bin/scripts/run_pipeline.sh data_processing_pipeline/tests/python/example1.json +echo "Generating report of $INPUT_FILE..." + +#Build report generator with output dir of cleaner in build file + +#Run report generator with output directory as param + +echo "Pipeline finished, check $OUTPUT_DIR for results" echo "====================PIPELINE-SUCCESSFUL====================" -chmod +x scripts/run_pipeline.sh \ No newline at end of file + +chmod +x scripts/run_pipeline.sh + +#TO RUN: + +# bash scripts/run_pipeline.sh +# example: bash scripts/run_pipeline.sh data_processing_pipeline/tests/python/example3.csv output/tests \ No newline at end of file diff --git a/tests/python/BUILD b/tests/python/BUILD index e0c7221..8a72cfe 100644 --- a/tests/python/BUILD +++ b/tests/python/BUILD @@ -11,6 +11,7 @@ pytest_test( "//tests/python/pytest:pytest_wrapper", ], imports = ["."], + data = glob(["*.csv", "*.json"]) ) filegroup( diff --git a/tests/python/macros.bzl b/tests/python/macros.bzl index 94650d9..cdf83f9 100644 --- a/tests/python/macros.bzl +++ b/tests/python/macros.bzl @@ -5,7 +5,7 @@ default_args = [ "--disable-warnings", "-v", "--tb=short", - "-color=yes", + "--color=yes", "-rfEP", ] diff --git a/tests/python/test_cleaner.py b/tests/python/test_cleaner.py index ab69e54..052d0dd 100644 --- a/tests/python/test_cleaner.py +++ b/tests/python/test_cleaner.py @@ -2,8 +2,8 @@ def test_handleNaNsCSV(): - cleaner = DataCleaner() - cleaner.readFile("tests/python/example1.csv") + cleaner = DataCleaner() #Hmm this now needs to be passed an output_dir for writing the cleaned file to + cleaner.readFile("data_processing_pipeline/tests/python/example1.csv") cleaner.handleNaNs() cleaner.writeFile() @@ -15,7 +15,7 @@ def test_handleNaNsCSV(): def test_normalizeTextCSV(): cleaner = DataCleaner() - cleaner.readFile("tests/python/example2.csv") + cleaner.readFile("data_processing_pipeline/tests/python/example2.csv") cleaner.normalizeText() cleaner.writeFile() @@ -31,7 +31,7 @@ def test_normalizeTextCSV(): def test_handleNaNsJSON(): cleaner = DataCleaner() - cleaner.readFile("tests/python/example1.json") + cleaner.readFile("data_processing_pipeline/tests/python/example1.json") cleaner.handleNaNs() cleaner.writeFile() @@ -45,7 +45,7 @@ def test_handleNaNsJSON(): def test_normalizeTextJSON(): cleaner = DataCleaner() - cleaner.readFile("tests/python/example2.json") + cleaner.readFile("data_processing_pipeline/tests/python/example2.json") cleaner.normalizeText() cleaner.writeFile() @@ -60,7 +60,7 @@ def test_normalizeTextJSON(): def test_completeCleaningCSV(): cleaner = DataCleaner() - cleaner.readFileAndClean("tests/python/example3.csv") + cleaner.readFileAndClean("data_processing_pipeline/tests/python/example3.csv") assert cleaner.filecontent == [ {'id': '1', 'name': 'alice', 'age': '30', 'score': '88'}, @@ -70,8 +70,8 @@ def test_completeCleaningCSV(): ] def test_completeCleaningJSON(): - cleaner = DataCleaner() - cleaner.readFileAndClean("tests/python/example3.json") + cleaner = DataCleaner() + cleaner.readFileAndClean("data_processing_pipeline/tests/python/example3.json") assert cleaner.filecontent == [ {'id': 1, 'name': 'alice', 'age': 30, 'score': 88}, From c310c24009760de6ad81d82248d687015ac0be29 Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Thu, 16 Oct 2025 11:56:47 +0200 Subject: [PATCH 06/18] Functional pipeline for one input file, hardcoded in libs/math/build (#20) --- libs/math/BUILD | 2 +- libs/math/defs.bzl | 34 ++++++++++++++++++++++++++++++++-- libs/math/generate_stats.cc | 2 +- scripts/run_pipeline.sh | 2 ++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/libs/math/BUILD b/libs/math/BUILD index a7b1a49..1f1cdaf 100644 --- a/libs/math/BUILD +++ b/libs/math/BUILD @@ -32,7 +32,7 @@ gen_stats( # take the cc_binary as exec tool tool = ":generate_stats", data = [ - "//examples:example.json", + "//output/test:example1_cleaned.json", ], visibility = [ "//:__subpackages__", diff --git a/libs/math/defs.bzl b/libs/math/defs.bzl index c6be4cd..5415097 100644 --- a/libs/math/defs.bzl +++ b/libs/math/defs.bzl @@ -1,12 +1,29 @@ def _gen_stats_impl(ctx): out = ctx.actions.declare_file("output.json") + #print("LEN CTX DATA: " + str(len(ctx.attr.data))) + #print(type(ctx.attr.data[0])) + data_files = [] + for t in ctx.attr.data: + if DefaultInfo in t: + data_files.extend(t[DefaultInfo].files.to_list()) + print("Extended data files with defaultinfo: " ) + print(t[DefaultInfo].data_runfiles.files.to_list()) + print(t[DefaultInfo].default_runfiles.files.to_list()) + #print(t[DefaultInfo].files_to_run.to_list()) + + else: + fail("Target %s does not provide DefaultInfo" % t.label) + + print("Collected data files:", [f.path for f in data_files]) + + #first_data = ctx.files.data[0][0] if type(ctx.attr.data) == list else ctx.files.data[0] # run cc_binary here ctx.actions.run( inputs = ctx.files.data, outputs = [out], # pass input and output file name as arguments - arguments = [ctx.files.data[0].path, out.path], + arguments = [collect_files(ctx.attr.data)[0].path, out.path], executable = ctx.executable.tool, tools = [ctx.executable.tool], use_default_shell_env = True, @@ -16,10 +33,23 @@ def _gen_stats_impl(ctx): files = depset([out]), )] + gen_stats = rule( implementation = _gen_stats_impl, attrs = { "tool": attr.label(executable = True, cfg = "exec"), "data": attr.label_list(allow_files = True), }, -) \ No newline at end of file +) + +def collect_files(targets): + files = [] + for t in targets: + if DefaultInfo in t: + info = t[DefaultInfo] + files.extend(info.files.to_list()) + files.extend(info.default_runfiles.files.to_list()) + files.extend(info.data_runfiles.files.to_list()) + + print(files) + return files \ No newline at end of file diff --git a/libs/math/generate_stats.cc b/libs/math/generate_stats.cc index c857ed5..e8b619b 100644 --- a/libs/math/generate_stats.cc +++ b/libs/math/generate_stats.cc @@ -14,7 +14,7 @@ int main(int argc, char* argv[]) } string filePath = argv[1]; - + //filePath = "data_processing_pipeline/" + filePath; int totalSize = filePath.size(); int suffixSize = 4; diff --git a/scripts/run_pipeline.sh b/scripts/run_pipeline.sh index c1cbd35..92105b5 100644 --- a/scripts/run_pipeline.sh +++ b/scripts/run_pipeline.sh @@ -52,6 +52,8 @@ echo "Generating report of $INPUT_FILE..." echo "Pipeline finished, check $OUTPUT_DIR for results" +bazel run //python_tools/report_generator:generator $OUTPUT_DIR + echo "====================PIPELINE-SUCCESSFUL====================" From 5c6e4fdece09a3af208410fe5299cd49dc1b61c6 Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Thu, 16 Oct 2025 13:40:42 +0200 Subject: [PATCH 07/18] Completed pipeline with hardcoded inputfile for rule, to run: bash scripts/run_pipeline.sh tests/python/example1.json output/test --- .github/workflows/ci.yml | 2 ++ libs/math/defs.bzl | 34 ++-------------------------- python_tools/data_cleaner/cleaner.py | 15 ++++++------ scripts/run_pipeline.sh | 6 ++--- tests/python/test_cleaner.py | 12 +++++----- 5 files changed, 20 insertions(+), 49 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 437957a..a2476c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,3 +37,5 @@ jobs: with: name: test-logs path: bazel-testlogs/tests/**/test.log + - name: run pipeline + run: bash scripts/run_pipeline.sh tests/python/example1.json output/test diff --git a/libs/math/defs.bzl b/libs/math/defs.bzl index 5415097..4aab4ca 100644 --- a/libs/math/defs.bzl +++ b/libs/math/defs.bzl @@ -1,29 +1,12 @@ def _gen_stats_impl(ctx): out = ctx.actions.declare_file("output.json") - #print("LEN CTX DATA: " + str(len(ctx.attr.data))) - #print(type(ctx.attr.data[0])) - data_files = [] - for t in ctx.attr.data: - if DefaultInfo in t: - data_files.extend(t[DefaultInfo].files.to_list()) - print("Extended data files with defaultinfo: " ) - print(t[DefaultInfo].data_runfiles.files.to_list()) - print(t[DefaultInfo].default_runfiles.files.to_list()) - #print(t[DefaultInfo].files_to_run.to_list()) - - else: - fail("Target %s does not provide DefaultInfo" % t.label) - - print("Collected data files:", [f.path for f in data_files]) - - #first_data = ctx.files.data[0][0] if type(ctx.attr.data) == list else ctx.files.data[0] # run cc_binary here ctx.actions.run( inputs = ctx.files.data, outputs = [out], # pass input and output file name as arguments - arguments = [collect_files(ctx.attr.data)[0].path, out.path], + arguments = [ctx.attr.data[0].path, out.path], executable = ctx.executable.tool, tools = [ctx.executable.tool], use_default_shell_env = True, @@ -33,23 +16,10 @@ def _gen_stats_impl(ctx): files = depset([out]), )] - gen_stats = rule( implementation = _gen_stats_impl, attrs = { "tool": attr.label(executable = True, cfg = "exec"), "data": attr.label_list(allow_files = True), }, -) - -def collect_files(targets): - files = [] - for t in targets: - if DefaultInfo in t: - info = t[DefaultInfo] - files.extend(info.files.to_list()) - files.extend(info.default_runfiles.files.to_list()) - files.extend(info.data_runfiles.files.to_list()) - - print(files) - return files \ No newline at end of file +) \ No newline at end of file diff --git a/python_tools/data_cleaner/cleaner.py b/python_tools/data_cleaner/cleaner.py index 4aed879..997ea07 100644 --- a/python_tools/data_cleaner/cleaner.py +++ b/python_tools/data_cleaner/cleaner.py @@ -40,7 +40,6 @@ def detect_filetype(self, filepath): :param str filepath: filepath to the file which should be read. """ def readFile(self, filepath): - print(f"===============================================================READING-FILEPATH: {filepath}===============================================================") data_location = self.r.Rlocation(filepath) self.filename = filepath self.filetype = self.detect_filetype(filepath) @@ -81,13 +80,13 @@ def writeFile(self): input_dir = os.path.dirname(self.filename) # -> Returns a path like data_processing_pipeline/tests/python output_dir = os.path.join(workspace, input_dir) # Right now creates output_dir like: /data_processing_pipeline/, need to remove one data_processing_pipeline before joining """ - print("Workspace: " + workspace + '\n' - "Cleaned filename: " + cleaned_filename + '\n' - "Output dir: " + output_dir + '\n' - "self.filename " + self.filename + '\n' - "basename: " + basename + '\n' - "ext: " + ext + '\n' - ) + #print("Workspace: " + workspace + '\n' + # "Cleaned filename: " + cleaned_filename + '\n' + # "Output dir: " + output_dir + '\n' + # "self.filename " + self.filename + '\n' + # "basename: " + basename + '\n' + # "ext: " + ext + '\n' + # ) os.makedirs(output_dir, exist_ok=True) #Makes dir as specified above output_path = os.path.join(output_dir, cleaned_filename) # -> '/output/cleaned/tests/python/example1_cleaned.csv, Hmm still results in -> /output/cleaned/tests/python/_cleaned diff --git a/scripts/run_pipeline.sh b/scripts/run_pipeline.sh index 92105b5..bc6e3f2 100644 --- a/scripts/run_pipeline.sh +++ b/scripts/run_pipeline.sh @@ -21,7 +21,7 @@ echo "====================RUNNING-PIPELINE-ON-$INPUT_FILE====================" echo "Fetching directories..." -WORKSPACE_DIR="$(bazel info workspace)" +WORKSPACE_DIR="$(bazel info workspace)/" BIN_DIR="$(bazel info bazel-bin)" @@ -37,12 +37,12 @@ bazel build $CLI_TARGET $PY_TARGET echo "Running C++ CLI tool on $INPUT_FILE..." #Call using bazel instead of direct path to bin -bazel run $CLI_TARGET -- $INPUT_FILE +bazel run $CLI_TARGET -- $WORKSPACE_DIR$INPUT_FILE #"$CLI_BIN" "$INPUT_FILE" echo "Running Python data cleaner tool on $INPUT_FILE..." -bazel run $PY_TARGET -- $INPUT_FILE "--output-dir=$OUTPUT_DIR" +bazel run $PY_TARGET -- $WORKSPACE_DIR$INPUT_FILE "--output-dir=$OUTPUT_DIR" echo "Generating report of $INPUT_FILE..." diff --git a/tests/python/test_cleaner.py b/tests/python/test_cleaner.py index 052d0dd..b40576d 100644 --- a/tests/python/test_cleaner.py +++ b/tests/python/test_cleaner.py @@ -2,7 +2,7 @@ def test_handleNaNsCSV(): - cleaner = DataCleaner() #Hmm this now needs to be passed an output_dir for writing the cleaned file to + cleaner = DataCleaner("tests/python") #Hmm this now needs to be passed an output_dir for writing the cleaned file to cleaner.readFile("data_processing_pipeline/tests/python/example1.csv") cleaner.handleNaNs() cleaner.writeFile() @@ -14,7 +14,7 @@ def test_handleNaNsCSV(): ] def test_normalizeTextCSV(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFile("data_processing_pipeline/tests/python/example2.csv") cleaner.normalizeText() cleaner.writeFile() @@ -30,7 +30,7 @@ def test_normalizeTextCSV(): def test_handleNaNsJSON(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFile("data_processing_pipeline/tests/python/example1.json") cleaner.handleNaNs() cleaner.writeFile() @@ -44,7 +44,7 @@ def test_handleNaNsJSON(): def test_normalizeTextJSON(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFile("data_processing_pipeline/tests/python/example2.json") cleaner.normalizeText() cleaner.writeFile() @@ -59,7 +59,7 @@ def test_normalizeTextJSON(): def test_completeCleaningCSV(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFileAndClean("data_processing_pipeline/tests/python/example3.csv") assert cleaner.filecontent == [ @@ -70,7 +70,7 @@ def test_completeCleaningCSV(): ] def test_completeCleaningJSON(): - cleaner = DataCleaner() + cleaner = DataCleaner("tests/python") cleaner.readFileAndClean("data_processing_pipeline/tests/python/example3.json") assert cleaner.filecontent == [ From b6b88c8ee1b93917665628ce469f3dbaf5fa5563 Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Thu, 16 Oct 2025 13:48:53 +0200 Subject: [PATCH 08/18] Included build files for the output folder, needed for the ci.yml to execute properly (#20) --- output/BUILD | 0 output/test/BUILD | 7 +++++++ 2 files changed, 7 insertions(+) create mode 100644 output/BUILD create mode 100644 output/test/BUILD diff --git a/output/BUILD b/output/BUILD new file mode 100644 index 0000000..e69de29 diff --git a/output/test/BUILD b/output/test/BUILD new file mode 100644 index 0000000..6149859 --- /dev/null +++ b/output/test/BUILD @@ -0,0 +1,7 @@ +filegroup( + name="output", + data = glob(["*.csv", "*.json"], allow_empty=True), + visibility = ["//:__subpackages__"], +) + +exports_files(glob(allow_empty=True, include=["*.csv", "*.json"])) \ No newline at end of file From 8dc3eadf45131b8a4b88a75dd987c3aaaa90325b Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Thu, 16 Oct 2025 14:01:20 +0200 Subject: [PATCH 09/18] Reverted change of defs.bzl (#20) --- libs/math/defs.bzl | 2 +- python_tools/data_cleaner/cleaner.py | 15 +-------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/libs/math/defs.bzl b/libs/math/defs.bzl index 4aab4ca..c6be4cd 100644 --- a/libs/math/defs.bzl +++ b/libs/math/defs.bzl @@ -6,7 +6,7 @@ def _gen_stats_impl(ctx): inputs = ctx.files.data, outputs = [out], # pass input and output file name as arguments - arguments = [ctx.attr.data[0].path, out.path], + arguments = [ctx.files.data[0].path, out.path], executable = ctx.executable.tool, tools = [ctx.executable.tool], use_default_shell_env = True, diff --git a/python_tools/data_cleaner/cleaner.py b/python_tools/data_cleaner/cleaner.py index 997ea07..6053a3b 100644 --- a/python_tools/data_cleaner/cleaner.py +++ b/python_tools/data_cleaner/cleaner.py @@ -73,20 +73,7 @@ def writeFile(self): if self.output_dir: #If output_dir is absolute, use it as is, else, join with workspace - output_dir = self.output_dir if os.path.isabs(self.output_dir) else os.path.join(workspace, self.output_dir) # This creates a output like: cwd/output/cleaned/ - - """ - else: - input_dir = os.path.dirname(self.filename) # -> Returns a path like data_processing_pipeline/tests/python - output_dir = os.path.join(workspace, input_dir) # Right now creates output_dir like: /data_processing_pipeline/, need to remove one data_processing_pipeline before joining - """ - #print("Workspace: " + workspace + '\n' - # "Cleaned filename: " + cleaned_filename + '\n' - # "Output dir: " + output_dir + '\n' - # "self.filename " + self.filename + '\n' - # "basename: " + basename + '\n' - # "ext: " + ext + '\n' - # ) + output_dir = self.output_dir if os.path.isabs(self.output_dir) else os.path.join(workspace, self.output_dir) # This creates a output like: cwd/output/cleaned/ os.makedirs(output_dir, exist_ok=True) #Makes dir as specified above output_path = os.path.join(output_dir, cleaned_filename) # -> '/output/cleaned/tests/python/example1_cleaned.csv, Hmm still results in -> /output/cleaned/tests/python/_cleaned From 2f8b444e7d7f479e670081cea21c359228003ca2 Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Thu, 16 Oct 2025 15:29:44 +0200 Subject: [PATCH 10/18] Resolved comments from Xin (#20) --- .github/workflows/ci.yml | 2 +- libs/math/BUILD | 2 +- libs/math/generate_stats.cc | 1 - python_tools/data_cleaner/cleaner.py | 3 ++ python_tools/report_generator/BUILD | 2 +- requirements_lock.txt | 76 ---------------------------- scripts/run_pipeline.sh | 37 ++++---------- tests/python/test_cleaner.py | 14 ++--- 8 files changed, 22 insertions(+), 115 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a2476c4..b604da3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,4 +38,4 @@ jobs: name: test-logs path: bazel-testlogs/tests/**/test.log - name: run pipeline - run: bash scripts/run_pipeline.sh tests/python/example1.json output/test + run: ./scripts/run_pipeline.sh tests/python/example1.json output/test \ No newline at end of file diff --git a/libs/math/BUILD b/libs/math/BUILD index 1f1cdaf..71b9c26 100644 --- a/libs/math/BUILD +++ b/libs/math/BUILD @@ -28,7 +28,7 @@ cc_binary( # usage: bazel build //libs/math:generate_stats_file gen_stats( - name = "generate_stats_file", + name = "generate_stats_pipeline", # take the cc_binary as exec tool tool = ":generate_stats", data = [ diff --git a/libs/math/generate_stats.cc b/libs/math/generate_stats.cc index e8b619b..4ff40ad 100644 --- a/libs/math/generate_stats.cc +++ b/libs/math/generate_stats.cc @@ -14,7 +14,6 @@ int main(int argc, char* argv[]) } string filePath = argv[1]; - //filePath = "data_processing_pipeline/" + filePath; int totalSize = filePath.size(); int suffixSize = 4; diff --git a/python_tools/data_cleaner/cleaner.py b/python_tools/data_cleaner/cleaner.py index 6053a3b..1c64532 100644 --- a/python_tools/data_cleaner/cleaner.py +++ b/python_tools/data_cleaner/cleaner.py @@ -40,6 +40,9 @@ def detect_filetype(self, filepath): :param str filepath: filepath to the file which should be read. """ def readFile(self, filepath): + workspace = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "./") + filepath = os.path.join(workspace, filepath) + data_location = self.r.Rlocation(filepath) self.filename = filepath self.filetype = self.detect_filetype(filepath) diff --git a/python_tools/report_generator/BUILD b/python_tools/report_generator/BUILD index 3bc94e1..c77242c 100644 --- a/python_tools/report_generator/BUILD +++ b/python_tools/report_generator/BUILD @@ -4,7 +4,7 @@ py_binary( # set data as custom generation rule as it produces an output file data = [ "@rules_python//python/runfiles", - "//libs/math:generate_stats_file", + "//libs/math:generate_stats_pipeline", ], ) diff --git a/requirements_lock.txt b/requirements_lock.txt index 9248e4f..e69de29 100644 --- a/requirements_lock.txt +++ b/requirements_lock.txt @@ -1,76 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.13 -# by the following command: -# -# bazel run //:requirements.update -# -iniconfig==2.1.0 \ - --hash=sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7 \ - --hash=sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760 - # via pytest -numpy==2.1.0 \ - --hash=sha256:08801848a40aea24ce16c2ecde3b756f9ad756586fb2d13210939eb69b023f5b \ - --hash=sha256:0937e54c09f7a9a68da6889362ddd2ff584c02d015ec92672c099b61555f8911 \ - --hash=sha256:0ab32eb9170bf8ffcbb14f11613f4a0b108d3ffee0832457c5d4808233ba8977 \ - --hash=sha256:0abb3916a35d9090088a748636b2c06dc9a6542f99cd476979fb156a18192b84 \ - --hash=sha256:0af3a5987f59d9c529c022c8c2a64805b339b7ef506509fba7d0556649b9714b \ - --hash=sha256:10e2350aea18d04832319aac0f887d5fcec1b36abd485d14f173e3e900b83e33 \ - --hash=sha256:15ef8b2177eeb7e37dd5ef4016f30b7659c57c2c0b57a779f1d537ff33a72c7b \ - --hash=sha256:1f817c71683fd1bb5cff1529a1d085a57f02ccd2ebc5cd2c566f9a01118e3b7d \ - --hash=sha256:24003ba8ff22ea29a8c306e61d316ac74111cebf942afbf692df65509a05f111 \ - --hash=sha256:30014b234f07b5fec20f4146f69e13cfb1e33ee9a18a1879a0142fbb00d47673 \ - --hash=sha256:343e3e152bf5a087511cd325e3b7ecfd5b92d369e80e74c12cd87826e263ec06 \ - --hash=sha256:378cb4f24c7d93066ee4103204f73ed046eb88f9ad5bb2275bb9fa0f6a02bd36 \ - --hash=sha256:398049e237d1aae53d82a416dade04defed1a47f87d18d5bd615b6e7d7e41d1f \ - --hash=sha256:3a3336fbfa0d38d3deacd3fe7f3d07e13597f29c13abf4d15c3b6dc2291cbbdd \ - --hash=sha256:442596f01913656d579309edcd179a2a2f9977d9a14ff41d042475280fc7f34e \ - --hash=sha256:44e44973262dc3ae79e9063a1284a73e09d01b894b534a769732ccd46c28cc62 \ - --hash=sha256:54139e0eb219f52f60656d163cbe67c31ede51d13236c950145473504fa208cb \ - --hash=sha256:5474dad8c86ee9ba9bb776f4b99ef2d41b3b8f4e0d199d4f7304728ed34d0300 \ - --hash=sha256:54c6a63e9d81efe64bfb7bcb0ec64332a87d0b87575f6009c8ba67ea6374770b \ - --hash=sha256:624884b572dff8ca8f60fab591413f077471de64e376b17d291b19f56504b2bb \ - --hash=sha256:6326ab99b52fafdcdeccf602d6286191a79fe2fda0ae90573c5814cd2b0bc1b8 \ - --hash=sha256:652e92fc409e278abdd61e9505649e3938f6d04ce7ef1953f2ec598a50e7c195 \ - --hash=sha256:6c1de77ded79fef664d5098a66810d4d27ca0224e9051906e634b3f7ead134c2 \ - --hash=sha256:76368c788ccb4f4782cf9c842b316140142b4cbf22ff8db82724e82fe1205dce \ - --hash=sha256:7a894c51fd8c4e834f00ac742abad73fc485df1062f1b875661a3c1e1fb1c2f6 \ - --hash=sha256:7dc90da0081f7e1da49ec4e398ede6a8e9cc4f5ebe5f9e06b443ed889ee9aaa2 \ - --hash=sha256:848c6b5cad9898e4b9ef251b6f934fa34630371f2e916261070a4eb9092ffd33 \ - --hash=sha256:899da829b362ade41e1e7eccad2cf274035e1cb36ba73034946fccd4afd8606b \ - --hash=sha256:8ab81ccd753859ab89e67199b9da62c543850f819993761c1e94a75a814ed667 \ - --hash=sha256:8fb49a0ba4d8f41198ae2d52118b050fd34dace4b8f3fb0ee34e23eb4ae775b1 \ - --hash=sha256:9156ca1f79fc4acc226696e95bfcc2b486f165a6a59ebe22b2c1f82ab190384a \ - --hash=sha256:9523f8b46485db6939bd069b28b642fec86c30909cea90ef550373787f79530e \ - --hash=sha256:a0756a179afa766ad7cb6f036de622e8a8f16ffdd55aa31f296c870b5679d745 \ - --hash=sha256:a0cdef204199278f5c461a0bed6ed2e052998276e6d8ab2963d5b5c39a0500bc \ - --hash=sha256:ab83adc099ec62e044b1fbb3a05499fa1e99f6d53a1dde102b2d85eff66ed324 \ - --hash=sha256:b34fa5e3b5d6dc7e0a4243fa0f81367027cb6f4a7215a17852979634b5544ee0 \ - --hash=sha256:b47c551c6724960479cefd7353656498b86e7232429e3a41ab83be4da1b109e8 \ - --hash=sha256:c4cd94dfefbefec3f8b544f61286584292d740e6e9d4677769bc76b8f41deb02 \ - --hash=sha256:c4f982715e65036c34897eb598d64aef15150c447be2cfc6643ec7a11af06574 \ - --hash=sha256:d8f699a709120b220dfe173f79c73cb2a2cab2c0b88dd59d7b49407d032b8ebd \ - --hash=sha256:dd94ce596bda40a9618324547cfaaf6650b1a24f5390350142499aa4e34e53d1 \ - --hash=sha256:de844aaa4815b78f6023832590d77da0e3b6805c644c33ce94a1e449f16d6ab5 \ - --hash=sha256:e5f0642cdf4636198a4990de7a71b693d824c56a757862230454629cf62e323d \ - --hash=sha256:f07fa2f15dabe91259828ce7d71b5ca9e2eb7c8c26baa822c825ce43552f4883 \ - --hash=sha256:f15976718c004466406342789f31b6673776360f3b1e3c575f25302d7e789575 \ - --hash=sha256:f358ea9e47eb3c2d6eba121ab512dfff38a88db719c38d1e67349af210bc7529 \ - --hash=sha256:f505264735ee074250a9c78247ee8618292091d9d1fcc023290e9ac67e8f1afa \ - --hash=sha256:f5ebbf9fbdabed208d4ecd2e1dfd2c0741af2f876e7ae522c2537d404ca895c3 \ - --hash=sha256:f6b26e6c3b98adb648243670fddc8cab6ae17473f9dc58c51574af3e64d61211 \ - --hash=sha256:f8e93a01a35be08d31ae33021e5268f157a2d60ebd643cfc15de6ab8e4722eb1 \ - --hash=sha256:fe76d75b345dc045acdbc006adcb197cc680754afd6c259de60d358d60c93736 \ - --hash=sha256:ffbd6faeb190aaf2b5e9024bac9622d2ee549b7ec89ef3a9373fa35313d44e0e - # via -r requirements.txt -packaging==25.0 \ - --hash=sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 \ - --hash=sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f - # via pytest -pluggy==1.6.0 \ - --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ - --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 - # via pytest -pytest==8.3.2 \ - --hash=sha256:4ba08f9ae7dcf84ded419494d229b48d0903ea6407b030eaec46df5e6a73bba5 \ - --hash=sha256:c132345d12ce551242c87269de812483f5bcc87cdbb4722e48487ba194f9fdce - # via -r requirements.txt diff --git a/scripts/run_pipeline.sh b/scripts/run_pipeline.sh index bc6e3f2..254a676 100644 --- a/scripts/run_pipeline.sh +++ b/scripts/run_pipeline.sh @@ -1,9 +1,12 @@ -#Order of running pipeline: data_cleaning --> cpp_cli_tool --> report_generator (?) -#Called by running bazel run //scripts:run_pipeline data.csv (arg1) +# TO RUN: ./scripts/run_pipeline.sh tests/python/example1.json output/test + +# example: ./scripts/run_pipeline.sh data_processing_pipeline/tests/python/example3.csv output/tests + +# Order of running pipeline cpp_cli_tool --> data_cleaner --> report_generator set -e -INPUT_FILE="$1" #$1 is the first argument, in this scenario the filepath +INPUT_FILE="$1" OUTPUT_DIR="$2" if [-z $INPUT_FILE]; then @@ -11,14 +14,8 @@ if [-z $INPUT_FILE]; then exit 1 fi -#Run CLI tool (?) - echo "====================RUNNING-PIPELINE-ON-$INPUT_FILE====================" -#bazel run //python_tools/data_cleaner:cleaner $INPUT_FILE -#bazel run //cpp_cli_tool:cli $INPUT_FILE -#RUN REPORT GENERATOR HERE EVENTUALLY - echo "Fetching directories..." WORKSPACE_DIR="$(bazel info workspace)/" @@ -28,38 +25,22 @@ BIN_DIR="$(bazel info bazel-bin)" CLI_TARGET="//apps/cpp_cli_tool:cli" PY_TARGET="//python_tools/data_cleaner:cleaner" -#CLI_BIN="$BIN_DIR/apps/cpp_cli_tool/cli" #Path to the cli tools BUILD file -#PY_BIN="$BIN_DIR/python_tools/data_cleaner/cleaner" - echo "Fetched directories..." bazel build $CLI_TARGET $PY_TARGET echo "Running C++ CLI tool on $INPUT_FILE..." -#Call using bazel instead of direct path to bin -bazel run $CLI_TARGET -- $WORKSPACE_DIR$INPUT_FILE -#"$CLI_BIN" "$INPUT_FILE" +bazel run $CLI_TARGET -- $WORKSPACE_DIR$INPUT_FILE echo "Running Python data cleaner tool on $INPUT_FILE..." + bazel run $PY_TARGET -- $WORKSPACE_DIR$INPUT_FILE "--output-dir=$OUTPUT_DIR" echo "Generating report of $INPUT_FILE..." -#Build report generator with output dir of cleaner in build file - -#Run report generator with output directory as param - echo "Pipeline finished, check $OUTPUT_DIR for results" bazel run //python_tools/report_generator:generator $OUTPUT_DIR -echo "====================PIPELINE-SUCCESSFUL====================" - - -chmod +x scripts/run_pipeline.sh - -#TO RUN: - -# bash scripts/run_pipeline.sh -# example: bash scripts/run_pipeline.sh data_processing_pipeline/tests/python/example3.csv output/tests \ No newline at end of file +echo "====================PIPELINE-SUCCESSFUL====================" \ No newline at end of file diff --git a/tests/python/test_cleaner.py b/tests/python/test_cleaner.py index b40576d..0af69ca 100644 --- a/tests/python/test_cleaner.py +++ b/tests/python/test_cleaner.py @@ -2,8 +2,8 @@ def test_handleNaNsCSV(): - cleaner = DataCleaner("tests/python") #Hmm this now needs to be passed an output_dir for writing the cleaned file to - cleaner.readFile("data_processing_pipeline/tests/python/example1.csv") + cleaner = DataCleaner("tests/python") + cleaner.readFile("tests/python/example1.csv") cleaner.handleNaNs() cleaner.writeFile() @@ -15,7 +15,7 @@ def test_handleNaNsCSV(): def test_normalizeTextCSV(): cleaner = DataCleaner("tests/python") - cleaner.readFile("data_processing_pipeline/tests/python/example2.csv") + cleaner.readFile("tests/python/example2.csv") cleaner.normalizeText() cleaner.writeFile() @@ -31,7 +31,7 @@ def test_normalizeTextCSV(): def test_handleNaNsJSON(): cleaner = DataCleaner("tests/python") - cleaner.readFile("data_processing_pipeline/tests/python/example1.json") + cleaner.readFile("tests/python/example1.json") cleaner.handleNaNs() cleaner.writeFile() @@ -45,7 +45,7 @@ def test_handleNaNsJSON(): def test_normalizeTextJSON(): cleaner = DataCleaner("tests/python") - cleaner.readFile("data_processing_pipeline/tests/python/example2.json") + cleaner.readFile("tests/python/example2.json") cleaner.normalizeText() cleaner.writeFile() @@ -60,7 +60,7 @@ def test_normalizeTextJSON(): def test_completeCleaningCSV(): cleaner = DataCleaner("tests/python") - cleaner.readFileAndClean("data_processing_pipeline/tests/python/example3.csv") + cleaner.readFileAndClean("tests/python/example3.csv") assert cleaner.filecontent == [ {'id': '1', 'name': 'alice', 'age': '30', 'score': '88'}, @@ -71,7 +71,7 @@ def test_completeCleaningCSV(): def test_completeCleaningJSON(): cleaner = DataCleaner("tests/python") - cleaner.readFileAndClean("data_processing_pipeline/tests/python/example3.json") + cleaner.readFileAndClean("tests/python/example3.json") assert cleaner.filecontent == [ {'id': 1, 'name': 'alice', 'age': 30, 'score': 88}, From 6ef2b039d0a4c02618909ff84deef9c431e8025f Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Thu, 16 Oct 2025 16:10:21 +0200 Subject: [PATCH 11/18] Fixes for ci (#20) --- python_tools/data_cleaner/cleaner.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python_tools/data_cleaner/cleaner.py b/python_tools/data_cleaner/cleaner.py index 1c64532..3757142 100644 --- a/python_tools/data_cleaner/cleaner.py +++ b/python_tools/data_cleaner/cleaner.py @@ -40,10 +40,7 @@ def detect_filetype(self, filepath): :param str filepath: filepath to the file which should be read. """ def readFile(self, filepath): - workspace = os.environ.get("BUILD_WORKSPACE_DIRECTORY", "./") - filepath = os.path.join(workspace, filepath) - - data_location = self.r.Rlocation(filepath) + data_location = self.r.Rlocation(f"_main/{filepath}") self.filename = filepath self.filetype = self.detect_filetype(filepath) From 196633ae6b057fb4d01ead564117eec72e6dddad Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Fri, 17 Oct 2025 08:42:04 +0200 Subject: [PATCH 12/18] Permission fix for ci (#20) --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b604da3..ece4d2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,5 +37,9 @@ jobs: with: name: test-logs path: bazel-testlogs/tests/**/test.log + + - name: make pipeline script executable + run: chmod 755 scripts/run_pipeline.sh + - name: run pipeline run: ./scripts/run_pipeline.sh tests/python/example1.json output/test \ No newline at end of file From fdf33c91b2c43c4a20f18ca316403b82c5b3c033 Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Fri, 17 Oct 2025 09:06:41 +0200 Subject: [PATCH 13/18] Commit to fix ci filepath normalization in data_cleaner (#20) --- python_tools/data_cleaner/cleaner.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/python_tools/data_cleaner/cleaner.py b/python_tools/data_cleaner/cleaner.py index 3757142..8b7f826 100644 --- a/python_tools/data_cleaner/cleaner.py +++ b/python_tools/data_cleaner/cleaner.py @@ -39,8 +39,8 @@ def detect_filetype(self, filepath): Method to read content of file depending on the filetype. :param str filepath: filepath to the file which should be read. """ - def readFile(self, filepath): - data_location = self.r.Rlocation(f"_main/{filepath}") + def readFile(self, filepath): + data_location = self.r.Rlocation(os.path.normpath(filepath)) self.filename = filepath self.filetype = self.detect_filetype(filepath) @@ -65,18 +65,17 @@ def readFile(self, filepath): def writeFile(self): workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', "./") - basename, ext = os.path.splitext(os.path.basename(self.filename)) # -> ('tests/python/example1', '.csv') - cleaned_filename = f"{basename}_cleaned{ext}" # -> 'tests/python/example1_cleaned.csv' + basename, ext = os.path.splitext(os.path.basename(self.filename)) + cleaned_filename = f"{basename}_cleaned{ext}" if self.output_dir is None: print("No output directory passed \n") if self.output_dir: - #If output_dir is absolute, use it as is, else, join with workspace - output_dir = self.output_dir if os.path.isabs(self.output_dir) else os.path.join(workspace, self.output_dir) # This creates a output like: cwd/output/cleaned/ + output_dir = self.output_dir if os.path.isabs(self.output_dir) else os.path.join(workspace, self.output_dir) - os.makedirs(output_dir, exist_ok=True) #Makes dir as specified above - output_path = os.path.join(output_dir, cleaned_filename) # -> '/output/cleaned/tests/python/example1_cleaned.csv, Hmm still results in -> /output/cleaned/tests/python/_cleaned + os.makedirs(output_dir, exist_ok=True) + output_path = os.path.join(output_dir, cleaned_filename) @@ -140,7 +139,7 @@ def main(): parser.add_argument("--read-file-only", action="store_true", help="Only read the file and not clean") parser.add_argument("--output-dir", default=None, help="Optional directory to write cleaned ouput files to (default: same directory as input)") args = parser.parse_args() - cleaner = DataCleaner(output_dir=args.output_dir) # Pass the output_dir here + cleaner = DataCleaner(output_dir=args.output_dir) if args.read_file_only: cleaner.readFile(filepath=args.filepath) From a57a51d462453f7c445a0f30f76396ce430e1fac Mon Sep 17 00:00:00 2001 From: SamuelPalmhager Date: Mon, 20 Oct 2025 09:19:16 +0200 Subject: [PATCH 14/18] Commit for fixing ci (#20) --- python_tools/data_cleaner/cleaner.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python_tools/data_cleaner/cleaner.py b/python_tools/data_cleaner/cleaner.py index 8b7f826..74d6fef 100644 --- a/python_tools/data_cleaner/cleaner.py +++ b/python_tools/data_cleaner/cleaner.py @@ -40,7 +40,13 @@ def detect_filetype(self, filepath): :param str filepath: filepath to the file which should be read. """ def readFile(self, filepath): - data_location = self.r.Rlocation(os.path.normpath(filepath)) + workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', None) + + + filepath = os.path.normpath(filepath) + filepath = f"{workspace}/{filepath}" if workspace and not os.path.isabs(filepath) else filepath + + data_location = self.r.Rlocation(filepath) self.filename = filepath self.filetype = self.detect_filetype(filepath) From 1079a58a4a147993dce9d754f4da2092f15c2a8a Mon Sep 17 00:00:00 2001 From: RaphaelRevivor Date: Mon, 20 Oct 2025 18:13:55 +0200 Subject: [PATCH 15/18] Fixed CI, part 2 (#20) --- python_tools/data_cleaner/cleaner.py | 3 +-- tests/python/BUILD | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/python_tools/data_cleaner/cleaner.py b/python_tools/data_cleaner/cleaner.py index 74d6fef..49ddc4d 100644 --- a/python_tools/data_cleaner/cleaner.py +++ b/python_tools/data_cleaner/cleaner.py @@ -40,8 +40,7 @@ def detect_filetype(self, filepath): :param str filepath: filepath to the file which should be read. """ def readFile(self, filepath): - workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', None) - + workspace = os.environ.get('TEST_WORKSPACE', None) filepath = os.path.normpath(filepath) filepath = f"{workspace}/{filepath}" if workspace and not os.path.isabs(filepath) else filepath diff --git a/tests/python/BUILD b/tests/python/BUILD index 8a72cfe..7d2af3b 100644 --- a/tests/python/BUILD +++ b/tests/python/BUILD @@ -11,7 +11,7 @@ pytest_test( "//tests/python/pytest:pytest_wrapper", ], imports = ["."], - data = glob(["*.csv", "*.json"]) + data = [":testdata"], ) filegroup( From 96f7e953ac784ee66ec1047b435fdf3bf53328f0 Mon Sep 17 00:00:00 2001 From: RaphaelRevivor Date: Fri, 24 Oct 2025 13:55:48 +0200 Subject: [PATCH 16/18] Delete unecessary filegroup (#20) --- output/test/BUILD | 6 ------ scripts/run_pipeline.sh | 2 -- 2 files changed, 8 deletions(-) diff --git a/output/test/BUILD b/output/test/BUILD index 6149859..2963406 100644 --- a/output/test/BUILD +++ b/output/test/BUILD @@ -1,7 +1 @@ -filegroup( - name="output", - data = glob(["*.csv", "*.json"], allow_empty=True), - visibility = ["//:__subpackages__"], -) - exports_files(glob(allow_empty=True, include=["*.csv", "*.json"])) \ No newline at end of file diff --git a/scripts/run_pipeline.sh b/scripts/run_pipeline.sh index 254a676..1cfaa55 100644 --- a/scripts/run_pipeline.sh +++ b/scripts/run_pipeline.sh @@ -1,7 +1,5 @@ # TO RUN: ./scripts/run_pipeline.sh tests/python/example1.json output/test -# example: ./scripts/run_pipeline.sh data_processing_pipeline/tests/python/example3.csv output/tests - # Order of running pipeline cpp_cli_tool --> data_cleaner --> report_generator set -e From b0a7397164e31169e0d35a0f729de1c5630355e6 Mon Sep 17 00:00:00 2001 From: RaphaelRevivor Date: Fri, 24 Oct 2025 16:00:08 +0200 Subject: [PATCH 17/18] Resolve CI issues and update ci yml file (#20) --- .github/workflows/ci.yml | 9 +++++++-- libs/math/BUILD | 14 +++++++++++++- python_tools/report_generator/BUILD | 15 ++++++++++++++- python_tools/report_generator/generator.py | 8 +++----- scripts/run_pipeline.sh | 2 +- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 815dd2d..dceaf0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,11 @@ jobs: - name: make pipeline script executable run: chmod 755 scripts/run_pipeline.sh - - name: run pipeline - run: ./scripts/run_pipeline.sh tests/python/example1.json output/test \ No newline at end of file + run: ./scripts/run_pipeline.sh tests/python/example1.json output/test + - name: upload pipeline results + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: pipeline-results + path: ./output/test/*.{json,html} \ No newline at end of file diff --git a/libs/math/BUILD b/libs/math/BUILD index e083e3d..99523ec 100644 --- a/libs/math/BUILD +++ b/libs/math/BUILD @@ -28,7 +28,7 @@ cc_binary( # usage: bazel build //libs/math:generate_stats_file gen_stats( - name = "generate_stats_pipeline", + name = "generate_stats_file_pipeline", # take the cc_binary as exec tool tool = ":generate_stats", data = [ @@ -39,6 +39,18 @@ gen_stats( ] ) +gen_stats( + name = "generate_stats_file", + # take the cc_binary as exec tool + tool = ":generate_stats", + data = [ + "//examples:example.csv", + ], + visibility = [ + "//:__subpackages__", + ] +) + gen_stats( name = "generate_stats_file_empty", # take the cc_binary as exec tool diff --git a/python_tools/report_generator/BUILD b/python_tools/report_generator/BUILD index 2eb12a2..89c5432 100644 --- a/python_tools/report_generator/BUILD +++ b/python_tools/report_generator/BUILD @@ -1,10 +1,23 @@ +py_binary( + name = "generator_pipeline", + main = "generator.py", + srcs = ["generator.py"], + # set data as custom generation rule as it produces an output file + data = [ + "@rules_python//python/runfiles", + "//libs/math:generate_stats_file_pipeline", + ], + visibility = ["//:__subpackages__"], +) + py_binary( name = "generator", + main = "generator.py", srcs = ["generator.py"], # set data as custom generation rule as it produces an output file data = [ "@rules_python//python/runfiles", - "//libs/math:generate_stats_pipeline", + "//libs/math:generate_stats_file", ], visibility = ["//:__subpackages__"], ) diff --git a/python_tools/report_generator/generator.py b/python_tools/report_generator/generator.py index c54fb93..d7b46a5 100644 --- a/python_tools/report_generator/generator.py +++ b/python_tools/report_generator/generator.py @@ -111,13 +111,11 @@ def generate_HTML_file(self): def main(): argparser = argparse.ArgumentParser(description="Parses arguments") - argparser.add_argument("output_dir", nargs="?", default=None, help="Path to output directory") + argparser.add_argument("--output_dir", nargs="?", default="", help="Path to output directory") + argparser.add_argument("--input_file", nargs="?", default="output.json", help="Path to input file") args = argparser.parse_args() - if args.output_dir is None: - rg = ReportGenerator() - else: - rg = ReportGenerator(args.output_dir) + rg = ReportGenerator(args.output_dir, args.input_file) rg.read_JSON_file() rg.handle_empty_values() diff --git a/scripts/run_pipeline.sh b/scripts/run_pipeline.sh index 1cfaa55..3a04584 100644 --- a/scripts/run_pipeline.sh +++ b/scripts/run_pipeline.sh @@ -39,6 +39,6 @@ echo "Generating report of $INPUT_FILE..." echo "Pipeline finished, check $OUTPUT_DIR for results" -bazel run //python_tools/report_generator:generator $OUTPUT_DIR +bazel run //python_tools/report_generator:generator_pipeline -- --output_dir $OUTPUT_DIR --input_file output_generate_stats_file_pipeline.json echo "====================PIPELINE-SUCCESSFUL====================" \ No newline at end of file From bc3fbb573c50957716ef299c6685c2d5ad015510 Mon Sep 17 00:00:00 2001 From: RaphaelRevivor Date: Fri, 24 Oct 2025 16:09:40 +0200 Subject: [PATCH 18/18] Fix CI issues, hopfully final (#20) --- .github/workflows/ci.yml | 4 +++- scripts/run_pipeline.sh | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dceaf0e..69a177f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,4 +49,6 @@ jobs: uses: actions/upload-artifact@v4 with: name: pipeline-results - path: ./output/test/*.{json,html} \ No newline at end of file + path: | + ./output/test/*.json + ./output/test/*.html \ No newline at end of file diff --git a/scripts/run_pipeline.sh b/scripts/run_pipeline.sh index 3a04584..db61deb 100644 --- a/scripts/run_pipeline.sh +++ b/scripts/run_pipeline.sh @@ -37,8 +37,8 @@ bazel run $PY_TARGET -- $WORKSPACE_DIR$INPUT_FILE "--output-dir=$OUTPUT_DIR" echo "Generating report of $INPUT_FILE..." -echo "Pipeline finished, check $OUTPUT_DIR for results" - bazel run //python_tools/report_generator:generator_pipeline -- --output_dir $OUTPUT_DIR --input_file output_generate_stats_file_pipeline.json +echo "Pipeline finished, check $(realpath $OUTPUT_DIR) for results" + echo "====================PIPELINE-SUCCESSFUL====================" \ No newline at end of file