Dev_cli_tool --> Main (#7, #20)

.github/workflows/ci.yml

@@ -39,3 +39,16 @@ jobs:
         with:
           name: test-logs
           path: bazel-testlogs/tests/**/test.log
+
+      - name: make pipeline script executable
+        run: chmod 755 scripts/run_pipeline.sh
+      - name: run pipeline
+        run: ./scripts/run_pipeline.sh tests/python/example1.json output/test
+      - name: upload pipeline results
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pipeline-results
+          path: |
+            ./output/test/*.json
+            ./output/test/*.html

apps/cpp_cli_tool/BUILD

@@ -1,3 +1,14 @@
-cc_library(
-
-)
+cc_binary(
+  name = "cli",
+  srcs = ["cli.cc"],
+  deps = [
+    "//libs/parser:parser_factory",
+    "//libs/math:stats",
+    "@bazel_tools//tools/cpp/runfiles",
+  ],
+  visibility  = ["//scripts:__subpackages__"],
+
+  data = ["//tests/python:testdata"],
+
+)
+

apps/cpp_cli_tool/cli.cc

@@ -0,0 +1,93 @@
+/**
+ * This should accept file path + type, use factory parser, run statistics, print summary.
+ * 
+ * DoD:
+ * bazel run //apps/cpp_cli_tool:cli -- data.csv works.
+ * Console output includes mean, median, variance, frequencies.
+ * 
+ */
+
+#include <iostream>
+#include <string>
+#include <iomanip>
+
+#include "tools/cpp/runfiles/runfiles.h"
+#include "libs/parser/parser_factory.h"
+#include "libs/math/stats.h"
+
+
+/**
+ * main function for CLI tool. Call by running bazel run //apps/cpp_cli_tool:cli --<filepath>
+ * @param argc argument count, @param argv an array of C strings (the arguments themselves)
+ * @return 0 on success, 1 on failure
+ */
+
+int main(int argc, char* argv[]){
+    if(argc < 2){
+        std::cout << "Usage: mytool <filepath>" << std::endl;
+        return 1;
+    }
+    std::string filename = argv[1];
+    std::string filetype = filename.substr(filename.find_last_of(".") + 1);
+
+    std::string err;
+    auto runfiles(bazel::tools::cpp::runfiles::Runfiles::Create(argv[0], &err));
+    if (runfiles == nullptr) {
+        std::cerr << "Error initializing runfiles: " << err << std::endl;
+        return 1;
+    }
+
+    std::string filepath = runfiles->Rlocation(filename); //<-- Will cause a seg fault if runfiles==nullptr :(
+
+
+    //Create parser w. parserfactory
+    static shared_ptr<Parser> parserPtr;
+    parserPtr = ParserFactory(filetype); //Evaluates to csv or json parser
+
+    parserPtr->parse(filepath); //Parses the file
+
+    Stats stats = Stats();
+
+    stats.readData(parserPtr->getEntries()); // Should pass in correct datatype 
+    stats.loadScores();
+    std::cout << "\n\n\n";
+
+    std::cout << std::setfill('=') << std::setw(25 + strlen("SCORES-STATISTICS")) << "SCORES-STATISTICS" << std::setw(25) << '\n';
+    std::cout << std::setfill(' ') << "Mean: " << stats.calcMean() << std::setw(strlen("Median: ") + 2);
+    std::cout << "Median: " << stats.calcMedian() << std::setw(strlen("Variance: " + 2)) << " Variance: " << stats.calcVariance();
+    std::cout << std::setw(strlen("Standard deviation: ") + 2) << " Standard deviation: " << stats.calcStandardDeviation();
+
+    auto data = parserPtr->getEntries();
+    std::vector<int> scores {};
+
+    for (size_t i = 0; i < data.size(); i++)
+    {
+        auto entry = parserPtr->getEntryById(std::to_string(i+1));
+        for (auto &&[k, v]: entry)
+        {
+            if(k == "score"){
+                scores.push_back(std::stoi(v));
+                break;
+            }
+        }
+    }
+
+    if(stats.calcMode().size() == scores.size()){
+        std::cout << std::setw(strlen("No identifiable mode") + 2) << "No identifiable mode";     
+    }else{
+        std::cout << std::setw(strlen("Mode: ") + 2) << "Mode: " << stats.calcMode()[0];
+    }
+    std::cout << "\n" << std::endl;
+
+    std::cout << std::setfill('=') << std::setw(25 + strlen("FREQUENCIES-OF-SCORES")) << "FREQUENCIES-OF-SCORES" << std::setw(25) << "\n";
+    for (auto &&[k,v] : stats.calcFrequencies())
+    {
+        std::cout << k << " appeared " << v << " time(s) ";
+    }
+    std::cout << std::endl;
+    std::cout << "\n\n";
+
+
+
+    return 0;
+}

libs/math/BUILD

@@ -27,6 +27,18 @@ cc_binary(
 )
 
 # usage: bazel build //libs/math:generate_stats_file
+gen_stats(
+  name = "generate_stats_file_pipeline",
+  # take the cc_binary as exec tool
+  tool = ":generate_stats",
+  data = [
+    "//output/test:example1_cleaned.json",
+  ],
+  visibility = [
+    "//:__subpackages__",
+  ]
+)
+
 gen_stats(
   name = "generate_stats_file",
   # take the cc_binary as exec tool

libs/math/generate_stats.cc

@@ -14,7 +14,6 @@ int main(int argc, char* argv[])
     }
 
     string filePath = argv[1];
-
     int totalSize = filePath.size();
     int suffixSize = 4;
 

libs/parser/BUILD

@@ -62,6 +62,6 @@ cc_library(
     ":json_parser",
     ":csv_parser",
   ],
-  visibility = ["//libs/math:__pkg__"],
   includes = ["."],
+  visibility = ["//:__subpackages__"],
 )

libs/parser/csv_parser.cc

@@ -29,6 +29,8 @@ void CsvParser::parse(const string& filename)
         stringstream ss(line);
         string header;
         while(getline(ss, header, ',')){
+            header.erase(remove(header.begin(), header.end(), '\r'), header.end());
+            header.erase(remove(header.begin(), header.end(), ' '), header.end());
             headers.push_back(header);
         }
     }
@@ -43,6 +45,9 @@ void CsvParser::parse(const string& filename)
 
         while(getline(ss, value, ',')){
             if(col < headers.size()){
+                value.erase(remove(value.begin(), value.end(), '\r'), value.end());
+                value.erase(remove(value.begin(), value.end(), ' '), value.end());
+
                 row[headers[col]] = value;
             }
             col++;

output/test/BUILD

@@ -0,0 +1 @@
+exports_files(glob(allow_empty=True, include=["*.csv", "*.json"]))

python_tools/data_cleaner/cleaner.py

@@ -9,14 +9,15 @@
 class DataCleaner: 
     """
     Class to read, clean and write the content of csv or json files.
-    :atribute str filename: Name of the file being processed.
-    :atribute str filetype: csv or json depending on filenames ending.
-    :atribute list filecontent: empty list representeing the content of the file being read.
-    :atribute runfiles r: Object which enables file reading in bazel environments.
-    """
-    def __init__(self):
-        self.filename = ""
-        self.filetype = ""
+    :attribute str filename: Name of the file being processed.
+    :attribute str filetype: csv or json depending on filenames ending.
+    :attribute list filecontent: empty list representeing the content of the file being read.
+    :attribute runfiles r: Object which enables file reading in bazel environments.
+    """
+    def __init__(self, output_dir=None):
+        self.filename = None
+        self.filetype = None
+        self.output_dir = output_dir
         self.filecontent = []
         self.r = runfiles.Create()
 
@@ -34,25 +35,19 @@ def detect_filetype(self, filepath):
         else: 
             raise ValueError("Unsupported filetype")
 
-    """
-    :Method to initialize the terminal arguments to call tests
-    """
-    def initArgs(self):
-        self.parser.add_argument("filepath", help = "Path to the input file")
-        self.parser.add_argument("--read-file-only", action="store_true", help="Reading a file without cleaning the data")
-
     """
     Method to read content of file depending on the filetype.
     :param str filepath: filepath to the file which should be read.
     """
-    def readFile(self, filepath): 
-        workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', 'data_processing_pipeline')
+    def readFile(self, filepath):
+        workspace = os.environ.get('TEST_WORKSPACE', None)
 
-        data_location = self.r.Rlocation(f"{workspace}/{filepath}")
-
-        self.filetype = self.detect_filetype(filepath) 
-        self.output = filepath.split(".")
-        self.output = f"{self.output[0]}_cleaned.{self.output[1]}" 
+        filepath = os.path.normpath(filepath)
+        filepath = f"{workspace}/{filepath}" if workspace and not os.path.isabs(filepath) else filepath
+
+        data_location = self.r.Rlocation(filepath)
+        self.filename = filepath
+        self.filetype = self.detect_filetype(filepath)
 
         self.filecontent.clear()
 
@@ -73,18 +68,30 @@ def readFile(self, filepath):
     Method to write the content of self.filecontent to a filepath.
     """
     def writeFile(self):
-        workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', 'data_processing_pipeline')
+        workspace = os.environ.get('BUILD_WORKSPACE_DIRECTORY', "./")
 
-        data_location = self.r.Rlocation(f"{workspace}/{self.output}")
+        basename, ext = os.path.splitext(os.path.basename(self.filename))
+        cleaned_filename = f"{basename}_cleaned{ext}"
+
+        if self.output_dir is None:
+            print("No output directory passed \n")
+
+        if self.output_dir:
+            output_dir = self.output_dir if os.path.isabs(self.output_dir) else os.path.join(workspace, self.output_dir)     
+
+        os.makedirs(output_dir, exist_ok=True)
+        output_path = os.path.join(output_dir, cleaned_filename)
+
+
 
         if(self.filetype == "csv"):
-            with(open(data_location, mode='w')) as file:
+            with(open(output_path, mode='w')) as file:
                 csv_writer = csv.DictWriter(file, fieldnames=self.filecontent[0].keys())
                 csv_writer.writeheader()
                 csv_writer.writerows(self.filecontent)
 
         elif(self.filetype == "json"):
-            with(open(data_location, mode='w')) as file:
+            with(open(output_path, mode='w')) as file:
                 json.dump(self.filecontent, file)
 
     """
@@ -135,13 +142,16 @@ def main():
     parser = argparse.ArgumentParser(description="Clean CSV/JSON files")
     parser.add_argument("filepath", help="Path to the input file")
     parser.add_argument("--read-file-only", action="store_true", help="Only read the file and not clean")
+    parser.add_argument("--output-dir", default=None, help="Optional directory to write cleaned ouput files to (default: same directory as input)")
     args = parser.parse_args()
-    cleaner = DataCleaner()
+    cleaner = DataCleaner(output_dir=args.output_dir) 
 
     if args.read_file_only:
         cleaner.readFile(filepath=args.filepath)
     else:
         cleaner.readFileAndClean(filepath=args.filepath)
+
+    return args
 
 if __name__ == '__main__':
-    main()
+    args = main()

python_tools/report_generator/BUILD

@@ -1,5 +1,18 @@
+py_binary(
+  name = "generator_pipeline",
+  main = "generator.py",
+  srcs = ["generator.py"],
+  # set data as custom generation rule as it produces an output file
+  data = [
+    "@rules_python//python/runfiles",
+    "//libs/math:generate_stats_file_pipeline",
+  ],
+  visibility = ["//:__subpackages__"],
+)
+
 py_binary(
   name = "generator",
+  main = "generator.py",
   srcs = ["generator.py"],
   # set data as custom generation rule as it produces an output file
   data = [

python_tools/report_generator/generator.py

@@ -111,13 +111,11 @@ def generate_HTML_file(self):
 
 def main():
     argparser = argparse.ArgumentParser(description="Parses arguments")
-    argparser.add_argument("output_dir", nargs="?", default=None, help="Path to output directory")
+    argparser.add_argument("--output_dir", nargs="?", default="", help="Path to output directory")
+    argparser.add_argument("--input_file", nargs="?", default="output.json", help="Path to input file")
     args = argparser.parse_args()
 
-    if args.output_dir is None:
-        rg = ReportGenerator()
-    else:
-        rg = ReportGenerator(args.output_dir)
+    rg = ReportGenerator(args.output_dir, args.input_file)
 
     rg.read_JSON_file()
     rg.handle_empty_values()

scripts/BUILD

@@ -0,0 +1,29 @@
+sh_library(
+    name = "run_pipeline_lib",
+    visibility = ["//visibility:public"],
+    #srcs = ["run_pipeline.sh"],
+    data = [
+        ":run_pipeline", #Sh_binary below
+    ],
+
+)
+
+sh_binary(
+    name = "run_pipeline",
+    srcs = ["run_pipeline.sh",],
+    #data = ["//tests/python:testdata",],
+    data = [
+        "//apps/cpp_cli_tool:cli",
+        "//python_tools/data_cleaner:cleaner",
+        "//tests/python:testdata",
+        ":bash_data"
+    ],
+    visibility = ["//visibility:__subpackages__"],
+    #deps = [":run_pipeline_lib"],
+)
+
+filegroup(
+    name = "bash_data",
+    srcs = glob(["*.csv", "*.json"]),
+    visibility = ["//:__subpackages__"],
+)