Refactor trainData type handling

.github/workflows/unittests.yml

@@ -12,8 +12,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-        with:
-          submodules: recursive
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
@@ -27,12 +25,8 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install torch
-
-      - name: Build and install the package
-        run: python setup.py install
+          pip install torch pytest
+          pip install -e .
 
       - name: Run unit tests
-        run: |
-          cd test
-          python runtests.py
+        run: pytest

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 3.4)
 project(djcdata)
 
 include(CMakePrintHelpers)
@@ -10,10 +10,13 @@ include_directories(${SOURCE_DIR}/../interface)
 
 file(GLOB_RECURSE SOURCES "${SOURCE_DIR}/*.cpp" "${SOURCE_DIR}/*.c")
 
-add_subdirectory(lib/pybind11)
-pybind11_add_module(compiled ${SOURCES} "${SOURCE_DIR}/bind/bindings.cpp")
+execute_process(
+    COMMAND ${PYTHON_EXECUTABLE} -m pybind11 --cmakedir
+    OUTPUT_VARIABLE pybind11_DIR
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+find_package(pybind11 CONFIG REQUIRED PATHS ${pybind11_DIR})
+pybind11_add_module(compiled ${SOURCES})
 
-#shared lib lib
-include_directories(lib/pybind11/include)
+# shared lib lib
 # add python etc to be added, also copying includes etc
 # add_library(djcdata SHARED ${SOURCES})

MANIFEST.in

@@ -1,4 +1,3 @@
 include CMakeLists.txt
-recursive-include lib *
 recursive-include src/djcdata/interface *
-recursive-include src/djcdata/src *
+recursive-include src/djcdata/src *

pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+requires = ["setuptools>=61", "wheel", "pybind11"]
+build-backend = "setuptools.build_meta"
+
+[tool.pytest.ini_options]
+testpaths = ["test"]

setup.py

@@ -88,6 +88,7 @@ def build_extension(self, ext):
     ext_modules=[CMakeExtension('djcdata/compiled')],
     cmdclass=dict(build_ext=CMakeBuild),
     zip_safe=False,
-    install_requires=['numpy'],
+    install_requires=['numpy', 'pybind11'],
+    setup_requires=['pybind11'],
     scripts=bins
 )

src/djcdata/interface/trainData.h

@@ -17,6 +17,7 @@
 #include "IO.h"
 
 #include <iostream>
+#include <memory>
 
 #include "pybind11/pybind11.h"
 #include "pybind11/numpy.h"
@@ -31,6 +32,12 @@ namespace djc{
 class typeContainer{
 public:
 
+    typeContainer() = default;
+    typeContainer(const typeContainer& rhs);
+    typeContainer& operator=(const typeContainer& rhs);
+    typeContainer(typeContainer&& rhs) = default;
+    typeContainer& operator=(typeContainer&& rhs) = default;
+
     void push_back(simpleArrayBase& a);
     void move_back(simpleArrayBase& a);
 
@@ -39,7 +46,6 @@ class typeContainer{
         return !(*this==rhs);
     }
 
-
     simpleArrayBase& at(size_t idx);
     const simpleArrayBase& at(size_t idx)const;
 
@@ -52,8 +58,7 @@ class typeContainer{
 
     void clear();
 
-    size_t size()const{return sorting_.size();}
-
+    size_t size()const{return arrays_.size();}
 
     void writeToFile(FILE *&) const;
     inline void readFromFile(FILE *&f){
@@ -67,12 +72,7 @@ class typeContainer{
 private:
     void readFromFile_priv(FILE *& f, bool justmetadata);
 
-    std::vector<simpleArray_float32> farrs_;
-    std::vector<simpleArray_int32> iarrs_;
-
-    enum typesorting{isfloat,isint};
-    std::vector<std::pair<typesorting,size_t> > sorting_;
-
+    std::vector<std::unique_ptr<simpleArrayBase> > arrays_;
 };
 
 

src/djcdata/src/bind/bindings.cpp

@@ -9,11 +9,16 @@
 
 namespace py = pybind11;
 
+template<class M>
+void makeBaseArr(M& m){
+    using namespace djc;
+    py::class_<simpleArrayBase>(m, "simpleArrayBase");
+}
 
 template<class T, class M>
 void makeArr(M& m, std::string name){
     using namespace djc;
-    py::class_<simpleArray<T> >(m, name.data())
+    py::class_<simpleArray<T>, simpleArrayBase >(m, name.data())
             .def(py::init())
 
             .def(py::self == py::self)
@@ -103,14 +108,9 @@ void makeTD(M & m, std::string name){
         //        }
         //    ))
 
-        .def("storeFeatureArray", static_cast<int (trainData::*)(simpleArray_float32 &)>(&trainData::storeFeatureArray))
-        .def("storeFeatureArray", static_cast<int (trainData::*)(simpleArray_int32 &)>(&trainData::storeFeatureArray))
-
-        .def("storeTruthArray", static_cast<int (trainData::*)(simpleArray_float32 &)>(&trainData::storeTruthArray))
-        .def("storeTruthArray", static_cast<int (trainData::*)(simpleArray_int32 &)>(&trainData::storeTruthArray))
-
-        .def("storeWeightArray", static_cast<int (trainData::*)(simpleArray_float32 &)>(&trainData::storeWeightArray))
-        .def("storeWeightArray", static_cast<int (trainData::*)(simpleArray_int32 &)>(&trainData::storeWeightArray))
+        .def("storeFeatureArray", static_cast<int (trainData::*)(simpleArrayBase&)>(&trainData::storeFeatureArray))
+        .def("storeTruthArray", static_cast<int (trainData::*)(simpleArrayBase&)>(&trainData::storeTruthArray))
+        .def("storeWeightArray", static_cast<int (trainData::*)(simpleArrayBase&)>(&trainData::storeWeightArray))
 
 
         .def("nFeatureArrays", &trainData::nFeatureArrays)
@@ -194,6 +194,7 @@ void makeTDG(M & m, std::string name){
 
 //warp it up
 PYBIND11_MODULE(compiled, m) {
+    makeBaseArr(m);
     makeArr<float>(m,"simpleArrayF");
     makeArr<int32_t>(m,"simpleArrayI");
 

src/djcdata/src/trainData.cpp

@@ -5,75 +5,88 @@ namespace py=pybind11;
 
 namespace djc{
 
-/*
- *
-    std::vector<simpleArray_float32> farrs_;
-    std::vector<simpleArray_int32> iarrs_;
+// typeContainer ----------------------------------------------------
+
+typeContainer::typeContainer(const typeContainer& rhs){
+    for(const auto& p: rhs.arrays_){
+        if(p->dtype() == simpleArrayBase::float32)
+            arrays_.emplace_back(new simpleArray_float32(
+                    dynamic_cast<const simpleArray_float32&>(*p)));
+        else if(p->dtype() == simpleArrayBase::int32)
+            arrays_.emplace_back(new simpleArray_int32(
+                    dynamic_cast<const simpleArray_int32&>(*p)));
+    }
+}
 
-    enum typesorting{isfloat,isint};
-    std::vector<std::pair<typesorting,size_t> > sorting_;
- */
+typeContainer& typeContainer::operator=(const typeContainer& rhs){
+    if(this!=&rhs){
+        arrays_.clear();
+        for(const auto& p: rhs.arrays_){
+            if(p->dtype() == simpleArrayBase::float32)
+                arrays_.emplace_back(new simpleArray_float32(
+                        dynamic_cast<const simpleArray_float32&>(*p)));
+            else if(p->dtype() == simpleArrayBase::int32)
+                arrays_.emplace_back(new simpleArray_int32(
+                        dynamic_cast<const simpleArray_int32&>(*p)));
+        }
+    }
+    return *this;
+}
 
 void typeContainer::push_back(simpleArrayBase& a){
-    if(a.dtype() == simpleArrayBase::float32){
-        farrs_.push_back(dynamic_cast<simpleArray_float32&>(a));
-        sorting_.push_back({isfloat,farrs_.size()-1});
-    }
-    else {//if(a.dtype() == simpleArrayBase::int32){
-        iarrs_.push_back(dynamic_cast<simpleArray_int32&>(a));
-        sorting_.push_back({isint,iarrs_.size()-1});
-    }
+    if(a.dtype() == simpleArrayBase::float32)
+        arrays_.emplace_back(new simpleArray_float32(
+                dynamic_cast<simpleArray_float32&>(a)));
+    else
+        arrays_.emplace_back(new simpleArray_int32(
+                dynamic_cast<simpleArray_int32&>(a)));
 }
+
 void typeContainer::move_back(simpleArrayBase& a){
-    if(a.dtype() == simpleArrayBase::float32){
-        farrs_.push_back(std::move(dynamic_cast<simpleArray_float32&>(a)));
-        sorting_.push_back({isfloat,farrs_.size()-1});
-    }
-    else {//if(a.dtype() == simpleArrayBase::int32){
-        iarrs_.push_back(std::move(dynamic_cast<simpleArray_int32&>(a)));
-        sorting_.push_back({isint,iarrs_.size()-1});
-    }
+    if(a.dtype() == simpleArrayBase::float32)
+        arrays_.emplace_back(new simpleArray_float32(
+                std::move(dynamic_cast<simpleArray_float32&>(a))));
+    else
+        arrays_.emplace_back(new simpleArray_int32(
+                std::move(dynamic_cast<simpleArray_int32&>(a))));
 }
+
 bool typeContainer::operator==(const typeContainer& rhs)const{
     if(size() != rhs.size())
         return false;
-    if(farrs_.size() != rhs.farrs_.size())
-        return false;
-
-    if(sorting_ != rhs.sorting_)
-        return false;
-
-    for(size_t i=0;i<farrs_.size();i++){
-        if(farrs_.at(i) != rhs.farrs_.at(i))
+    for(size_t i=0;i<arrays_.size();i++){
+        auto& a = *arrays_.at(i);
+        const auto& b = *rhs.arrays_.at(i);
+        if(a.dtype() != b.dtype())
             return false;
-    }
-    for(size_t i=0;i<iarrs_.size();i++){
-        if(iarrs_.at(i) != rhs.iarrs_.at(i))
+        if(a.dtype() == simpleArrayBase::float32){
+            if(dynamic_cast<const simpleArray_float32&>(a) !=
+               dynamic_cast<const simpleArray_float32&>(b))
+                return false;
+        }
+        else if(a.dtype() == simpleArrayBase::int32){
+            if(dynamic_cast<const simpleArray_int32&>(a) !=
+               dynamic_cast<const simpleArray_int32&>(b))
+                return false;
+        }
+        else
             return false;
     }
     return true;
 }
-simpleArrayBase& typeContainer::at(size_t idx){
-    if(idx>=sorting_.size())
-        throw std::out_of_range("typeContainer::at: requested "+std::to_string(idx)+" of "+std::to_string(sorting_.size()));
-    auto s = sorting_.at(idx);
-    if(s.first == isfloat)
-        return farrs_.at(s.second);
-    else //if(s.first == isint)
-        return iarrs_.at(s.second);
 
+simpleArrayBase& typeContainer::at(size_t idx){
+    if(idx>=arrays_.size())
+        throw std::out_of_range("typeContainer::at: requested "+std::to_string(idx)+" of "+std::to_string(arrays_.size()));
+    return *arrays_.at(idx);
 }
+
 const simpleArrayBase& typeContainer::at(size_t idx)const{
-    if(idx>=sorting_.size())
-        throw std::out_of_range("typeContainer::at: requested "+std::to_string(idx)+" of "+std::to_string(sorting_.size()));
-    auto s = sorting_.at(idx);
-    if(s.first == isfloat)
-        return farrs_.at(s.second);
-    else //if(s.first == isint)
-        return iarrs_.at(s.second);
+    if(idx>=arrays_.size())
+        throw std::out_of_range("typeContainer::at: requested "+std::to_string(idx)+" of "+std::to_string(arrays_.size()));
+    return *arrays_.at(idx);
 }
 
-
 simpleArray_float32& typeContainer::at_asfloat32(size_t idx){
     if(at(idx).dtype() != simpleArrayBase::float32)
         throw std::runtime_error("typeContainer::at_asfloat32: is not float32");
@@ -98,23 +111,15 @@ const simpleArray_int32& typeContainer::at_asint32(size_t idx)const{
     return dynamic_cast<const simpleArray_int32&>(at(idx));
 }
 
-
 void typeContainer::clear(){
-    farrs_.clear();
-    iarrs_.clear();
-    sorting_.clear();
+    arrays_.clear();
 }
 
 void typeContainer::writeToFile(FILE *& ofile) const{
     size_t isize=size();
     io::writeToFile(&isize,ofile);
-    for(const auto& i: sorting_){
-        if(i.first == isfloat){
-            farrs_.at(i.second).addToFileP(ofile);
-        }
-        else {// if(i.first == isint){
-            iarrs_.at(i.second).addToFileP(ofile);
-        }
+    for(const auto& a : arrays_){
+        a->addToFileP(ofile);
     }
 }
 
@@ -123,18 +128,20 @@ void typeContainer::readFromFile_priv(FILE *& ifile, bool justmetadata){
     size_t isize = 0;
     io::readFromFile(&isize,ifile);
     for(size_t i=0;i<isize;i++){
-        simpleArray_float32 tmp;//type doesn't matter
+        simpleArray_float32 tmp; //type doesn't matter, just for util
         auto dtype = tmp.readDtypeTypeFromFileP(ifile);
         if(dtype == simpleArrayBase::float32){
-            simpleArray_float32 farr;
-            farr.readFromFileP(ifile,justmetadata);
-            move_back(farr);
+            auto arr = std::make_unique<simpleArray_float32>();
+            arr->readFromFileP(ifile,justmetadata);
+            arrays_.emplace_back(std::move(arr));
         }
-        else{ //if(dtype==simpleArrayBase::int32){
-            simpleArray_int32 iarr;
-            iarr.readFromFileP(ifile,justmetadata);
-            move_back(iarr);
+        else if(dtype == simpleArrayBase::int32){
+            auto arr = std::make_unique<simpleArray_int32>();
+            arr->readFromFileP(ifile,justmetadata);
+            arrays_.emplace_back(std::move(arr));
         }
+        else
+            throw std::runtime_error("typeContainer::readFromFile_priv: unsupported dtype");
     }
 }