Minor changes and bugfixes

README.md

@@ -28,7 +28,10 @@ on Windows follow any installation instructions and install libmicrohttpd, curl
 2. gperftools: https://code.google.com/p/gperftools/
 
 Finally, call
-> bazel build //...
+> bazel build -c opt //...
+
+if glog and gflags are not found, try running the command below instead
+> bazel build --cxxopt -I/usr/local/include --linkopt -L/usr/local/lib -c opt //...
 
 To run tests, call
 > bazel test //...

WORKSPACE

@@ -1,8 +1,10 @@
-git_repository(
-  name = "org_pubref_rules_protobuf",
-  remote = "https://github.com/pubref/rules_protobuf",
-  tag = "v0.8.1",
-#  commit = "d9523f3d443b6a4f3fabc72051d84eb5474d7745"
+workspace(name = "com_ethz_srl_nice2predict")
+
+http_archive(
+    name = "org_pubref_rules_protobuf",
+    sha256 = "fb9852446b5ba688cd7178a60ff451623e4112d015c6adfe0e9a06c5d2dedc08",
+    strip_prefix = "rules_protobuf-0.8.1/",
+    url = "https://github.com/pubref/rules_protobuf/archive/v0.8.1.tar.gz",
 )
 
 load("@org_pubref_rules_protobuf//cpp:rules.bzl", "cpp_proto_repositories")
@@ -23,3 +25,19 @@ new_patched_http_archive(
   strip_prefix = "sparsehash-sparsehash-2.0.3/",
   patch_file = "//third_party:sparsehash.patch",
 )
+
+new_http_archive(
+    name = "com_github_open_source_parsers_jsoncpp",
+    build_file = "//third_party:BUILD.jsoncpp",
+    sha256 = "3671ba6051e0f30849942cc66d1798fdf0362d089343a83f704c09ee7156604f",
+    strip_prefix = "jsoncpp-1.8.3/",
+    url = "https://github.com/open-source-parsers/jsoncpp/archive/1.8.3.tar.gz",
+)
+
+new_http_archive(
+    name = "com_github_cinemast_libjson_rpc_cpp",
+    build_file = "//third_party:BUILD.jsonrpc",
+    sha256 = "888c10f4be145dfe99e007d5298c90764fb73b58effb2c6a3fc522a5b60a18c6",
+    strip_prefix = "libjson-rpc-cpp-1.0.0",
+    url = "https://github.com/cinemast/libjson-rpc-cpp/archive/v1.0.0.tar.gz",
+)

base/readerutil.h

@@ -43,7 +43,10 @@ class InputRecordReader {
 template <class ProtoClass>
 class FileInputRecordReader : public InputRecordReader<ProtoClass> {
  public:
-  explicit FileInputRecordReader(const std::string& filename) : reader(filename), has_prefetched(false) {
+  explicit FileInputRecordReader(const std::string& filename, const int64 max_records=-1) :
+      reader(filename),
+      has_prefetched(false),
+      max_records_(max_records){
   }
   virtual ~FileInputRecordReader() override {
     reader.Close();
@@ -57,12 +60,16 @@ class FileInputRecordReader : public InputRecordReader<ProtoClass> {
     }
     *proto = std::move(prefetched_proto);
     has_prefetched = false;
-    return true;
+    if (max_records_ != 0) {
+      max_records_--;
+      return true;
+    }
+    return false;
   }
 
   virtual bool ReachedEnd() override {
     std::lock_guard<std::mutex> lock(reader_mutex);
-    return !has_prefetched && !PrefetchProto();
+    return (!has_prefetched && !PrefetchProto()) || max_records_ == 0;
   }
 
  private:
@@ -79,13 +86,16 @@ class FileInputRecordReader : public InputRecordReader<ProtoClass> {
   ProtoClass prefetched_proto;
   bool has_prefetched;
   std::mutex reader_mutex;
+  int64 max_records_;
 };
 
 
 template <>
 class FileInputRecordReader<std::string> : public InputRecordReader<std::string> {
  public:
-  explicit FileInputRecordReader(const std::string& filename) : file(filename) {
+  explicit FileInputRecordReader(const std::string& filename, const int64 max_records=-1) :
+      file(filename),
+      max_records_(max_records) {
     CHECK(exists(filename)) << "File '" << filename << "' does not exist!";
   }
   virtual ~FileInputRecordReader() override {
@@ -103,57 +113,23 @@ class FileInputRecordReader<std::string> : public InputRecordReader<std::string>
       }
       std::getline(file, *s);  // Read until we get a non-empty line.
     }
-    return true;
+    if (max_records_ != 0) {
+      max_records_--;
+      return true;
+    }
+    return false;
   }
 
   virtual bool ReachedEnd() override {
     std::lock_guard<std::mutex> lock(filemutex);
-    return file.eof();
+    return file.eof() || max_records_ == 0;
   }
  private:
   inline bool exists (const std::string& name) {
     return ( access( name.c_str(), F_OK ) != -1 );
   }
-};
-
-class FileListRecordReader : public InputRecordReader<std::string> {
-public:
-  explicit FileListRecordReader(const std::vector<std::string>& filelist) : filelist_(filelist), file_index_(0) {
-  }
-  virtual ~FileListRecordReader() override {
-  }
-
-  virtual bool Read(std::string* s) override {
-    s->clear();
-
-    std::string filename;
-    {
-      std::lock_guard<std::mutex> lock(file_index_mutex_);
-      if (file_index_ >= filelist_.size()) {
-        return false;
-      }
-      filename = filelist_[file_index_];
-      file_index_++;
-    }
 
-    CHECK(exists(filename)) << "File '" << filename << "' does not exist!";
-    ReadFileToStringOrDie(filename.c_str(), s);
-    return true;
-  }
-
-  virtual bool ReachedEnd() override {
-    std::lock_guard<std::mutex> lock(file_index_mutex_);
-    return file_index_ >= filelist_.size();
-  }
-
-private:
-  inline bool exists (const std::string& name) {
-    return ( access( name.c_str(), F_OK ) != -1 );
-  }
-
-  const std::vector<std::string>& filelist_;
-  size_t file_index_;
-  std::mutex file_index_mutex_;
+  int64 max_records_;
 };
 
 template <class T>
@@ -228,33 +204,20 @@ class RecordInput {
 template <class T>
 class FileRecordInput : public RecordInput<T> {
 public:
-  explicit FileRecordInput(const std::string& filename) : filename_(filename) {
+  explicit FileRecordInput(const std::string& filename, const int64 max_records=-1) :
+      filename_(filename),
+      max_records_(max_records) {
   }
   virtual ~FileRecordInput() override {
   }
 
   virtual InputRecordReader<T>* CreateReader() override {
-    return new FileInputRecordReader<T>(filename_);
+    return new FileInputRecordReader<T>(filename_, max_records_);
   }
 
 private:
   std::string filename_;
-};
-
-// Input where each records is the contents of a file.
-class FileListRecordInput : public RecordInput<std::string> {
-public:
-  explicit FileListRecordInput(std::vector<std::string>&& files) : files_(files) {
-  }
-  virtual ~FileListRecordInput() override {
-  }
-
-  virtual InputRecordReader<std::string>* CreateReader() override {
-    return new FileListRecordReader(files_);
-  }
-
-private:
-  std::vector<std::string> files_;
+  int64 max_records_;
 };
 
 /**
@@ -317,7 +280,6 @@ class CrossValidationReader : public InputRecordReader<T> {
       if ((training_ && (row_id_ % num_folds_) != fold_id_) ||
           (!training_ && (row_id_ % num_folds_) == fold_id_)) {
         return underlying_reader_->Read(s);
-        break;
       } else {
         T tmp;
         underlying_reader_->Read(&tmp);