Throttle late packet messages

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "xGPU"]
+	path = xGPU
+	url = https://github.com/GPU-correlators/xGPU
+[submodule "bitshuffle"]
+	path = bitshuffle
+	url = https://github.com/kiyo-masui/bitshuffle
+[submodule "hashpipe"]
+	path = hashpipe
+	url = https://github.com/hera-team/hashpipe.git

INSTALL

@@ -27,4 +27,11 @@ Example:
 If xGPU is installed under "/usr/local/xgpu16_256_32_titan", you can tell
 configure to find it there like this:
 
-$ ./configure --with-xgpu=/usr/local/xgpu16_256_32_titan
+$ ./configure --with-xgpu=/usr/local/xgpu-1080-dp4a
+
+If you need to build xGPU, it is included as a git submodule in this repository.
+Go into the xGPU/src directory and run (eg, for HERA H2C):
+make clean
+make NSTATION=192 NFREQUENCY=384 NTIME=2048 NTIME_PIPE=1024 CUDA_ARCH=sm_61 DP4A=yes
+sudo make install prefix=/usr/local/xgpu-1080-dp4a
+

install_hera.sh

@@ -0,0 +1,33 @@
+#! /bin/bash
+
+#default to 192 antennas
+n_ants=192
+
+while getopts ":a:h:" opt; do
+  case ${opt} in
+    a )
+      n_ants=$OPTARG
+      ;;
+    \? )
+      echo "Usage: $(basename $0) [-a]"
+      echo "  -a : number of antennas to compile for and install"
+      exit
+      ;;
+  esac
+done
+
+echo "Building and installing for ${n_ants} antennas"
+sleep 1;
+
+cd xGPU/src
+
+make clean
+make NSTATION=$n_ants NFREQUENCY=384 NTIME=2048 NTIME_PIPE=1024 CUDA_ARCH=sm_61 DP4A=yes
+sudo make install prefix="/usr/local/xgpu-1080-dp4a-384c-${n_ants}a"
+
+cd ../../src
+
+./configure --with-xgpu="/usr/local/xgpu-1080-dp4a-384c-${n_ants}a"
+make clean
+make
+sudo make install

print-nic-stats.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+ethtool -S eth3 | grep rx_dropped
+ethtool -S eth3 | grep vport_dropped
+ethtool -S eth3 | grep rx_fifo_errors
+ethtool -S eth3 | grep rx_missed_errors
+ethtool -S eth5 | grep rx_dropped
+ethtool -S eth5 | grep vport_dropped
+ethtool -S eth5 | grep rx_fifo_errors
+ethtool -S eth5 | grep rx_missed_errors

src/Makefile.am

@@ -1,46 +1,99 @@
+BUILD_USER = $(shell whoami)
+BUILD_HOST = $(shell hostname)
+BUILD_DATE = $(shell date +%Y/%m/%d-%H:%M%z)
+GIT_VERSION = $(shell git describe --abbrev=10 --dirty --always --tags)
+
 ACLOCAL_AMFLAGS = -I m4
 AM_CPPFLAGS     =
 
 AM_CPPFLAGS += -I"@HASHPIPE_INCDIR@"
 AM_CPPFLAGS += -I"@XGPU_INCDIR@"
+AM_CPPFLAGS += -I/usr/include/hdf5/serial/
+AM_CPPFLAGS += -I/usr/include/hiredis/
 
 # AM_CFLAGS is used for all C compiles
-AM_CFLAGS = -ggdb -fPIC -O3 -Wall -Werror -fno-strict-aliasing -mavx
+#AM_CFLAGS = -ggdb -fPIC -O3 -Wall -Werror -fno-strict-aliasing -mavx2
+AM_CFLAGS = -fPIC -O3 -Wall -fno-strict-aliasing -mavx2 -march=native -funroll-loops \
+            -DGIT_VERSION=\"$(GIT_VERSION)\" -DBUILD_USER=\"$(BUILD_USER)\" \
+            -DBUILD_HOST=\"$(BUILD_HOST)\" -DBUILD_DATE=\"$(BUILD_DATE)\"
 
 # Convenience variables to group source files
 paper_databuf = paper_databuf.h \
-	        paper_databuf.c
+	            paper_databuf.c
 
-paper_threads = paper_fake_net_thread.c       \
-		paper_gpu_cpu_output_thread.c \
-		paper_gpu_output_thread.c     \
-		paper_gpu_thread.c            \
-		paper_fluff_thread.c          \
-		paper_net_thread.c            \
-		paper_crc_thread.c
+paper_threads = paper_fake_net_thread.c          \
+                paper_gpu_thread.c               \
+                paper_fluff_thread.c             \
+                hera_pktsock_thread.c            \
+                hera_ibv_thread.c                \
+                hera_fake_gpu_thread.c           \
+                hera_gpu_output_thread.c         \
+                hera_gpu_bda_thread.c            \
+                hera_bda_output_thread.c         \
+                hera_null_thread.c               \
+                hera_catcher_net_thread_bda.c    \
+                hera_catcher_net_thread.c        \
+                hera_catcher_disk_thread_bda.c   \
+                hera_catcher_disk_thread.c       \
+                hera_catcher_autocorr_thread.c   \
+                lzf/lzf_c.c                      \
+                lzf/lzf_d.c                      \
+                lzf_filter.c                     \
+                lzf_filter.h
 
 paper_fluff = paper_fluff.h \
-	      paper_fluff.c
+	          paper_fluff.c
+
+hera_capture = hera_hdf5.h \
+               hera_write_hdf5.c
 
 # This is the paper_gpu plugin itself
 lib_LTLIBRARIES = paper_gpu.la
 paper_gpu_la_SOURCES  = $(paper_databuf) $(paper_threads) $(paper_fluff)
-paper_gpu_la_LIBADD   = -lrt -lxgpu
+paper_gpu_la_LIBADD   = -lrt -lxgpu -lhashpipe_ibverbs
 paper_gpu_la_LDFLAGS  = -avoid-version -module -shared -export-dynamic
 paper_gpu_la_LDFLAGS += -L"@XGPU_LIBDIR@" -Wl,-rpath,"@XGPU_LIBDIR@"
 paper_gpu_la_LDFLAGS += -L"@HASHPIPE_LIBDIR@" -Wl,-rpath,"@HASHPIPE_LIBDIR@"
+# Include HDF5 Libraries
+paper_gpu_la_LDFLAGS += -L/usr/lib/x86_64-linux-gnu/hdf5/serial
+paper_gpu_la_LDFLAGS += -lhdf5_hl -lhdf5 -lhiredis
+# for intel compiler:
+#paper_gpu_la_LDFLAGS += -L/usr/local/intel/compilers_and_libraries_2019.4.243/linux/compiler/lib/intel64_lin -Wl,-rpath,/usr/local/intel/compilers_and_libraries_2019.4.243/linux/compiler/lib/intel64_lin
+# Include HDF5 compression libraries
+#paper_gpu_la_LDFLAGS += -L/usr/local/lib/python2.7/dist-packages/bitshuffle/plugin
+#paper_gpu_la_LDFLAGS += -lh5LZF -lh5bshuf
+
 
 # Installed scripts
-dist_bin_SCRIPTS = paper_init.sh \
-		   paper_crc_init.sh \
-		   xeng_init.sh
+dist_bin_SCRIPTS = scripts/paper_init.sh              \
+                   scripts/paper_init_ibv.sh          \
+                   scripts/paper_crc_init.sh          \
+                   scripts/xtor_down.sh               \
+                   scripts/xtor_up.py                 \
+                   scripts/hera_catcher_init.sh       \
+                   scripts/hera_catcher_up.py         \
+                   scripts/hera_catcher_down.sh       \
+                   scripts/hera_catcher_take_data.py  \
+                   scripts/hera_catcher_stop_data.py  \
+                   scripts/tweak-perf.sh              \
+                   scripts/hera_ctl.py                \
+                   scripts/hera_create_bda_config.py  \
+                   scripts/hera_make_hdf5_template.py \
+                   scripts/hera_make_hdf5_template_bda.py 
 
 # Non-installed test programs
-noinst_PROGRAMS = paper_fluff_test
+noinst_PROGRAMS = paper_fluff_test hera_write_hdf5
 paper_fluff_test_SOURCES = paper_fluff_test.c $(paper_fluff)
 # The paper_fluff_test_CFLAGS is an autotools workaround.
 # http://www.gnu.org/software/automake/manual/html_node/Objects-created-both-with-libtool-and-without.html
 paper_fluff_test_CFLAGS = $(AM_CFLAGS)
 paper_fluff_test_LDADD = -lrt
 
+hera_write_hdf5_SOURCES = hera_write_hdf5 $(hera_capture)
+# The hera_write_hdf5_CFLAGS is an autotools workaround.
+# http://www.gnu.org/software/automake/manual/html_node/Objects-created-both-with-libtool-and-without.html
+hera_write_hdf5_CFLAGS = -O3 -Wall -Werror -fno-strict-aliasing -mavx2 -march=native -funroll-loops
+hera_write_hdf5_LDADD = -lrt -lpthread -lsz -lz -ldl -lm
+hera_write_hdf5_LDFLAGS = -L/usr/lib/x86_64-linux-gnu/hdf5/serial /usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5_hl.a /usr/lib/x86_64-linux-gnu/hdf5/serial/libhdf5.a
+
 # vi: set ts=8 noet :