From 9aa0265e94212a64e776217301a36cc72381e4fb Mon Sep 17 00:00:00 2001 From: Siddhesh Poyarekar Date: Fri, 1 Mar 2019 23:00:48 +0530 Subject: [PATCH 1/7] Tighten up timing computation The current timing computation has an implicit assumption that the benchmark ran for 10 seconds when in reality it will be a bit off, by a few microseconds, if not milliseconds. Fix this by recording the start and end times at the entry and end points of each thread and then use the earliest start and latest end times to get an estimate of the total time the script iterations ran for. Also increase the precision of the ops/s rate. --- main.c | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/main.c b/main.c index 3573f7f..528d5e6 100644 --- a/main.c +++ b/main.c @@ -9,6 +9,7 @@ #include #include #include +#include #include "lua.h" #include "lualib.h" @@ -17,15 +18,20 @@ typedef struct { uint64_t ctr; char *file; - double start_ns; + uint64_t start_ns; + uint64_t end_ns; } lua_task; void *wrapper(void *arg) { lua_task *task = (lua_task*)arg; + struct timespec start_time; struct timespec cur_time; - double start_time_ns = task->start_ns; uint64_t ctr = 0; + clock_gettime(CLOCK_MONOTONIC, &start_time); + uint64_t start_time_ns = start_time.tv_sec * 1e9 + start_time.tv_nsec; + uint64_t cur_time_ns = start_time_ns; + lua_State *L; L = luaL_newstate(); luaL_openlibs(L); @@ -38,10 +44,11 @@ void *wrapper(void *arg) { int cb_ref = luaL_ref(L, LUA_REGISTRYINDEX); + do { clock_gettime(CLOCK_MONOTONIC, &cur_time); - double cur_time_ns = cur_time.tv_sec * 1e9 + cur_time.tv_nsec; + cur_time_ns = cur_time.tv_sec * 1e9 + cur_time.tv_nsec; if (start_time_ns + 10 * 1e9 < cur_time_ns) { break; } @@ -58,6 +65,8 @@ void *wrapper(void *arg) { } while(1); task->ctr = ctr; + task->start_ns = start_time_ns; + task->end_ns = cur_time_ns; return NULL; } @@ -122,7 +131,6 @@ int main(int argc, char *argv[]) { size_t len; struct stat s; uint64_t total = 0; - struct timespec start_time; cmd_line_options arguments = (cmd_line_options){.file_name = "", .q = 8, .c = 1}; argp_parse (&argp, argc, argv, 0, 0, &arguments); @@ -131,19 +139,30 @@ int main(int argc, char *argv[]) { threads = malloc(nthreads * sizeof(pthread_t)); tasks = malloc(nthreads * sizeof(lua_task)); - clock_gettime(CLOCK_MONOTONIC, &start_time); - double start_time_ns = start_time.tv_sec * 1e9 + start_time.tv_nsec; - for (i = 0; i < nthreads; i++) { - tasks[i] = (lua_task){.file = arguments.file_name, .start_ns = start_time_ns}; + tasks[i] = (lua_task){.file = arguments.file_name}; pthread_create(&threads[i], NULL, wrapper, &tasks[i]); } + uint64_t start_time_ns = 0; + uint64_t end_time_ns = 0; + for (i = 0; i < nthreads; i++) { pthread_join(threads[i], NULL); total += tasks[i].ctr; + /* Select the earliest starting time and the latest end time. */ + if (start_time_ns == 0) { + start_time_ns = tasks[i].start_ns; + end_time_ns = tasks[i].end_ns; + } + else { + start_time_ns = MIN (start_time_ns, tasks[i].start_ns); + end_time_ns = MAX (end_time_ns, tasks[i].end_ns); + } } + uint64_t duration = end_time_ns - start_time_ns; + fprintf(stderr, "Total times executed: %"PRId64"\n", total); - fprintf(stdout, "ops/s %.2f\n", (double)total / 10); + fprintf(stdout, "ops/s %.4f\n", (double)total * 1e9 / duration); } From 6c80eb1f510992d4db91fe382585d6e60d9bb12e Mon Sep 17 00:00:00 2001 From: Siddhesh Poyarekar Date: Fri, 1 Mar 2019 23:45:54 +0530 Subject: [PATCH 2/7] Add error checking for all functions that could fail --- main.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/main.c b/main.c index 528d5e6..37f5338 100644 --- a/main.c +++ b/main.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "lua.h" #include "lualib.h" @@ -28,7 +29,13 @@ void *wrapper(void *arg) { struct timespec cur_time; uint64_t ctr = 0; - clock_gettime(CLOCK_MONOTONIC, &start_time); + int status = clock_gettime(CLOCK_MONOTONIC, &start_time); + + if (status != 0) { + fprintf(stderr, "clock_gettime failed: %s\n", strerror (errno)); + exit(1); + } + uint64_t start_time_ns = start_time.tv_sec * 1e9 + start_time.tv_nsec; uint64_t cur_time_ns = start_time_ns; @@ -36,17 +43,22 @@ void *wrapper(void *arg) { L = luaL_newstate(); luaL_openlibs(L); - int status = luaL_loadfile(L, task->file); - if (status) { - fprintf(stderr, "Couldn't load file: %s\n", lua_tostring(L, -1)); - exit(1); - } + status = luaL_loadfile(L, task->file); + if (status) { + fprintf(stderr, "Couldn't load file: %s\n", lua_tostring(L, -1)); + exit(1); + } int cb_ref = luaL_ref(L, LUA_REGISTRYINDEX); do { - clock_gettime(CLOCK_MONOTONIC, &cur_time); + status = clock_gettime(CLOCK_MONOTONIC, &cur_time); + if (status != 0) { + fprintf(stderr, "clock_gettime failed: %s\n", strerror (errno)); + exit(1); + } + cur_time_ns = cur_time.tv_sec * 1e9 + cur_time.tv_nsec; if (start_time_ns + 10 * 1e9 < cur_time_ns) { @@ -133,22 +145,46 @@ int main(int argc, char *argv[]) { uint64_t total = 0; cmd_line_options arguments = (cmd_line_options){.file_name = "", .q = 8, .c = 1}; - argp_parse (&argp, argc, argv, 0, 0, &arguments); + int ret = argp_parse (&argp, argc, argv, 0, 0, &arguments); + + if (ret != 0) { + fprintf (stderr, "Unexpected error while parsing command line arguments: %s\n", + strerror (ret)); + return 1; + } int nthreads = arguments.c; threads = malloc(nthreads * sizeof(pthread_t)); tasks = malloc(nthreads * sizeof(lua_task)); + if (threads == NULL || tasks == NULL) { + fprintf(stderr, "Unable to allocate memory: %s\n", strerror (errno)); + return 2; + } + for (i = 0; i < nthreads; i++) { tasks[i] = (lua_task){.file = arguments.file_name}; - pthread_create(&threads[i], NULL, wrapper, &tasks[i]); + int ret = pthread_create(&threads[i], NULL, wrapper, &tasks[i]); + + if (ret != 0) { + fprintf (stderr, "Cannot create thread(%d): %s\n", i, + strerror (errno)); + return 3; + } } uint64_t start_time_ns = 0; uint64_t end_time_ns = 0; for (i = 0; i < nthreads; i++) { - pthread_join(threads[i], NULL); + int ret = pthread_join(threads[i], NULL); + + if (ret != 0) { + fprintf (stderr, "Thread join failed(%d): %s\n", i, + strerror (errno)); + return 4; + } + total += tasks[i].ctr; /* Select the earliest starting time and the latest end time. */ if (start_time_ns == 0) { @@ -165,4 +201,6 @@ int main(int argc, char *argv[]) { fprintf(stderr, "Total times executed: %"PRId64"\n", total); fprintf(stdout, "ops/s %.4f\n", (double)total * 1e9 / duration); + + return 0; } From f6f6dee5d184bb6b3a64d4d15505613008925f3b Mon Sep 17 00:00:00 2001 From: Siddhesh Poyarekar Date: Sat, 2 Mar 2019 00:02:28 +0530 Subject: [PATCH 3/7] Make benchmark execution time configurable at build Add a BENCH_DURATION macro that allows users to increase or reduce the benchmark execution time. --- Makefile | 6 +++++- main.c | 7 ++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index dfb1a71..0576b47 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,13 @@ ifeq ($(shell uname), Darwin) argp:=-largp -pagezero_size 10000 -image_base 100000000 endif +ifdef BENCH_DURATION + DURATION = -DBENCH_DURATION=$(BENCH_DURATION) +endif + all: LuaJIT/Makefile cd ./LuaJIT && make -j - $(CC) main.c -O3 -c -o main.o -I ./LuaJIT/src + $(CC) main.c $(DURATION) -O3 -c -o main.o -I ./LuaJIT/src $(CC) main.o -lpthread ${argp} ./LuaJIT/src/libluajit.a -lm -ldl -o bench LuaJIT/Makefile: diff --git a/main.c b/main.c index 37f5338..ee2fe2b 100644 --- a/main.c +++ b/main.c @@ -23,6 +23,11 @@ typedef struct { uint64_t end_ns; } lua_task; +/* Benchmark execution duration in seconds. */ +#ifndef BENCH_DURATION +# define BENCH_DURATION 10 +#endif + void *wrapper(void *arg) { lua_task *task = (lua_task*)arg; struct timespec start_time; @@ -61,7 +66,7 @@ void *wrapper(void *arg) { cur_time_ns = cur_time.tv_sec * 1e9 + cur_time.tv_nsec; - if (start_time_ns + 10 * 1e9 < cur_time_ns) { + if (start_time_ns + BENCH_DURATION * 1e9 < cur_time_ns) { break; } From 78e2ef3b9cb78b79c898922b6a8a2f3a447d1fe3 Mon Sep 17 00:00:00 2001 From: Siddhesh Poyarekar Date: Sat, 2 Mar 2019 00:18:54 +0530 Subject: [PATCH 4/7] New make target to run benchmarks Add a new target "make run" to run all *.lua benchmarks. Also modify the output of the benchmark so that it is amenable to simple parsing. --- Makefile | 4 ++++ main.c | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0576b47..119cd05 100644 --- a/Makefile +++ b/Makefile @@ -24,3 +24,7 @@ distclean: rm -rf LuaJIT mkdir LuaJIT rm bench main.o + +run: bench + @echo "::" + @ls *.lua | while read f; do echo -n "$$f:"; ./bench $$f; done diff --git a/main.c b/main.c index ee2fe2b..47ad02c 100644 --- a/main.c +++ b/main.c @@ -204,8 +204,7 @@ int main(int argc, char *argv[]) { uint64_t duration = end_time_ns - start_time_ns; - fprintf(stderr, "Total times executed: %"PRId64"\n", total); - fprintf(stdout, "ops/s %.4f\n", (double)total * 1e9 / duration); + printf("%"PRId64":%.4f\n", total, (double) total * 1e9 / duration); return 0; } From 5d1c3cbbf85127d6a81720c3b4e7330c3d3e52de Mon Sep 17 00:00:00 2001 From: Siddhesh Poyarekar Date: Wed, 6 Mar 2019 19:40:18 +0530 Subject: [PATCH 5/7] Allow specifying a different LuaJIT path to build the benchmark This allows building the benchmark binary against different LuaJIT static libraries and not just the one in the submodule. This allows users the freedom to test and post changes without changing the state of the submodule. --- Makefile | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 119cd05..29933f3 100644 --- a/Makefile +++ b/Makefile @@ -8,16 +8,23 @@ ifdef BENCH_DURATION DURATION = -DBENCH_DURATION=$(BENCH_DURATION) endif -all: LuaJIT/Makefile - cd ./LuaJIT && make -j - $(CC) main.c $(DURATION) -O3 -c -o main.o -I ./LuaJIT/src - $(CC) main.o -lpthread ${argp} ./LuaJIT/src/libluajit.a -lm -ldl -o bench +ifndef LUAJIT_PATH + LUAJIT_PATH = ./LuaJIT +endif + +LUAJIT_A = $(LUAJIT_PATH)/src/libluajit.a + +all: $(LUAJIT_PATH)/src/libluajit.a + $(CC) main.c $(DURATION) -O3 -c -o main.o -I $(LUAJIT_PATH)/src + $(CC) main.o -lpthread ${argp} $^ -lm -ldl -o bench + +$(LUAJIT_A): $(LUAJIT_PATH)/Makefile + make -C $(LUAJIT_PATH) -j LuaJIT/Makefile: git submodule update --init --recursive clean: - cd ./LuaJIT && make clean rm bench main.o distclean: From 31dd9d1e36dfb89e877646ee4068cdc054bc8d92 Mon Sep 17 00:00:00 2001 From: Siddhesh Poyarekar Date: Mon, 11 Mar 2019 20:11:15 +0530 Subject: [PATCH 6/7] Clean up the timing logic a bit --- main.c | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/main.c b/main.c index 47ad02c..ca05e5b 100644 --- a/main.c +++ b/main.c @@ -28,6 +28,8 @@ typedef struct { # define BENCH_DURATION 10 #endif +#define TIMESPEC_NS(t) ((t).tv_sec * 1e9 + (t).tv_nsec) + void *wrapper(void *arg) { lua_task *task = (lua_task*)arg; struct timespec start_time; @@ -41,8 +43,8 @@ void *wrapper(void *arg) { exit(1); } - uint64_t start_time_ns = start_time.tv_sec * 1e9 + start_time.tv_nsec; - uint64_t cur_time_ns = start_time_ns; + uint64_t cur_time_ns = TIMESPEC_NS (start_time); + uint64_t stop_time_ns = TIMESPEC_NS (start_time) + BENCH_DURATION * 1e9; lua_State *L; L = luaL_newstate(); @@ -58,18 +60,6 @@ void *wrapper(void *arg) { do { - status = clock_gettime(CLOCK_MONOTONIC, &cur_time); - if (status != 0) { - fprintf(stderr, "clock_gettime failed: %s\n", strerror (errno)); - exit(1); - } - - - cur_time_ns = cur_time.tv_sec * 1e9 + cur_time.tv_nsec; - if (start_time_ns + BENCH_DURATION * 1e9 < cur_time_ns) { - break; - } - lua_rawgeti(L, LUA_REGISTRYINDEX, cb_ref); int result = lua_pcall(L, 0, 0, 0); if (result) { @@ -78,11 +68,17 @@ void *wrapper(void *arg) { } ctr++; + status = clock_gettime(CLOCK_MONOTONIC, &cur_time); + if (status != 0) { + fprintf(stderr, "clock_gettime failed: %s\n", strerror (errno)); + exit(1); + } - } while(1); + cur_time_ns = TIMESPEC_NS (cur_time); + } while (stop_time_ns > cur_time_ns);; task->ctr = ctr; - task->start_ns = start_time_ns; + task->start_ns = TIMESPEC_NS (start_time); task->end_ns = cur_time_ns; return NULL; From bffa9320b8463e7fe46296eda850bd4797590816 Mon Sep 17 00:00:00 2001 From: Siddhesh Poyarekar Date: Mon, 11 Mar 2019 20:14:42 +0530 Subject: [PATCH 7/7] Fix spelling of concurrency --- main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.c b/main.c index ca05e5b..fc838d3 100644 --- a/main.c +++ b/main.c @@ -102,7 +102,7 @@ static char doc[] = static char args_doc[] = "file1"; static struct argp_option options[] = { - {"concurency", 'c', "concurency", 0, "Number of threads" }, + {"concurrency", 'c', "concurrency", 0, "Number of threads" }, { 0 } };