Replace YAJL with yyjson for JSON parsing and generation

.github/workflows/test.yaml

@@ -38,21 +38,32 @@ jobs:
 
           install: |
             apt-get update -q -y
-            apt-get install -q -y python3 automake libtool autotools-dev git make cmake pkg-config gcc wget xz-utils
+            apt-get install -q -y python3 automake libtool autotools-dev git make pkg-config gcc wget xz-utils
 
           run: |
             find $(pwd) -name '.git' -exec bash -c 'git config --global --add safe.directory ${0%/.git}' {} \;
-            ./autogen.sh --enable-embedded-yajl
-            ./configure --enable-embedded-yajl CFLAGS='-Wall -Wextra -Werror'
+            ./autogen.sh --enable-embedded-yyjson
+            ./configure --enable-embedded-yyjson CFLAGS='-Wall -Wextra -Werror'
             # Run make check first to catch test failures with accessible logs
             make -j $(nproc)
             make check || (cat test-suite.log 2>/dev/null; exit 1)
             # Now run distcheck for distribution validation
-            make -j $(nproc) distcheck DISTCHECK_CONFIGURE_FLAGS="--enable-embedded-yajl"
+            make -j $(nproc) distcheck DISTCHECK_CONFIGURE_FLAGS="--enable-embedded-yyjson"
             # check that the working dir is clean
             git describe --broken --dirty --all | grep -qv dirty
             make clean
 
+  fuzzing:
+    runs-on: ubuntu-latest
+    name: Fuzzing
+    steps:
+      - uses: actions/checkout@v3.0.2
+        with:
+          submodules: true
+          set-safe-directory: true
+      - run: sudo docker build -t libocispec-fuzzing tests/fuzzing
+      - run: sudo docker run --rm -e RUN_TIME=120 -v ${PWD}:/libocispec libocispec-fuzzing
+
   test_and_build_rust_bindings:
     name: test and build rust bindings
     runs-on: ubuntu-latest

.gitmodules

@@ -4,6 +4,6 @@
 [submodule "image-spec"]
 	path = image-spec
 	url = https://github.com/opencontainers/image-spec
-[submodule "yajl"]
-	path = yajl
-	url = https://github.com/containers/yajl.git
+[submodule "yyjson"]
+	path = yyjson
+	url = https://github.com/ibireme/yyjson.git

Makefile.am

@@ -1,11 +1,8 @@
-DIST_SUBDIRS = yajl
-SUBDIRS = yajl
-
 AM_CFLAGS = $(WARN_CFLAGS) -I$(top_srcdir)/src -I$(top_builddir)/src
 
-if HAVE_EMBEDDED_YAJL
-AM_CFLAGS += -I$(top_srcdir)/yajl/src/headers
-endif HAVE_EMBEDDED_YAJL
+if HAVE_EMBEDDED_YYJSON
+AM_CFLAGS += -I$(top_srcdir)/yyjson/src
+endif HAVE_EMBEDDED_YYJSON
 
 CLEANFILES = $(man_MANS) src/runtime_spec_stamp src/image_spec_stamp src/image_manifest_stamp src/basic-test_stamp
 
@@ -53,6 +50,7 @@ SOURCE_FILES =							\
 	src/ocispec/basic_test_top_double_array_obj.c \
 	src/ocispec/basic_test_top_double_array_refobj.c \
 	src/ocispec/basic_test_top_double_array_string.c \
+	src/ocispec/basic_test_int64_values.c \
 	src/ocispec/basic_test_map_string_string_array.c
 
 HEADER_FILES = $(SOURCE_FILES:.c=.h)
@@ -147,6 +145,8 @@ src/ocispec/basic_test_double_array_item.h \
 	src/ocispec/basic_test_top_double_array_obj.c \
 	src/ocispec/basic_test_top_double_array_refobj.c \
 	src/ocispec/basic_test_top_double_array_string.c \
+	src/ocispec/basic_test_int64_values.h \
+	src/ocispec/basic_test_int64_values.c \
 	src/ocispec/basic_test_map_string_string_array.c: src/basic-test_stamp
 
 $(HEADER_FILES): %.h: %.c src/ocispec/generate.py
@@ -157,17 +157,20 @@ libocispec_la_SOURCES = $(BUILT_SOURCES) \
 	src/ocispec/read-file.c \
 	src/ocispec/json_common.c
 
+if HAVE_EMBEDDED_YYJSON
+libocispec_la_SOURCES += yyjson/src/yyjson.c
+endif
+
 TMP_H_FILES = $(HEADER_FILES:.h=.h.tmp)
 TMP_C_FILES = $(SOURCE_FILES:.c=.c.tmp)
 
 CLEANFILES += $(HEADER_FILES) $(SOURCE_FILES) $(TMP_H_FILES) $(TMP_C_FILES)
 
 TESTS_LDADD = libocispec.la $(SELINUX_LIBS)
 
-if HAVE_EMBEDDED_YAJL
-TESTS_LDADD += yajl/libyajl.la
+if HAVE_EMBEDDED_YYJSON
 else
-TESTS_LDADD += $(YAJL_LIBS)
+TESTS_LDADD += $(YYJSON_LIBS)
 endif
 
 libocispec_a_SOURCES =
@@ -222,6 +225,9 @@ tests_test_13_LDADD = $(TESTS_LDADD)
 tests_test_14_SOURCES = tests/test-14.c
 tests_test_14_LDADD = $(TESTS_LDADD)
 
+tests_test_15_SOURCES = tests/test-15.c
+tests_test_15_LDADD = $(TESTS_LDADD)
+
 src_ocispec_validate_SOURCES = src/ocispec/validate.c
 src_ocispec_validate_LDADD = $(TESTS_LDADD)
 
@@ -238,7 +244,8 @@ TESTS = tests/test-1 \
 	tests/test-11 \
 	tests/test-12 \
 	tests/test-13 \
-	tests/test-14
+	tests/test-14 \
+	tests/test-15
 
 noinst_PROGRAMS = src/ocispec/validate $(TESTS)
 
@@ -269,6 +276,7 @@ EXTRA_DIST = autogen.sh \
 	tests/data/top_double_array_string.json \
 	tests/data/config-netdevices.json \
 	tests/data/map-string-string-array.json \
+	tests/data/int64_values.json \
 	tests/test-spec \
 	src/ocispec/generate.py \
 	src/ocispec/headers.py \
@@ -281,12 +289,12 @@ EXTRA_DIST = autogen.sh \
 	image-spec \
 	src/ocispec/json_common.h \
 	src/ocispec/json_common.c \
-	src/yajl
+	yyjson/src/yyjson.h
 
 sync:
 	(cd image-spec; git pull https://github.com/opencontainers/image-spec)
 	(cd runtime-spec; git pull https://github.com/opencontainers/runtime-spec)
-	(cd yajl; git pull https://github.com/containers/yajl main)
+
 
 generate: src/runtime_spec_stamp src/image_spec_stamp src/image_manifest_stamp src/basic-test_stamp
 

README.md

@@ -11,7 +11,7 @@ from C, and generate json string from corresponding struct.
 The parser is generated directly from the JSON schema in the source repository.
 
 ## Installation
-Expects [yajl](https://github.com/containers/yajl) to be installed and linkable.
+Expects [yyjson](https://github.com/ibireme/yyjson) to be installed and linkable.
 ```sh
 $ ./autogen.sh
 $ ./configure

autogen.sh

@@ -1,7 +1,5 @@
 #!/bin/sh
 
-(cd yajl; ./autogen.sh)
-
 git submodule update --init --recursive
 
 test -n "$srcdir" || srcdir=`dirname "$0"`

configure.ac

@@ -12,23 +12,26 @@ AM_INIT_AUTOMAKE([1.11 -Wno-portability foreign tar-ustar no-dist-gzip dist-xz s
 AM_MAINTAINER_MODE([enable])
 AM_SILENT_RULES([yes])
 
-AM_EXTRA_RECURSIVE_TARGETS([yajl])
-AC_CONFIG_SUBDIRS([yajl])
-
-AC_ARG_ENABLE(embedded-yajl,
-AS_HELP_STRING([--enable-embedded-yajl], [Statically link a modified yajl version]),
+AC_ARG_ENABLE(embedded-yyjson,
+AS_HELP_STRING([--enable-embedded-yyjson], [Statically link the embedded yyjson version]),
 [
 case "${enableval}" in
-  yes) embedded_yajl=true ;;
-  no)  embedded_yajl=false ;;
-  *) AC_MSG_ERROR(bad value ${enableval} for --enable-embedded-yajl) ;;
-esac],[embedded_yajl=false])
-
-AM_CONDITIONAL([HAVE_EMBEDDED_YAJL], [test x"$embedded_yajl" = xtrue])
-AM_COND_IF([HAVE_EMBEDDED_YAJL], [], [
-AC_SEARCH_LIBS(yajl_tree_get, [yajl], [AC_DEFINE([HAVE_YAJL], 1, [Define if libyajl is available])], [AC_MSG_ERROR([*** libyajl headers not found])])
-PKG_CHECK_MODULES([YAJL], [yajl >= 2.0.0])
+  yes) embedded_yyjson=true ;;
+  no)  embedded_yyjson=false ;;
+  *) AC_MSG_ERROR(bad value ${enableval} for --enable-embedded-yyjson) ;;
+esac],[embedded_yyjson=auto])
+
+if test x"$embedded_yyjson" != xtrue; then
+PKG_CHECK_MODULES([YYJSON], [yyjson >= 0.8.0], [embedded_yyjson=false], [
+  if test x"$embedded_yyjson" = xauto; then
+    AC_MSG_NOTICE([yyjson not found on the system, using the embedded version])
+    embedded_yyjson=true
+  else
+    AC_MSG_ERROR([yyjson >= 0.8.0 not found and embedded yyjson is disabled])
+  fi
 ])
+fi
+AM_CONDITIONAL([HAVE_EMBEDDED_YYJSON], [test x"$embedded_yyjson" = xtrue])
 
 # Optionally install the library.
 AC_ARG_ENABLE(libocispec-install,

ocispec.pc.in

@@ -5,7 +5,7 @@ includedir=@includedir@
 
 Name: @PACKAGE_NAME@
 Description: A library for easily parsing [OCI runtime](https://github.com/opencontainers/runtime-spec) and [OCI image](https://github.com/opencontainers/image-spec) files.
-Requires: yajl
+Requires: yyjson
 Version: @PACKAGE_VERSION@
 Libs: -L${libdir} -locispec
 Cflags: -I${includedir}/ocispec

src/ocispec/generate.py

@@ -31,6 +31,8 @@
 import sys
 import json
 import argparse
+import subprocess
+import shutil
 
 from collections import OrderedDict
 import helpers
@@ -731,6 +733,11 @@ def reflection(schema_info, gen_ref):
             except:
                 pass
 
+    clang_format = shutil.which('clang-format')
+    if clang_format:
+        for filepath in [schema_info.header.name, schema_info.source.name]:
+            subprocess.run([clang_format, '-i', filepath], check=False)
+
     if gen_ref is True:
         if schema_info.refs:
             for reffile in schema_info.refs.values():

src/ocispec/headers.py

@@ -38,7 +38,7 @@ def append_header_arr(obj, header, prefix):
     if not obj.subtypobj or obj.subtypname:
         return
 
-    header.append("typedef struct {\n")
+    header.append("typedef struct\n{\n")
     for i in obj.subtypobj:
         if i.typ == 'array':
             c_typ = helpers.get_prefixed_pointer(i.name, i.subtyp, prefix) or \
@@ -47,21 +47,21 @@ def append_header_arr(obj, header, prefix):
                 c_typ = helpers.get_name_substr(i.name, prefix)
 
             if not helpers.is_compound_type(i.subtyp):
-                header.append(f"    {c_typ}{' ' if '*' not in c_typ else ''}*{i.fixname};\n")
+                header.append(f"  {c_typ}{' ' if '*' not in c_typ else ''}*{i.fixname};\n")
             else:
-                header.append(f"    {c_typ} **{i.fixname};\n")
-            header.append(f"    size_t {i.fixname + '_len'};\n\n")
+                header.append(f"  {c_typ} **{i.fixname};\n")
+            header.append(f"  size_t {i.fixname + '_len'};\n\n")
         else:
             c_typ = helpers.get_prefixed_pointer(i.name, i.typ, prefix) or \
                 helpers.get_map_c_types(i.typ)
-            header.append(f"    {c_typ}{' ' if '*' not in c_typ else ''}{i.fixname};\n")
+            header.append(f"  {c_typ}{' ' if '*' not in c_typ else ''}{i.fixname};\n")
     for i in obj.subtypobj:
         if helpers.is_numeric_type(i.typ) or i.typ == 'boolean':
-            header.append(f"    unsigned int {i.fixname}_present : 1;\n")
+            header.append(f"  unsigned int {i.fixname}_present : 1;\n")
     typename = helpers.get_name_substr(obj.name, prefix)
-    header.append(f"}}\n{typename};\n\n")
+    header.append(f"}} {typename};\n\n")
     header.append(f"void free_{typename} ({typename} *ptr);\n\n")
-    header.append(f"{typename} *make_{typename} ({json_api.VAL_TYPE} tree, const struct parser_context *ctx, parser_error *err);\n\n")
+    header.append(f"{typename} *make_{typename} ({json_api.VAL_TYPE}tree, const struct parser_context *ctx, parser_error *err);\n\n")
 
 
 def append_header_map_str_obj(obj, header, prefix):
@@ -71,16 +71,16 @@ def append_header_map_str_obj(obj, header, prefix):
     History: 2019-06-17
     '''
     child = obj.children[0]
-    header.append("typedef struct {\n")
-    header.append("    char **keys;\n")
+    header.append("typedef struct\n{\n")
+    header.append("  char **keys;\n")
     if helpers.valid_basic_map_name(child.typ):
         c_typ = helpers.get_prefixed_pointer("", child.typ, "")
     elif child.subtypname:
         c_typ = child.subtypname +  " *"
     else:
         c_typ = helpers.get_prefixed_pointer(child.name, child.typ, prefix)
-    header.append(f"    {c_typ}{' ' if '*' not in c_typ else ''}*{child.fixname};\n")
-    header.append("    size_t len;\n")
+    header.append(f"  {c_typ}{' ' if '*' not in c_typ else ''}*{child.fixname};\n")
+    header.append("  size_t len;\n")
 
 
 def append_header_child_arr(child, header, prefix):
@@ -105,16 +105,16 @@ def append_header_child_arr(child, header, prefix):
         dflag = "*"
 
     if helpers.valid_basic_map_name(child.subtyp):
-        header.append(f"    {helpers.make_basic_map_name(child.subtyp)} **{child.fixname};\n")
+        header.append(f"  {helpers.make_basic_map_name(child.subtyp)} **{child.fixname};\n")
     elif not helpers.is_compound_type(child.subtyp):
-        header.append(f"    {c_typ}{' ' if '*' not in c_typ else ''}*{dflag}{child.fixname};\n")
+        header.append(f"  {c_typ}{' ' if '*' not in c_typ else ''}*{dflag}{child.fixname};\n")
     else:
-        header.append(f"    {c_typ}{' ' if '*' not in c_typ else ''}**{dflag}{child.fixname};\n")
+        header.append(f"  {c_typ}{' ' if '*' not in c_typ else ''}**{dflag}{child.fixname};\n")
 
     if child.nested_array and not helpers.valid_basic_map_name(child.subtyp):
-        header.append(f"    size_t *{child.fixname + '_item_lens'};\n")
+        header.append(f"  size_t *{child.fixname + '_item_lens'};\n")
 
-    header.append(f"    size_t {child.fixname + '_len'};\n\n")
+    header.append(f"  size_t {child.fixname + '_len'};\n\n")
 
 def append_header_child_others(child, header, prefix):
     '''
@@ -130,7 +130,7 @@ def append_header_child_others(child, header, prefix):
         c_typ = helpers.get_prefixed_pointer(child.subtypname, child.typ, "")
     else:
         c_typ = helpers.get_prefixed_pointer(child.name, child.typ, prefix)
-    header.append(f"    {c_typ}{' ' if '*' not in c_typ else ''}{child.fixname};\n\n")
+    header.append(f"  {c_typ}{' ' if '*' not in c_typ else ''}{child.fixname};\n\n")
 
 
 def append_type_c_header(obj, header, prefix):
@@ -153,29 +153,29 @@ def append_type_c_header(obj, header, prefix):
     elif obj.typ == 'object':
         if obj.subtypname is not None:
             return
-        header.append("typedef struct {\n")
+        header.append("typedef struct\n{\n")
         if obj.children is None:
-            header.append("    char unuseful; // unuseful definition to avoid empty struct\n")
+            header.append("  char unuseful; // unuseful definition to avoid empty struct\n")
         present_tags = []
         for i in obj.children or []:
             if helpers.is_numeric_type(i.typ) or i.typ == 'boolean':
-                present_tags.append(f"    unsigned int {i.fixname}_present : 1;\n")
+                present_tags.append(f"  unsigned int {i.fixname}_present : 1;\n")
             if i.typ == 'array':
                 append_header_child_arr(i, header, prefix)
             else:
                 append_header_child_others(i, header, prefix)
         if obj.children is not None:
-            header.append(f"    {json_api.VAL_TYPE} _residual;\n")
+            header.append(f"  {json_api.RESIDUAL_TYPE}_residual;\n")
         if len(present_tags) > 0:
             header.append("\n")
             for tag in present_tags:
                 header.append(tag)
     typename = helpers.get_prefixed_name(obj.name, prefix)
-    header.append(f"}}\n{typename};\n\n")
+    header.append(f"}} {typename};\n\n")
     header.append(f"void free_{typename} ({typename} *ptr);\n\n")
     header.append(f"{typename} *clone_{typename} ({typename} *src);\n")
-    header.append(f"{typename} *make_{typename} ({json_api.VAL_TYPE} tree, const struct parser_context *ctx, parser_error *err);\n\n")
-    header.append(f"{json_api.GEN_STATUS_TYPE} gen_{typename} ({json_api.GEN_TYPE} g, const {typename} *ptr, const struct parser_context *ctx, parser_error *err);\n\n")
+    header.append(f"{typename} *make_{typename} ({json_api.VAL_TYPE}tree, const struct parser_context *ctx, parser_error *err);\n\n")
+    header.append(f"{json_api.GEN_STATUS_TYPE} gen_{typename} ({json_api.GEN_TYPE}g, const {typename} *ptr, const struct parser_context *ctx, parser_error *err);\n\n")
 
 def header_reflect_top_array(obj, prefix, header):
     c_typ = helpers.get_prefixed_pointer(obj.name, obj.subtyp, prefix) or \
@@ -189,14 +189,14 @@ def header_reflect_top_array(obj, prefix, header):
         return
 
     typename = helpers.get_top_array_type_name(obj.name, prefix)
-    header.append("typedef struct {\n")
+    header.append("typedef struct\n{\n")
     if obj.nested_array:
-        header.append(f"    {c_typ}{' ' if '*' not in c_typ else ''}**items;\n")
-        header.append("    size_t *subitem_lens;\n\n")
+        header.append(f"  {c_typ}{' ' if '*' not in c_typ else ''}**items;\n")
+        header.append("  size_t *subitem_lens;\n\n")
     else:
-        header.append(f"    {c_typ}{' ' if '*' not in c_typ else ''}*items;\n")
-    header.append("    size_t len;\n\n")
-    header.append(f"}}\n{typename};\n\n")
+        header.append(f"  {c_typ}{' ' if '*' not in c_typ else ''}*items;\n")
+    header.append("  size_t len;\n\n")
+    header.append(f"}} {typename};\n\n")
 
 
     header.append(f"void free_{typename} ({typename} *ptr);\n\n")