From cb0274dbc6778783eb6e46a482a49e3ace4cea70 Mon Sep 17 00:00:00 2001
From: Tim Bird <tim.bird@sony.com>
Date: Fri, 14 Nov 2025 09:58:18 -0700
Subject: [PATCH] Add SPDX analysis utilities

Add some helper utilities to get information on the status of
SPDX-License-Identifier lines in source files reported in esstra data.

'esstra-full-paths' is a brief utility to convert Directories and
filenames in 'esstra show' output into full paths (used by has-spdx-id.py)
This utility could possibly be replaced with a command line argument
to the 'esstra' utility to show full paths.  I started looking at that
but the output handling in the 'esstra' utility was a bit more
complex than I was expecting.  This still might be the better way to
handle this issue.

'has-spdx-id.py' is a utility to scan a file or list of files and report
if they have SPDX-License-Identifier lines.  It can also generate a
count of files with and without the desired SPDX line.

'esstra-to-spdx-list.sh' is a program to take esstra data (as reported
by 'esstra show', or as saved into a standalone file), and pipe it
through standard Linux utilities and has-spdx-id.py to generate a report
on the status of source files (whether they have SPDX-License-Identifier
lines or not).

These tools were used on the Linux kernel, using an Ubuntu (24.4) x86_64 kernel
configuration for my Dell Desktop machine.  The preliminary results for
just the Linux kernel binary file (vmlinux at the top level directory)
were:
  Files with SPDX lines: 6252
  Files without SPDX lines: 577

Note that this omitted other files that would normally be associated
with a full kernel built, such as all kernel module sources, and
the entry and decompressor code used in a compressed kernel image
(such as bzImage)

Signed-off-by: Tim Bird <tim.bird@sony.com>
---
 Makefile                    |   8 ++-
 util/esstra-full-paths      |  17 ++++++
 util/esstra-to-spdx-list.sh |  54 ++++++++++++++++++
 util/has-spdx-id.py         | 110 ++++++++++++++++++++++++++++++++++++
 4 files changed, 188 insertions(+), 1 deletion(-)
 create mode 100755 util/esstra-full-paths
 create mode 100755 util/esstra-to-spdx-list.sh
 create mode 100755 util/has-spdx-id.py

diff --git a/Makefile b/Makefile
index e60d2ab..4a1c63a 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,9 @@ PREFIX ?= /usr/local
 ESSTRACORE := esstracore.so
 ESSTRALINK := esstralink.so
 ESSTRAUTIL := esstra
+UTIL2 := esstra-full-paths
+UTIL3 := has-spdx-id.py
+UTIL4 := esstra-to-spdx-list.sh
 
 SUBDIRS := core link util
 
@@ -28,7 +31,7 @@ all clean:
 
 install: all
 	install -m 0755 -D -t $(INSTALLDIR_PLUGIN) core/$(ESSTRACORE) link/$(ESSTRALINK)
-	install -m 0755 -D -t $(INSTALLDIR_BIN) util/$(ESSTRAUTIL)
+	install -m 0755 -D -t $(INSTALLDIR_BIN) util/$(ESSTRAUTIL) util/$(UTIL2) util/$(UTIL3) util/$(UTIL4)
 
 install-specs: all
 	@gcc -dumpspecs | \
@@ -40,6 +43,9 @@ uninstall: uninstall-specs
 	rm -f $(INSTALLDIR_PLUGIN)/$(ESSTRACORE)
 	rm -f $(INSTALLDIR_PLUGIN)/$(ESSTRALINK)
 	rm -f $(INSTALLDIR_BIN)/$(ESSTRAUTIL)
+	rm -f $(INSTALLDIR_BIN)/$(UTIL2)
+	rm -f $(INSTALLDIR_BIN)/$(UTIL3)
+	rm -f $(INSTALLDIR_BIN)/$(UTIL4)
 
 uninstall-specs:
 	@[ -e $(SPECFILE) ] && \
diff --git a/util/esstra-full-paths b/util/esstra-full-paths
new file mode 100755
index 0000000..8890ec7
--- /dev/null
+++ b/util/esstra-full-paths
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+# esstra-full-paths
+#   parse esstra output and show full filepaths
+#
+# do I need to strip off a base dir here?
+
+import fileinput
+
+cur_dir = ""
+for line in fileinput.input():
+    line = line.strip()
+    if line.startswith("- Directory:"):
+        cur_dir = line.split("Directory: ", 1)[1]
+        continue
+    if line.startswith("- File:"):
+        filename = line.split("File: ", 1)[1]
+        print( "- Path: " + cur_dir + "/" + filename)
diff --git a/util/esstra-to-spdx-list.sh b/util/esstra-to-spdx-list.sh
new file mode 100755
index 0000000..b3932bd
--- /dev/null
+++ b/util/esstra-to-spdx-list.sh
@@ -0,0 +1,54 @@
+#!/bin/sh
+# esstra-to-spdx-list.sh - convert esstra output into an spdx list
+#   usage: esstra-to-spdx-list.sh <filename>
+#
+
+usage() {
+    cat <<HERE
+Usage: esstra-to-spdx-list.sh [-b {binary_filename}] [-e {esstra_filename}] [-x]
+
+Scan the files listed in either the esstra data in the binary file specified
+(or list in the estra data file listed), and show whether they have
+SPDX-License-Identifier headers - indicating the license for that file.
+
+Options:
+  -h         Show this usage help
+  -b {file}  Specify a binary file to extract esstra data from to use
+  -e {file}  Specify an esstra data file to use for SPDX scanning
+  -x         Show only files missing an SPDX header
+HERE
+}
+
+if [ "$1" = "-h" ] ; then
+    usage
+    exit 0
+fi
+
+if [ "$1" = "-b" ] ; then
+    shift
+    binary_file="$1"
+    shift
+fi
+if [ "$1" = "-e" ] ; then
+    shift
+    esstra_file="$1"
+    shift
+fi
+
+if [ "$1" = "-x" ] ; then
+    X_ARG="-x"
+    shift
+fi
+
+#esstra show -r $binary_file | grep InputFileName | sed "s/  InputFileName: //" | has-spdx-id.py -c -
+#cat $input_file | grep InputFileName | sed "s/InputFileName: //" | has-spdx-id.sh -c -
+#
+
+if [ -n "$binary_file" ] ; then
+    # grab and sort all File: entries
+    esstra show -r $binary_file | esstra-full-paths | grep Path: | sed "s/- Path: //" | sort | uniq | has-spdx-id.py -c -
+fi
+
+if [ -n "$esstra_file" ] ; then
+    cat $esstra_file | esstra-full-paths | grep Path: | sed "s/- Path: //" | sort | uniq | has-spdx-id.py -c $X_ARG -
+fi
diff --git a/util/has-spdx-id.py b/util/has-spdx-id.py
new file mode 100755
index 0000000..ef4d92b
--- /dev/null
+++ b/util/has-spdx-id.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+# has-spdx-id.py - indicate whether the given file has an SPDX identifier line
+#  in the first 20 lines of the file
+#
+
+import sys
+import os
+
+def usage():
+    print("""Usage: has-spdx-id.sh [options] {filepath}"
+  Show whether a file (or files) has an SPDX-License-Identifier line,
+  in the first 20 lines of the file.
+
+  If {filepath} is '-', then read a list of filepaths to check from stdin.
+
+Shows:
+  1 if filepath has an SPDX-License-Identifier
+  0 otherwise
+
+The return code of the program is 0 if the last filepath scanned has
+an SPDX line, and 1 otherwise.  This means that if only one file is
+scanned you can use the program return status to indicate whether the
+filepath has and SPDX line or not.
+
+Options:
+  -h     Show this usage help
+  -q     Don't show the filepath along with the result (q=quiet)
+  -c     Show counts of files with and without SPDX lines
+  -x     Show only files without an SPDX line
+""")
+    sys.exit(0)
+
+def main():
+    # parse command line arguments
+    quiet = False
+    show_count = False
+    show_only_missing = False
+    if "-h" in sys.argv:
+        usage()
+    if "-q" in sys.argv:
+        quiet = True
+        sys.argv.remove("-q")
+    if "-c" in sys.argv:
+        show_count = True
+        sys.argv.remove("-c")
+    if "-x" in sys.argv:
+        show_only_missing = True
+        sys.argv.remove("-x")
+
+    if len(sys.argv)<1:
+        print("Error: missing argument")
+        usage()
+
+    filepaths = sys.argv[1:]
+
+    if "-" in filepaths:
+        pos = filepaths.index("-")
+        lines = sys.stdin.readlines()
+        filepaths[pos:pos+1] = [line.strip() for line in lines]
+
+    with_count = 0
+    without_count = 0
+    has_spdx = False
+
+    for filepath in filepaths:
+        if not os.path.isfile(filepath):
+            print(f"Warning: {filepath} is not a file")
+            continue
+
+        # read the first 20 lines of the file
+        lines = []
+        cur_file = open(filepath, "r")
+        try:
+            for i in range(20):
+                lines.append(cur_file.readline().strip())
+        except:
+            pass
+
+        cur_file.close()
+
+        has_spdx = False
+        for line in lines:
+            if "SPDX-License-Identifier:" in line:
+                # check for valid prefixes here:
+                if line[0:5] in ['//SPD','// SP', '/* SP']:
+                    has_spdx = True
+                    break
+
+        prefix = ""
+        if not quiet:
+           prefix = filepath + " - "
+        if has_spdx:
+            with_count += 1
+            if not show_only_missing:
+                print(prefix + "1")
+        else:
+            without_count += 1
+            print(prefix + "0")
+
+    if show_count:
+        print(f"Files with SPDX lines: {with_count}")
+        print(f"Files without SPDX lines: {without_count}")
+
+    if has_spdx:
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()