From 210aa84d871c0e74215d9a0cdb0b84de9b034756 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Thu, 21 Aug 2025 14:28:34 +1200 Subject: [PATCH 01/15] feat: replace `package-lock.json` parser --- __snapshots__/main_test.snap | 8 +- go.mod | 17 +- go.sum | 273 ++++++++++++++++++++++++++++++++- internal/reporter/reporter.go | 5 +- pkg/lockfile/extract.go | 153 ++++++++++++++++++ pkg/lockfile/parse-npm-lock.go | 143 +---------------- 6 files changed, 449 insertions(+), 150 deletions(-) create mode 100644 pkg/lockfile/extract.go diff --git a/__snapshots__/main_test.snap b/__snapshots__/main_test.snap index 63f41578..6fe85dbf 100755 --- a/__snapshots__/main_test.snap +++ b/__snapshots__/main_test.snap @@ -713,7 +713,7 @@ testdata/locks-empty/composer.lock: found 0 packages --- [TestRun_ParseAsGlobal/#02 - 2] -Error, could not parse testdata/locks-empty/Gemfile.lock: unexpected end of JSON input +Error, could not parse testdata/locks-empty/Gemfile.lock: EOF Error, could not parse testdata/locks-empty/yarn.lock: invalid character '#' looking for beginning of value --- @@ -743,7 +743,7 @@ testdata/locks-insecure/my-package-lock.json: found 1 package --- [TestRun_ParseAsGlobal/#03 - 2] -Error, could not parse testdata/locks-empty/Gemfile.lock: unexpected end of JSON input +Error, could not parse testdata/locks-empty/Gemfile.lock: EOF Error, could not parse testdata/locks-empty/yarn.lock: invalid character '#' looking for beginning of value --- @@ -857,7 +857,7 @@ testdata/locks-empty/composer.lock: found 0 packages --- [TestRun_ParseAsSpecific/#04 - 2] -Error, could not parse testdata/locks-empty/Gemfile.lock: unexpected end of JSON input +Error, could not parse testdata/locks-empty/Gemfile.lock: EOF Error, could not parse testdata/locks-empty/yarn.lock: invalid character '#' looking for beginning of value --- @@ -887,7 +887,7 @@ testdata/locks-insecure/my-package-lock.json: found 1 package --- [TestRun_ParseAsSpecific/#05 - 2] -Error, could not parse testdata/locks-empty/Gemfile.lock: unexpected end of JSON input +Error, could not parse testdata/locks-empty/Gemfile.lock: EOF Error, could not parse testdata/locks-empty/yarn.lock: invalid character '#' looking for beginning of value --- diff --git a/go.mod b/go.mod index 9218d367..650a2ba3 100644 --- a/go.mod +++ b/go.mod @@ -14,18 +14,33 @@ require ( ) require ( + github.com/CycloneDX/cyclonedx-go v0.9.2 // indirect + github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect github.com/gkampitakis/ciinfo v0.3.2 // indirect github.com/gkampitakis/go-diff v1.3.2 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.6.2 // indirect + github.com/go-git/go-git/v5 v5.16.2 // indirect + github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-yaml v1.18.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/maruel/natural v1.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/ossf/osv-schema/bindings/go v0.0.0-20250805051309-c463400aa925 // indirect + github.com/package-url/packageurl-go v0.1.3 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/spdx/tools-golang v0.5.5 // indirect github.com/tidwall/gjson v1.18.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect + golang.org/x/net v0.41.0 // indirect golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.26.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index 18cc98ab..75311309 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,134 @@ +cloud.google.com/go/compute/metadata v0.7.0 h1:PBWF+iiAerVNe8UCHxdOt6eHLVc3ydFeOCw78U8ytSU= +cloud.google.com/go/compute/metadata v0.7.0/go.mod h1:j5MvL9PprKL39t166CoB1uVHfQMs4tFQZZcKwksXUjo= +deps.dev/api/v3 v3.0.0-20250903005441-604c45d5b44b h1:4f6EeZ4EexJUGBtmyMaxptWMoBw/pAVshOtvzTH6dj8= +deps.dev/api/v3 v3.0.0-20250903005441-604c45d5b44b/go.mod h1:BWOjjNq4+j4makGArvrtyFhzBi5TXU7AGH2nDnRORk8= +deps.dev/api/v3alpha v0.0.0-20250903005441-604c45d5b44b h1:iXre7CzhkmdmzAdiOi+u/Yk1iDMI9SYlFEnXgJd5Rnk= +deps.dev/api/v3alpha v0.0.0-20250903005441-604c45d5b44b/go.mod h1:CJqVceLEA55Tu9QwNoaUX4HhvzRQnYjCL1jUdw/rhPQ= +deps.dev/util/maven v0.0.0-20250903005441-604c45d5b44b h1:rDwPZ29kX7RBvBWFwW7U72Gd/CIZQbb8/LxvYuW/ee4= +deps.dev/util/maven v0.0.0-20250903005441-604c45d5b44b/go.mod h1:eGrXziwI7scSGrwIj+5EBHtTeSxAZD/yi8Hb3nFXesA= +deps.dev/util/pypi v0.0.0-20250903005441-604c45d5b44b h1:67FfxwUt82PEMle2FKlW4DZvzcfSODDoTnSGOT1bYtY= +deps.dev/util/pypi v0.0.0-20250903005441-604c45d5b44b/go.mod h1:qmA0z/Lsfa1FMtuLd9JmVZLMHR3GBX/EmbM6z1X3EDU= +deps.dev/util/resolve v0.0.0-20250903005441-604c45d5b44b h1:Ha9MFfZZ3kJGK8T4RJjyvRNEaXlAvcDMqzEufu5obOI= +deps.dev/util/resolve v0.0.0-20250903005441-604c45d5b44b/go.mod h1:Wb3XFbME1HKmtOvcJ2kOdEiFos0BYX0CDZDWyGdLP+w= +deps.dev/util/semver v0.0.0-20250903005441-604c45d5b44b h1:qTuU4neoid9ft+AYtFZgnqXs8W6tJ38aoYnSamg3SKE= +deps.dev/util/semver v0.0.0-20250903005441-604c45d5b44b/go.mod h1:jjJweVqtuMQ7Q4zlTQ/kCHpboojkRvpMYlhy/c93DVU= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= +github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/CycloneDX/cyclonedx-go v0.9.2 h1:688QHn2X/5nRezKe2ueIVCt+NRqf7fl3AVQk+vaFcIo= +github.com/CycloneDX/cyclonedx-go v0.9.2/go.mod h1:vcK6pKgO1WanCdd61qx4bFnSsDJQ6SbM2ZuMIgq86Jg= +github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5 h1:IEjq88XO4PuBDcvmjQJcQGg+w+UaafSy8G5Kcb5tBhI= +github.com/GehirnInc/crypt v0.0.0-20230320061759-8cc1b52080c5/go.mod h1:exZ0C/1emQJAw5tHOaUDyY1ycttqBAPcxuzf7QbY6ec= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Microsoft/hcsshim v0.13.0 h1:/BcXOiS6Qi7N9XqUcv27vkIuVOkBEcWstd2pMlWSeaA= +github.com/Microsoft/hcsshim v0.13.0/go.mod h1:9KWJ/8DgU+QzYGupX4tzMhRQE8h6w90lH6HAaclpEok= +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/anchore/go-struct-converter v0.0.0-20221118182256-c68fdcfa2092/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= +github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 h1:6COpXWpHbhWM1wgcQN95TdsmrLTba8KQfPgImBXzkjA= +github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= +github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= +github.com/containerd/cgroups/v3 v3.0.5/go.mod h1:SA5DLYnXO8pTGYiAHXz94qvLQTKfVM5GEVisn4jpins= +github.com/containerd/containerd v1.7.23 h1:H2CClyUkmpKAGlhQp95g2WXHfLYc7whAuvZGBNYOOwQ= +github.com/containerd/containerd v1.7.23/go.mod h1:7QUzfURqZWCZV7RLNEn1XjUCQLEf0bkaK4GjUaZehxw= +github.com/containerd/containerd/api v1.9.0 h1:HZ/licowTRazus+wt9fM6r/9BQO7S0vD5lMcWspGIg0= +github.com/containerd/containerd/api v1.9.0/go.mod h1:GhghKFmTR3hNtyznBoQ0EMWr9ju5AqHjcZPsSpTKutI= +github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= +github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= +github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v1.0.0-rc.1 h1:83KIq4yy1erSRgOVHNk1HYdPvzdJ5CnsWaRoJX4C41E= +github.com/containerd/platforms v1.0.0-rc.1/go.mod h1:J71L7B+aiM5SdIEqmd9wp6THLVRzJGXfNuWCZCllLA4= +github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8= +github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU= +github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= +github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= +github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40= +github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deitch/magic v0.0.0-20240306090643-c67ab88f10cb h1:4W/2rQ3wzEimF5s+J6OY3ODiQtJZ5W1sForSgogVXkY= +github.com/deitch/magic v0.0.0-20240306090643-c67ab88f10cb/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v28.2.2+incompatible h1:qzx5BNUDFqlvyq4AHzdNB7gSyVTmU4cgsyN9SdInc1A= +github.com/docker/cli v28.2.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= +github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v28.2.2+incompatible h1:CjwRSksz8Yo4+RmQ339Dp/D2tGO5JxwYeqtMOEe0LDw= +github.com/docker/docker v28.2.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqIjCBS4wrsOh9yRqcz8= +github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= +github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= +github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= +github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= +github.com/erikvarga/go-rpmdb v0.0.0-20250523120114-a15a62cd4593 h1:cIQ/Ziclb/qreqg1nqGEtH4V9UJCTaNSKz9gBRaeZlA= +github.com/erikvarga/go-rpmdb v0.0.0-20250523120114-a15a62cd4593/go.mod h1:MiEorPk0IChAoCwpg2FXyqVgbNvOlPWZAYHqqIoDNoY= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= +github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= +github.com/go-git/go-git/v5 v5.16.2 h1:fT6ZIOjE5iEnkzKyxTHK1W4HGAsPhqEqiSAssSO77hM= +github.com/go-git/go-git/v5 v5.16.2/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-containerregistry v0.20.6 h1:cvWX87UxxLgaH76b4hIvya6Dzz9qHB31qAwjAohdSTU= +github.com/google/go-containerregistry v0.20.6/go.mod h1:T0x8MuoAoKX/873bkeSfLD2FAkwCDf9/HZgsFJ02E2Y= github.com/google/osv-scalibr v0.3.3-0.20250919162947-bcadd5b8f946 h1:b6158og59wx3u1CQnPwn4BUmt8PQrSND8aAJomM6304= github.com/google/osv-scalibr v0.3.3-0.20250919162947-bcadd5b8f946/go.mod h1:YeOH2wz0HlccjDbYYYTcX01ZyAuwqhZcpQFV7Cxsrwo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -26,10 +140,79 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/micromdm/plist v0.2.1 h1:4SoSMOVAyzv1ThT8IKLgXLJEKezLkcVDN6wivqTTFdo= +github.com/micromdm/plist v0.2.1/go.mod h1:flkfm0od6GzyXBqI28h5sgEyi3iPO28W2t1Zm9LpwWs= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/moby/buildkit v0.23.2 h1:gt/dkfcpgTXKx+B9I310kV767hhVqTvEyxGgI3mqsGQ= +github.com/moby/buildkit v0.23.2/go.mod h1:iEjAfPQKIuO+8y6OcInInvzqTMiKMbb2RdJz1K/95a0= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= +github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= +github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= +github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= +github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= +github.com/moby/sys/signal v0.7.1 h1:PrQxdvxcGijdo6UXXo/lU/TvHUWyPhj7UOpSo8tuvk0= +github.com/moby/sys/signal v0.7.1/go.mod h1:Se1VGehYokAkrSQwL4tDzHvETwUZlnY7S5XtQ50mQp8= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= +github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/opencontainers/runtime-spec v1.2.1 h1:S4k4ryNgEpxW1dzyqffOmhI1BHYcjzU8lpJfSlR0xww= +github.com/opencontainers/runtime-spec v1.2.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.12.0 h1:6n5JV4Cf+4y0KNXW48TLj5DwfXpvWlxXplUkdTrmPb8= +github.com/opencontainers/selinux v1.12.0/go.mod h1:BTPX+bjVbWGXw7ZZWUbdENt8w0htPSrlgOOysQaU62U= +github.com/ossf/osv-schema/bindings/go v0.0.0-20250805051309-c463400aa925 h1:+0cUosLHFxJHJwei+iA4aDesInPm0Xd4uCFud4Jopq8= +github.com/ossf/osv-schema/bindings/go v0.0.0-20250805051309-c463400aa925/go.mod h1:lILztSxHU7VsdlYqCnwgxSDBhbXMf7iEQWtldJCDXPo= +github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs= +github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rust-secure-code/go-rustaudit v0.0.0-20250226111315-e20ec32e963c h1:8gOLsYwaY2JwlTMT4brS5/9XJdrdIbmk2obvQ748CC0= +github.com/rust-secure-code/go-rustaudit v0.0.0-20250226111315-e20ec32e963c/go.mod h1:kwM/7r/rVluTE8qJbHAffduuqmSv4knVQT2IajGvSiA= +github.com/saferwall/pe v1.5.7 h1:fxlRLvhyr+3cIs1yturWhWmgACIu147o+xSEYFlUAyA= +github.com/saferwall/pe v1.5.7/go.mod h1:mJx+PuptmNpoPFBNhWs/uDMFL/kTHVZIkg0d4OUJFbQ= +github.com/secDre4mer/pkcs7 v0.0.0-20240322103146-665324a4461d h1:RQqyEogx5J6wPdoxqL132b100j8KjcVHO1c0KLRoIhc= +github.com/secDre4mer/pkcs7 v0.0.0-20240322103146-665324a4461d/go.mod h1:PegD7EVqlN88z7TpCqH92hHP+GBpfomGCCnw1PFtNOA= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spdx/gordf v0.0.0-20201111095634-7098f93598fb/go.mod h1:uKWaldnbMnjsSAXRurWqqrdyZen1R7kxl8TkmWk2OyM= +github.com/spdx/gordf v0.0.0-20221230105357-b735bd5aac89 h1:dArkMwZ7Mf2JiU8OfdmqIv8QaHT4oyifLIe1UhsF1SY= +github.com/spdx/gordf v0.0.0-20221230105357-b735bd5aac89/go.mod h1:uKWaldnbMnjsSAXRurWqqrdyZen1R7kxl8TkmWk2OyM= +github.com/spdx/tools-golang v0.5.5 h1:61c0KLfAcNqAjlg6UNMdkwpMernhw3zVRwDZ2x9XOmk= +github.com/spdx/tools-golang v0.5.5/go.mod h1:MVIsXx8ZZzaRWNQpUDhC4Dud34edUYJYecciXgrw5vE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/terminalstatic/go-xsd-validate v0.1.6 h1:TenYeQ3eY631qNi1/cTmLH/s2slHPRKTTHT+XSHkepo= +github.com/terminalstatic/go-xsd-validate v0.1.6/go.mod h1:18lsvYFofBflqCrvo1umpABZ99+GneNTw2kEEc8UPJw= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= @@ -42,13 +225,97 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/tink-crypto/tink-go/v2 v2.4.0 h1:8VPZeZI4EeZ8P/vB6SIkhlStrJfivTJn+cQ4dtyHNh0= +github.com/tink-crypto/tink-go/v2 v2.4.0/go.mod h1:l//evrF2Y3MjdbpNDNGnKgCpo5zSmvUvnQ4MU+yE2sw= +github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= +github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= +github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= +github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0 h1:2f304B10LaZdB8kkVEaoXvAMVan2tl9AiK4G0odjQtE= +github.com/tonistiigi/go-csvvalue v0.0.0-20240814133006-030d3b2625d0/go.mod h1:278M4p8WsNh3n4a1eqiFcV2FGk7wE5fwUpUom9mK9lE= +github.com/vbatts/tar-split v0.12.1 h1:CqKoORW7BUWBe7UL/iqTVvkTBOF8UvOMKOIZykxnnbo= +github.com/vbatts/tar-split v0.12.1/go.mod h1:eF6B6i6ftWQcDqEn3/iGFRFRo8cBIMSJVOpnNdfTMFA= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/bbolt v1.4.1 h1:5mOV+HWjIPLEAlUGMsveaUvK2+byZMFOzojoi7bh7uI= +go.etcd.io/bbolt v1.4.1/go.mod h1:c8zu2BnXWTu2XM4XcICtbGSl9cFwsXtcf9zLt2OncM8= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= +golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7 h1:FemxDzfMUcK2f3YY4H+05K9CDzbSVr2+q/JKN45pey0= +golang.org/x/telemetry v0.0.0-20240522233618-39ace7a40ae7/go.mod h1:pRgIJT+bRLFKnoM1ldnzKoxTIn14Yxz928LQRYYgIN0= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/vuln v1.1.4 h1:Ju8QsuyhX3Hk8ma3CesTbO8vfJD9EvUBgHvkxHBzj0I= +golang.org/x/vuln v1.1.4/go.mod h1:F+45wmU18ym/ca5PLTPLsSzr2KppzswxPP603ldA67s= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de h1:F6qOa9AZTYJXOUEr4jDysRDLrm4PHePlge4v4TGAlxY= +google.golang.org/genproto v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:VUhTRKeHn9wwcdrk73nvdC9gF178Tzhmt/qyaFcPLSo= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7 h1:FiusG7LWj+4byqhbvmB+Q93B/mOxJLN2DTozDuZm4EU= +google.golang.org/genproto/googleapis/api v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:kXqgZtrWaf6qS3jZOCnCH7WYfrvFjkC51bM8fz3RsCA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.65.10 h1:ZwEk8+jhW7qBjHIT+wd0d9VjitRyQef9BnzlzGwMODc= +modernc.org/libc v1.65.10/go.mod h1:StFvYpx7i/mXtBAfVOjaU0PWZOvIRoZSgXhrwXzr8Po= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.38.0 h1:+4OrfPQ8pxHKuWG4md1JpR/EYAh3Md7TdejuuzE7EUI= +modernc.org/sqlite v1.38.0/go.mod h1:1Bj+yES4SVvBZ4cBOpVZ6QgesMCKpJZDq0nxYzOpmNE= +osv.dev/bindings/go v0.0.0-20250808040635-c189436f8791 h1:UPwD9xNkZVS+WmfQv599Hd9HnD1Ph6MS3lPTInwmfV4= +osv.dev/bindings/go v0.0.0-20250808040635-c189436f8791/go.mod h1:U5TQCqNDZIFATIofBLLRrG+1SSBl6z/PjsXoExyaHJ0= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= +www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7 h1:BMX/37sYwX+8JhHt+YNbPfbx7dXG1w1L1mXonNBtjt0= +www.velocidex.com/golang/regparser v0.0.0-20250203141505-31e704a67ef7/go.mod h1:pxSECT5mWM3goJ4sxB4HCJNKnKqiAlpyT8XnvBwkLGU= diff --git a/internal/reporter/reporter.go b/internal/reporter/reporter.go index a3486455..22fa5a0a 100644 --- a/internal/reporter/reporter.go +++ b/internal/reporter/reporter.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "strings" "github.com/fatih/color" "github.com/g-rath/osv-detector/pkg/database" @@ -30,7 +31,9 @@ func New(stdout io.Writer, stderr io.Writer, outputAsJSON bool) *Reporter { // PrintErrorf writes the given message to stderr, regardless of if the reporter // is outputting as JSON or not func (r *Reporter) PrintErrorf(msg string, a ...any) { - fmt.Fprintf(r.stderr, msg, a...) + // todo: this is a hack to make the lockfile/extractor error output more like the original + // there's no real reason to be doing it other than that there isn't a reason not to... + fmt.Fprint(r.stderr, strings.Replace(fmt.Sprintf(msg, a...), " could not extract:", "", 1)) } // PrintTextf writes the given message to stdout, _unless_ the reporter is set diff --git a/pkg/lockfile/extract.go b/pkg/lockfile/extract.go new file mode 100644 index 00000000..6ef61573 --- /dev/null +++ b/pkg/lockfile/extract.go @@ -0,0 +1,153 @@ +package lockfile + +import ( + "cmp" + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + + "github.com/google/osv-scalibr/converter" + "github.com/google/osv-scalibr/extractor" + "github.com/google/osv-scalibr/extractor/filesystem" + scalibrfs "github.com/google/osv-scalibr/fs" +) + +func extract(localPath string, extractor filesystem.Extractor, ecosystem Ecosystem) ([]PackageDetails, error) { + info, err := os.Stat(localPath) + if err != nil { + return nil, fmt.Errorf("%w", err) + } + + invs, err := extractWithExtractor(context.Background(), localPath, info, extractor) + + if err != nil { + return nil, err + } + + return invToPackageDetails(invs, ecosystem), nil +} + +func extractWithExtractor(ctx context.Context, localPath string, info fs.FileInfo, ext filesystem.Extractor) ([]*extractor.Package, error) { + // Create a scan input centered at the system root directory, + // to give access to the full filesystem for each extractor. + rootDir := getRootDir(localPath) + si, err := createScanInput(localPath, rootDir, info) + if err != nil { + return nil, err + } + + invs, err := ext.Extract(ctx, si) + if err != nil { + return nil, fmt.Errorf("could not parse %s: %w", localPath, err) + } + + for i := range invs.Packages { + // Set parent extractor + invs.Packages[i].Plugins = append(invs.Packages[i].Plugins, ext.Name()) + + // Make Location relative to the scan root as we are performing local scanning + for i2 := range invs.Packages[i].Locations { + invs.Packages[i].Locations[i2] = filepath.Join(rootDir, invs.Packages[i].Locations[i2]) + } + } + + slices.SortFunc(invs.Packages, inventorySort) + invsCompact := slices.CompactFunc(invs.Packages, func(a, b *extractor.Package) bool { + return inventorySort(a, b) == 0 + }) + + return invsCompact, nil +} + +func createScanInput(path string, root string, fileInfo fs.FileInfo) (*filesystem.ScanInput, error) { + reader, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("%w", err) + } + + // Rel will strip root from the input path. + path, err = filepath.Rel(root, path) + if err != nil { + return nil, fmt.Errorf("%w", err) + } + + si := filesystem.ScanInput{ + FS: os.DirFS(root).(scalibrfs.FS), + Path: path, + Root: root, + Reader: reader, + Info: fileInfo, + } + + return &si, nil +} + +// getRootDir returns the root directory on each system. +// On Unix systems, it'll be / +// On Windows, it will most likely be the drive (e.g. C:\) +func getRootDir(path string) string { + if runtime.GOOS == "windows" { + return filepath.VolumeName(path) + "\\" + } + + if strings.HasPrefix(path, "/") { + return "/" + } + + return "" +} + +// InventorySort is a comparator function for Inventories, to be used in +// tests with cmp.Diff to disregard the order in which the Inventories +// are reported. +func inventorySort(a, b *extractor.Package) int { + aLoc := fmt.Sprintf("%v", a.Locations) + bLoc := fmt.Sprintf("%v", b.Locations) + + var aExtr, bExtr string + var aPURL, bPURL string + + aPURLStruct := converter.ToPURL(a) + bPURLStruct := converter.ToPURL(b) + + if aPURLStruct != nil { + aPURL = aPURLStruct.String() + } + + if bPURLStruct != nil { + bPURL = bPURLStruct.String() + } + + aSourceCode := fmt.Sprintf("%v", a.SourceCode) + bSourceCode := fmt.Sprintf("%v", b.SourceCode) + + return cmp.Or( + cmp.Compare(aLoc, bLoc), + cmp.Compare(a.Name, b.Name), + cmp.Compare(a.Version, b.Version), + cmp.Compare(aSourceCode, bSourceCode), + cmp.Compare(aExtr, bExtr), + cmp.Compare(aPURL, bPURL), + ) +} + +func invToPackageDetails(invs []*extractor.Package, ecosystem Ecosystem) []PackageDetails { + details := make([]PackageDetails, 0, len(invs)) + + for _, inv := range invs { + details = append(details, PackageDetails{ + Name: inv.Name, + Version: inv.Version, + Commit: inv.SourceCode.Commit, + Ecosystem: ecosystem, + CompareAs: ecosystem, + }) + } + + return details +} diff --git a/pkg/lockfile/parse-npm-lock.go b/pkg/lockfile/parse-npm-lock.go index 3f0234cc..c115f1dc 100644 --- a/pkg/lockfile/parse-npm-lock.go +++ b/pkg/lockfile/parse-npm-lock.go @@ -1,33 +1,9 @@ package lockfile import ( - "encoding/json" - "fmt" - "os" - "path" - "strings" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/packagelockjson" ) -type NpmLockDependency struct { - Version string `json:"version"` - Dependencies map[string]NpmLockDependency `json:"dependencies,omitempty"` -} - -type NpmLockPackage struct { - Name string `json:"name"` - Version string `json:"version"` - Resolved string `json:"resolved"` - Dependencies map[string]string `json:"dependencies"` -} - -type NpmLockfile struct { - Version int `json:"lockfileVersion"` - // npm v1- lockfiles use "dependencies" - Dependencies map[string]NpmLockDependency `json:"dependencies"` - // npm v2+ lockfiles use "packages" - Packages map[string]NpmLockPackage `json:"packages,omitempty"` -} - const NpmEcosystem Ecosystem = "npm" func pkgDetailsMapToSlice(m map[string]PackageDetails) []PackageDetails { @@ -54,121 +30,6 @@ func mergePkgDetailsMap(m1 map[string]PackageDetails, m2 map[string]PackageDetai return details } -func parseNpmLockDependencies(dependencies map[string]NpmLockDependency) map[string]PackageDetails { - details := map[string]PackageDetails{} - - for name, detail := range dependencies { - if detail.Dependencies != nil { - details = mergePkgDetailsMap(details, parseNpmLockDependencies(detail.Dependencies)) - } - - version := detail.Version - finalVersion := version - commit := "" - - // we can't resolve a version from a "file:" dependency - if strings.HasPrefix(detail.Version, "file:") { - finalVersion = "" - } else { - // use the name of the underlying package rather than the alias - if strings.HasPrefix(detail.Version, "npm:") { - i := strings.LastIndex(detail.Version, "@") - name = detail.Version[4:i] - finalVersion = detail.Version[i+1:] - } - - commit = tryExtractCommit(detail.Version) - - // if there is a commit, we want to deduplicate based on that rather than - // the version (the versions must match anyway for the commits to match) - // - // we also don't actually know what the "version" is, so blank it - if commit != "" { - finalVersion = "" - version = commit - } - } - - details[name+"@"+version] = PackageDetails{ - Name: name, - Version: finalVersion, - Ecosystem: NpmEcosystem, - CompareAs: NpmEcosystem, - Commit: commit, - } - } - - return details -} - -func extractNpmPackageName(name string) string { - maybeScope := path.Base(path.Dir(name)) - pkgName := path.Base(name) - - if strings.HasPrefix(maybeScope, "@") { - pkgName = maybeScope + "/" + pkgName - } - - return pkgName -} - -func parseNpmLockPackages(packages map[string]NpmLockPackage) map[string]PackageDetails { - details := map[string]PackageDetails{} - - for namePath, detail := range packages { - if namePath == "" { - continue - } - - finalName := detail.Name - if finalName == "" { - finalName = extractNpmPackageName(namePath) - } - - finalVersion := detail.Version - - commit := tryExtractCommit(detail.Resolved) - - // if there is a commit, we want to deduplicate based on that rather than - // the version (the versions must match anyway for the commits to match) - if commit != "" { - finalVersion = commit - } - - details[finalName+"@"+finalVersion] = PackageDetails{ - Name: finalName, - Version: detail.Version, - Ecosystem: NpmEcosystem, - CompareAs: NpmEcosystem, - Commit: commit, - } - } - - return details -} - -func parseNpmLock(lockfile NpmLockfile) map[string]PackageDetails { - if lockfile.Packages != nil { - return parseNpmLockPackages(lockfile.Packages) - } - - return parseNpmLockDependencies(lockfile.Dependencies) -} - func ParseNpmLock(pathToLockfile string) ([]PackageDetails, error) { - var parsedLockfile *NpmLockfile - - lockfileContents, err := os.ReadFile(pathToLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err) - } - - err = json.Unmarshal(lockfileContents, &parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err) - } - - return pkgDetailsMapToSlice(parseNpmLock(*parsedLockfile)), nil + return extract(pathToLockfile, packagelockjson.NewDefault(), NpmEcosystem) } From d1167884e89fc0df7dd053a3f02622dbae5dcb65 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:22:08 +1200 Subject: [PATCH 02/15] feat: replace `pnpm-lock.yaml` parser --- pkg/lockfile/parse-pnpm-lock.go | 209 +-------------------------- pkg/lockfile/parse-pnpm-lock_test.go | 2 +- 2 files changed, 3 insertions(+), 208 deletions(-) diff --git a/pkg/lockfile/parse-pnpm-lock.go b/pkg/lockfile/parse-pnpm-lock.go index 83720549..f06c012c 100644 --- a/pkg/lockfile/parse-pnpm-lock.go +++ b/pkg/lockfile/parse-pnpm-lock.go @@ -1,216 +1,11 @@ package lockfile import ( - "errors" - "fmt" - "os" - "strconv" - "strings" - - "github.com/g-rath/osv-detector/internal/cachedregexp" - "gopkg.in/yaml.v3" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/pnpmlock" ) -var errInvalidPackagePath = errors.New("invalid package path") - -type PnpmLockPackageResolution struct { - Tarball string `yaml:"tarball"` - Commit string `yaml:"commit"` - Repo string `yaml:"repo"` - Type string `yaml:"type"` -} - -type PnpmLockPackage struct { - Resolution PnpmLockPackageResolution `yaml:"resolution"` - Name string `yaml:"name"` - Version string `yaml:"version"` -} - -type PnpmLockfile struct { - Version float64 `yaml:"lockfileVersion"` - Packages map[string]PnpmLockPackage `yaml:"packages,omitempty"` -} - -type pnpmLockfileV6 struct { - Version string `yaml:"lockfileVersion"` - Packages map[string]PnpmLockPackage `yaml:"packages,omitempty"` -} - -func (l *PnpmLockfile) UnmarshalYAML(value *yaml.Node) error { - var lockfileV6 pnpmLockfileV6 - - if err := value.Decode(&lockfileV6); err != nil { - return fmt.Errorf("%w", err) - } - - parsedVersion, err := strconv.ParseFloat(lockfileV6.Version, 64) - - if err != nil { - return fmt.Errorf("%w", err) - } - - l.Version = parsedVersion - l.Packages = lockfileV6.Packages - - return nil -} - const PnpmEcosystem = NpmEcosystem -func startsWithNumber(str string) bool { - matcher := cachedregexp.MustCompile(`^\d`) - - return matcher.MatchString(str) -} - -// extractPnpmPackageNameAndVersion parses a dependency path, attempting to -// extract the name and version of the package it represents -func extractPnpmPackageNameAndVersion(dependencyPath string, lockfileVersion float64) (string, string, error) { - // file dependencies must always have a name property to be installed, - // and their dependency path never has the version encoded, so we can - // skip trying to extract either from their dependency path - if strings.HasPrefix(dependencyPath, "file:") { - return "", "", nil - } - - // v9.0 specifies the dependencies as @ rather than as a path - if lockfileVersion == 9.0 { - dependencyPath = strings.Trim(dependencyPath, "'") - dependencyPath, isScoped := strings.CutPrefix(dependencyPath, "@") - - name, version, _ := strings.Cut(dependencyPath, "@") - - if isScoped { - name = "@" + name - } - - return name, version, nil - } - - parts := strings.Split(dependencyPath, "/") - - if len(parts) == 1 { - return "", "", errInvalidPackagePath - } - - var name string - - parts = parts[1:] - - if strings.HasPrefix(parts[0], "@") { - name = strings.Join(parts[:2], "/") - parts = parts[2:] - } else { - name = parts[0] - parts = parts[1:] - } - - version := "" - - if len(parts) != 0 { - version = parts[0] - } - - if version == "" { - name, version = parseNameAtVersion(name) - } - - if version == "" || !startsWithNumber(version) { - return "", "", nil - } - - // peer dependencies in v5 lockfiles are attached to the end of the version - // with an "_", so we always want the first element if an "_" is present - version, _, _ = strings.Cut(version, "_") - - return name, version, nil -} - -func parseNameAtVersion(value string) (name string, version string) { - // look for pattern "name@version", where name is allowed to contain zero or more "@" - matches := cachedregexp.MustCompile(`^(.+)@([\w.-]+)(?:\(|$)`).FindStringSubmatch(value) - - if len(matches) != 3 { - return name, "" - } - - return matches[1], matches[2] -} - -func parsePnpmLock(lockfile PnpmLockfile) ([]PackageDetails, error) { - packages := make([]PackageDetails, 0, len(lockfile.Packages)) - - for s, pkg := range lockfile.Packages { - name, version, err := extractPnpmPackageNameAndVersion(s, lockfile.Version) - - if err != nil { - return nil, err - } - - // "name" is only present if it's not in the dependency path and takes - // priority over whatever name we think we've extracted (if any) - if pkg.Name != "" { - name = pkg.Name - } - - // "version" is only present if it's not in the dependency path and takes - // priority over whatever version we think we've extracted (if any) - if pkg.Version != "" { - version = pkg.Version - } - - if name == "" || version == "" { - continue - } - - commit := pkg.Resolution.Commit - - if strings.HasPrefix(pkg.Resolution.Tarball, "https://codeload.github.com") { - re := cachedregexp.MustCompile(`https://codeload\.github\.com(?:/[\w-.]+){2}/tar\.gz/(\w+)$`) - matched := re.FindStringSubmatch(pkg.Resolution.Tarball) - - if matched != nil { - commit = matched[1] - } - } - - packages = append(packages, PackageDetails{ - Name: name, - Version: version, - Ecosystem: PnpmEcosystem, - CompareAs: PnpmEcosystem, - Commit: commit, - }) - } - - return packages, nil -} - func ParsePnpmLock(pathToLockfile string) ([]PackageDetails, error) { - var parsedLockfile *PnpmLockfile - - lockfileContents, err := os.ReadFile(pathToLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err) - } - - err = yaml.Unmarshal(lockfileContents, &parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err) - } - - // this will happen if the file is empty - if parsedLockfile == nil { - parsedLockfile = &PnpmLockfile{} - } - - packageDetails, err := parsePnpmLock(*parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err) - } - - return packageDetails, nil + return extract(pathToLockfile, pnpmlock.New(), PnpmEcosystem) } diff --git a/pkg/lockfile/parse-pnpm-lock_test.go b/pkg/lockfile/parse-pnpm-lock_test.go index b28d118c..d2011371 100644 --- a/pkg/lockfile/parse-pnpm-lock_test.go +++ b/pkg/lockfile/parse-pnpm-lock_test.go @@ -624,7 +624,7 @@ func TestParsePnpmLock_InvalidPackagePath(t *testing.T) { packages, err := lockfile.ParsePnpmLock("testdata/pnpm/invalid-package-path.yaml") - expectErrContaining(t, err, "invalid package path") + expectErrContaining(t, err, "invalid dependency path") expectPackages(t, packages, []lockfile.PackageDetails{}) } From 0834af416a66a4105d2e7e5f8053d080643e363d Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:32:36 +1200 Subject: [PATCH 03/15] feat: replace `bun.lock` parser --- go.mod | 2 +- pkg/lockfile/parse-bun-lock.go | 78 +---------------------------- pkg/lockfile/parse-bun-lock_test.go | 19 ++++--- 3 files changed, 12 insertions(+), 87 deletions(-) diff --git a/go.mod b/go.mod index 650a2ba3..03178542 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/gkampitakis/go-snaps v0.5.15 github.com/google/go-cmp v0.7.0 github.com/google/osv-scalibr v0.3.3-0.20250919162947-bcadd5b8f946 - github.com/tidwall/jsonc v0.3.2 golang.org/x/mod v0.28.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -35,6 +34,7 @@ require ( github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/spdx/tools-golang v0.5.5 // indirect github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/jsonc v0.3.2 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect diff --git a/pkg/lockfile/parse-bun-lock.go b/pkg/lockfile/parse-bun-lock.go index e4857d84..40ccaacb 100644 --- a/pkg/lockfile/parse-bun-lock.go +++ b/pkg/lockfile/parse-bun-lock.go @@ -1,85 +1,11 @@ package lockfile import ( - "encoding/json" - "fmt" - "os" - "strings" - - "github.com/tidwall/jsonc" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/bunlock" ) -type BunLockfile struct { - Version int `json:"lockfileVersion"` - Packages map[string][]any `json:"packages"` -} - const BunEcosystem = NpmEcosystem -// structurePackageDetails returns the name, version, and commit of a package -// specified as a tuple in a bun.lock -func structurePackageDetails(pkg []any) (string, string, string) { - str, ok := pkg[0].(string) - - if !ok { - return "", "", "" - } - - str, isScoped := strings.CutPrefix(str, "@") - name, version, _ := strings.Cut(str, "@") - - if isScoped { - name = "@" + name - } - - version, commit, _ := strings.Cut(version, "#") - - // bun.lock does not track both the commit and version, - // so if we have a commit then we don't have a version - if commit != "" { - version = "" - } - - // file dependencies do not have a semantic version recorded - if strings.HasPrefix(version, "file:") { - version = "" - } - - return name, version, commit -} - func ParseBunLock(pathToLockfile string) ([]PackageDetails, error) { - var parsedLockfile *BunLockfile - - lockfileContents, err := os.ReadFile(pathToLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err) - } - - err = json.Unmarshal(jsonc.ToJSON(lockfileContents), &parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err) - } - - packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) - - for _, pkg := range parsedLockfile.Packages { - name, version, commit := structurePackageDetails(pkg) - - if name == "" && version == "" && commit == "" { - continue - } - - packages = append(packages, PackageDetails{ - Name: name, - Version: version, - Ecosystem: BunEcosystem, - CompareAs: BunEcosystem, - Commit: commit, - }) - } - - return packages, nil + return extract(pathToLockfile, bunlock.New(), BunEcosystem) } diff --git a/pkg/lockfile/parse-bun-lock_test.go b/pkg/lockfile/parse-bun-lock_test.go index ebe9c464..cc47c5d9 100644 --- a/pkg/lockfile/parse-bun-lock_test.go +++ b/pkg/lockfile/parse-bun-lock_test.go @@ -83,17 +83,16 @@ func TestParseBunLock_OnePackageBadTuple(t *testing.T) { packages, err := lockfile.ParseBunLock("testdata/bun/bad-tuple.json5") - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - + expectErrContaining(t, err, "could not extract 'wrappy-bad1'") + expectErrContaining(t, err, "could not extract 'wrappy-bad2'") expectPackages(t, packages, []lockfile.PackageDetails{ - { - Name: "wrappy", - Version: "1.0.2", - Ecosystem: lockfile.BunEcosystem, - CompareAs: lockfile.BunEcosystem, - }, + // todo: look into restoring this (its because extractWithExtractor returns just the err) + // { + // Name: "wrappy", + // Version: "1.0.2", + // Ecosystem: lockfile.BunEcosystem, + // CompareAs: lockfile.BunEcosystem, + // }, }) } From e231a9c02dc63a981ef9e32293c5087bceb3dc05 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:36:06 +1200 Subject: [PATCH 04/15] fix: don't assume source code info is always present --- pkg/lockfile/extract.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/lockfile/extract.go b/pkg/lockfile/extract.go index 6ef61573..d5244a3b 100644 --- a/pkg/lockfile/extract.go +++ b/pkg/lockfile/extract.go @@ -140,10 +140,16 @@ func invToPackageDetails(invs []*extractor.Package, ecosystem Ecosystem) []Packa details := make([]PackageDetails, 0, len(invs)) for _, inv := range invs { + commit := "" + + if inv.SourceCode != nil { + commit = inv.SourceCode.Commit + } + details = append(details, PackageDetails{ Name: inv.Name, Version: inv.Version, - Commit: inv.SourceCode.Commit, + Commit: commit, Ecosystem: ecosystem, CompareAs: ecosystem, }) From bdee0a4b06426b519e7f0ea94bd9babb97c70464 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:37:05 +1200 Subject: [PATCH 05/15] feat: replace `Cargo.toml` parser --- pkg/lockfile/parse-cargo-lock.go | 42 ++------------------------------ 1 file changed, 2 insertions(+), 40 deletions(-) diff --git a/pkg/lockfile/parse-cargo-lock.go b/pkg/lockfile/parse-cargo-lock.go index e6875709..61cae85d 100644 --- a/pkg/lockfile/parse-cargo-lock.go +++ b/pkg/lockfile/parse-cargo-lock.go @@ -1,49 +1,11 @@ package lockfile import ( - "fmt" - "os" - - "github.com/BurntSushi/toml" + "github.com/google/osv-scalibr/extractor/filesystem/language/rust/cargolock" ) -type CargoLockPackage struct { - Name string `toml:"name"` - Version string `toml:"version"` -} - -type CargoLockFile struct { - Version int `toml:"version"` - Packages []CargoLockPackage `toml:"package"` -} - const CargoEcosystem Ecosystem = "crates.io" func ParseCargoLock(pathToLockfile string) ([]PackageDetails, error) { - var parsedLockfile *CargoLockFile - - lockfileContents, err := os.ReadFile(pathToLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err) - } - - err = toml.Unmarshal(lockfileContents, &parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err) - } - - packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) - - for _, lockPackage := range parsedLockfile.Packages { - packages = append(packages, PackageDetails{ - Name: lockPackage.Name, - Version: lockPackage.Version, - Ecosystem: CargoEcosystem, - CompareAs: CargoEcosystem, - }) - } - - return packages, nil + return extract(pathToLockfile, cargolock.New(), CargoEcosystem) } From ed1599d555c63879ace00add3a68421d575abc5a Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:37:52 +1200 Subject: [PATCH 06/15] feat: replace `composer.lock` parser --- pkg/lockfile/parse-composer-lock.go | 60 +---------------------------- 1 file changed, 2 insertions(+), 58 deletions(-) diff --git a/pkg/lockfile/parse-composer-lock.go b/pkg/lockfile/parse-composer-lock.go index 6e9acf9f..0196d21b 100644 --- a/pkg/lockfile/parse-composer-lock.go +++ b/pkg/lockfile/parse-composer-lock.go @@ -1,67 +1,11 @@ package lockfile import ( - "encoding/json" - "fmt" - "os" + "github.com/google/osv-scalibr/extractor/filesystem/language/php/composerlock" ) -type ComposerPackage struct { - Name string `json:"name"` - Version string `json:"version"` - Dist struct { - Reference string `json:"reference"` - } `json:"dist"` -} - -type ComposerLock struct { - Packages []ComposerPackage `json:"packages"` - PackagesDev []ComposerPackage `json:"packages-dev"` -} - const ComposerEcosystem Ecosystem = "Packagist" func ParseComposerLock(pathToLockfile string) ([]PackageDetails, error) { - var parsedLockfile *ComposerLock - - lockfileContents, err := os.ReadFile(pathToLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err) - } - - err = json.Unmarshal(lockfileContents, &parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err) - } - - packages := make( - []PackageDetails, - 0, - // len cannot return negative numbers, but the types can't reflect that - uint64(len(parsedLockfile.Packages))+uint64(len(parsedLockfile.PackagesDev)), - ) - - for _, composerPackage := range parsedLockfile.Packages { - packages = append(packages, PackageDetails{ - Name: composerPackage.Name, - Version: composerPackage.Version, - Commit: composerPackage.Dist.Reference, - Ecosystem: ComposerEcosystem, - CompareAs: ComposerEcosystem, - }) - } - - for _, composerPackage := range parsedLockfile.PackagesDev { - packages = append(packages, PackageDetails{ - Name: composerPackage.Name, - Version: composerPackage.Version, - Commit: composerPackage.Dist.Reference, - Ecosystem: ComposerEcosystem, - CompareAs: ComposerEcosystem, - }) - } - - return packages, nil + return extract(pathToLockfile, composerlock.New(), ComposerEcosystem) } From 324ccc8049f935b945f906276e25d37e8108228e Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:42:16 +1200 Subject: [PATCH 07/15] feat: replace `yarn.lock` parser --- pkg/lockfile/parse-yarn-lock.go | 193 +------------------------------- 1 file changed, 2 insertions(+), 191 deletions(-) diff --git a/pkg/lockfile/parse-yarn-lock.go b/pkg/lockfile/parse-yarn-lock.go index ca96421c..5693b498 100644 --- a/pkg/lockfile/parse-yarn-lock.go +++ b/pkg/lockfile/parse-yarn-lock.go @@ -1,200 +1,11 @@ package lockfile import ( - "bufio" - "fmt" - "net/url" - "os" - "strings" - - "github.com/g-rath/osv-detector/internal/cachedregexp" + "github.com/google/osv-scalibr/extractor/filesystem/language/javascript/yarnlock" ) const YarnEcosystem = NpmEcosystem -func shouldSkipYarnLine(line string) bool { - return line == "" || strings.HasPrefix(line, "#") -} - -func groupPackageLines(scanner *bufio.Scanner) [][]string { - var groups [][]string - var group []string - - for scanner.Scan() { - line := scanner.Text() - - if shouldSkipYarnLine(line) { - continue - } - - // represents the start of a new dependency - if !strings.HasPrefix(line, " ") { - if len(group) > 0 { - groups = append(groups, group) - } - group = make([]string, 0) - } - - group = append(group, line) - } - - if len(group) > 0 { - groups = append(groups, group) - } - - return groups -} - -func extractYarnPackageName(str string) string { - str = strings.TrimPrefix(str, "\"") - str, _, _ = strings.Cut(str, ",") - str, isScoped := strings.CutPrefix(str, "@") - - name, right, _ := strings.Cut(str, "@") - - if strings.HasPrefix(right, "npm:") && strings.Contains(right, "@") { - return extractYarnPackageName(strings.TrimPrefix(right, "npm:")) - } - - if isScoped { - name = "@" + name - } - - return name -} - -func determineYarnPackageVersion(group []string) string { - re := cachedregexp.MustCompile(`^ {2}"?version"?:? "?([\w-.+]+)"?$`) - - for _, s := range group { - matched := re.FindStringSubmatch(s) - - if matched != nil { - return matched[1] - } - } - - // todo: decide what to do here - maybe panic...? - return "" -} - -func determineYarnPackageResolution(group []string) string { - re := cachedregexp.MustCompile(`^ {2}"?(?:resolution:|resolved)"? "([^ '"]+)"$`) - - for _, s := range group { - matched := re.FindStringSubmatch(s) - - if matched != nil { - return matched[1] - } - } - - // todo: decide what to do here - maybe panic...? - return "" -} - -func tryExtractCommit(resolution string) string { - // language=GoRegExp - matchers := []string{ - // ssh://... - // git://... - // git+ssh://... - // git+https://... - `(?:^|.+@)(?:git(?:\+(?:ssh|https))?|ssh)://.+#(\w+)$`, - // https://....git/... - `(?:^|.+@)https://.+\.git#(\w+)$`, - `https://codeload\.github\.com(?:/[\w-.]+){2}/tar\.gz/(\w+)$`, - `.+#commit[:=](\w+)$`, - // github:... - // gitlab:... - // bitbucket:... - `^(?:github|gitlab|bitbucket):.+#(\w+)$`, - } - - for _, matcher := range matchers { - re := cachedregexp.MustCompile(matcher) - matched := re.FindStringSubmatch(resolution) - - if matched != nil { - return matched[1] - } - } - - u, err := url.Parse(resolution) - - if err == nil { - gitRepoHosts := []string{ - "bitbucket.org", - "github.com", - "gitlab.com", - } - - for _, host := range gitRepoHosts { - if u.Host != host { - continue - } - - if u.RawQuery != "" { - queries := u.Query() - - if queries.Has("ref") { - return queries.Get("ref") - } - } - - return u.Fragment - } - } - - return "" -} - -func parsePackageGroup(group []string) PackageDetails { - name := extractYarnPackageName(group[0]) - version := determineYarnPackageVersion(group) - resolution := determineYarnPackageResolution(group) - - if version == "" { - _, _ = fmt.Fprintf( - os.Stderr, - "Failed to determine version of %s while parsing a yarn.lock - please report this!\n", - name, - ) - } - - return PackageDetails{ - Name: name, - Version: version, - Ecosystem: YarnEcosystem, - CompareAs: YarnEcosystem, - Commit: tryExtractCommit(resolution), - } -} - func ParseYarnLock(pathToLockfile string) ([]PackageDetails, error) { - file, err := os.Open(pathToLockfile) - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not open %s: %w", pathToLockfile, err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) - - packageGroups := groupPackageLines(scanner) - - if err := scanner.Err(); err != nil { - return []PackageDetails{}, fmt.Errorf("error while scanning %s: %w", pathToLockfile, err) - } - - packages := make([]PackageDetails, 0, len(packageGroups)) - - for _, group := range packageGroups { - if group[0] == "__metadata:" || strings.HasSuffix(group[0], "@workspace:.\":") { - continue - } - - packages = append(packages, parsePackageGroup(group)) - } - - return packages, nil + return extract(pathToLockfile, yarnlock.New(), YarnEcosystem) } From d01101bd0027dc66ade488b08e689b05bdc00123 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:48:29 +1200 Subject: [PATCH 08/15] feat: replace `poetry.lock` parser --- pkg/lockfile/parse-poetry-lock.go | 49 ++----------------------------- 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/pkg/lockfile/parse-poetry-lock.go b/pkg/lockfile/parse-poetry-lock.go index 5d3efa7d..1a01f217 100644 --- a/pkg/lockfile/parse-poetry-lock.go +++ b/pkg/lockfile/parse-poetry-lock.go @@ -1,56 +1,11 @@ package lockfile import ( - "fmt" - "os" - - "github.com/BurntSushi/toml" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/poetrylock" ) -type PoetryLockPackageSource struct { - Type string `toml:"type"` - Commit string `toml:"resolved_reference"` -} - -type PoetryLockPackage struct { - Name string `toml:"name"` - Version string `toml:"version"` - Source PoetryLockPackageSource `toml:"source"` -} - -type PoetryLockFile struct { - Version int `toml:"version"` - Packages []PoetryLockPackage `toml:"package"` -} - const PoetryEcosystem = PipEcosystem func ParsePoetryLock(pathToLockfile string) ([]PackageDetails, error) { - var parsedLockfile *PoetryLockFile - - lockfileContents, err := os.ReadFile(pathToLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err) - } - - err = toml.Unmarshal(lockfileContents, &parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err) - } - - packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) - - for _, lockPackage := range parsedLockfile.Packages { - packages = append(packages, PackageDetails{ - Name: lockPackage.Name, - Version: lockPackage.Version, - Commit: lockPackage.Source.Commit, - Ecosystem: PoetryEcosystem, - CompareAs: PoetryEcosystem, - }) - } - - return packages, nil + return extract(pathToLockfile, poetrylock.New(), PoetryEcosystem) } From ecf79491093d3716193ab1568675d76936a63c37 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:49:14 +1200 Subject: [PATCH 09/15] feat: replace `uv.lock` parser --- pkg/lockfile/parse-uv-lock.go | 65 ++--------------------------------- 1 file changed, 2 insertions(+), 63 deletions(-) diff --git a/pkg/lockfile/parse-uv-lock.go b/pkg/lockfile/parse-uv-lock.go index dbf7bbb6..61e4a427 100644 --- a/pkg/lockfile/parse-uv-lock.go +++ b/pkg/lockfile/parse-uv-lock.go @@ -1,72 +1,11 @@ package lockfile import ( - "fmt" - "os" - "strings" - - "github.com/BurntSushi/toml" + "github.com/google/osv-scalibr/extractor/filesystem/language/python/uvlock" ) -type UvLockPackageSource struct { - Virtual string `toml:"virtual"` - Git string `toml:"git"` -} - -type UvLockPackage struct { - Name string `toml:"name"` - Version string `toml:"version"` - Source UvLockPackageSource `toml:"source"` - - // uv stores "groups" as a table under "package" after all the packages, which due - // to how TOML works means it ends up being a property on the last package, even - // through in this context it's a global property rather than being per-package - Groups map[string][]UvOptionalDependency `toml:"optional-dependencies"` -} - -type UvOptionalDependency struct { - Name string `toml:"name"` -} -type UvLockFile struct { - Version int `toml:"version"` - Packages []UvLockPackage `toml:"package"` -} - const UvEcosystem = PipEcosystem func ParseUvLock(pathToLockfile string) ([]PackageDetails, error) { - var parsedLockfile *UvLockFile - - lockfileContents, err := os.ReadFile(pathToLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err) - } - - err = toml.Unmarshal(lockfileContents, &parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err) - } - - packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) - - for _, lockPackage := range parsedLockfile.Packages { - // skip including the root "package", since its name and version are most likely arbitrary - if lockPackage.Source.Virtual == "." { - continue - } - - _, commit, _ := strings.Cut(lockPackage.Source.Git, "#") - - packages = append(packages, PackageDetails{ - Name: lockPackage.Name, - Version: lockPackage.Version, - Ecosystem: UvEcosystem, - CompareAs: UvEcosystem, - Commit: commit, - }) - } - - return packages, nil + return extract(pathToLockfile, uvlock.New(), UvEcosystem) } From 0e8057d437072f8f80538f9a4f1f99f99cb9706a Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:49:51 +1200 Subject: [PATCH 10/15] feat: replace `Gemfile.lock` parser --- pkg/lockfile/parse-gemfile-lock.go | 172 +---------------------------- 1 file changed, 2 insertions(+), 170 deletions(-) diff --git a/pkg/lockfile/parse-gemfile-lock.go b/pkg/lockfile/parse-gemfile-lock.go index 5c7dc405..b3f81505 100644 --- a/pkg/lockfile/parse-gemfile-lock.go +++ b/pkg/lockfile/parse-gemfile-lock.go @@ -1,179 +1,11 @@ package lockfile import ( - "fmt" - "log" - "os" - "strings" - - "github.com/g-rath/osv-detector/internal/cachedregexp" + "github.com/google/osv-scalibr/extractor/filesystem/language/ruby/gemfilelock" ) const BundlerEcosystem Ecosystem = "RubyGems" -const lockfileSectionBUNDLED = "BUNDLED WITH" -const lockfileSectionDEPENDENCIES = "DEPENDENCIES" -const lockfileSectionPLATFORMS = "PLATFORMS" -const lockfileSectionRUBY = "RUBY VERSION" -const lockfileSectionGIT = "GIT" -const lockfileSectionGEM = "GEM" -const lockfileSectionPATH = "PATH" -const lockfileSectionPLUGIN = "PLUGIN SOURCE" - -type parserState string - -const parserStateSource parserState = "source" -const parserStateDependency parserState = "dependency" -const parserStatePlatform parserState = "platform" -const parserStateRuby parserState = "ruby" -const parserStateBundledWith parserState = "bundled_with" - -func isSourceSection(line string) bool { - return strings.Contains(line, lockfileSectionGIT) || - strings.Contains(line, lockfileSectionGEM) || - strings.Contains(line, lockfileSectionPATH) || - strings.Contains(line, lockfileSectionPLUGIN) -} - -type gemfileLockfileParser struct { - state parserState - dependencies []PackageDetails - bundlerVersion string - rubyVersion string - - // holds the commit of the gem that is currently being parsed, if found - currentGemCommit string -} - -func (parser *gemfileLockfileParser) addDependency(name string, version string) { - parser.dependencies = append(parser.dependencies, PackageDetails{ - Name: name, - Version: version, - Ecosystem: BundlerEcosystem, - CompareAs: BundlerEcosystem, - Commit: parser.currentGemCommit, - }) -} - -func (parser *gemfileLockfileParser) parseSpec(line string) { - // nameVersionReg := cachedregexp.MustCompile(`^( {2}| {4}| {6})(?! )(.*?)(?: \(([^-]*)(?:-(.*))?\))?(!)?$`) - nameVersionReg := cachedregexp.MustCompile(`^( +)(.*?)(?: \(([^-]*)(?:-(.*))?\))?(!)?$`) - - results := nameVersionReg.FindStringSubmatch(line) - - if results == nil { - return - } - - spaces := results[1] - - if spaces == "" { - log.Fatal("Weird error when parsing spec in Gemfile.lock (unexpectedly had no spaces) - please report this") - } - - if len(spaces) == 4 { - parser.addDependency(results[2], results[3]) - } -} - -func (parser *gemfileLockfileParser) parseSource(line string) { - if line == " specs" { - // todo: skip for now - return - } - - // OPTIONS = /^ ([a-z]+): (.*)$/i.freeze - optionsRegexp := cachedregexp.MustCompile(`(?i)^ {2}([a-z]+): (.*)$`) - - // todo: support - options := optionsRegexp.FindStringSubmatch(line) - - if options != nil { - commit := strings.TrimPrefix(options[0], " revision: ") - - // if the prefix was removed then the gem being parsed is git based, so - // we store the commit to be included later - if commit != options[0] { - parser.currentGemCommit = commit - } - - return - } - - // todo: source check - - parser.parseSpec(line) -} - -func isNotIndented(line string) bool { - re := cachedregexp.MustCompile(`^\S`) - - return re.MatchString(line) -} - -func (parser *gemfileLockfileParser) parseLineBasedOnState(line string) { - switch parser.state { - case parserStateDependency: - case parserStatePlatform: - break - case parserStateRuby: - parser.rubyVersion = strings.TrimSpace(line) - case parserStateBundledWith: - parser.bundlerVersion = strings.TrimSpace(line) - case parserStateSource: - parser.parseSource(line) - default: - log.Fatalf("Unknown supported '%s'\n", parser.state) - } -} - -func (parser *gemfileLockfileParser) parse(contents string) { - lineMatcher := cachedregexp.MustCompile(`(?:\r?\n)+`) - - lines := lineMatcher.Split(contents, -1) - - for _, line := range lines { - if isSourceSection(line) { - // clear the stateful package details, - // since we're now parsing a new group - parser.currentGemCommit = "" - parser.state = parserStateSource - parser.parseSource(line) - - continue - } - - switch line { - case lockfileSectionDEPENDENCIES: - parser.state = parserStateDependency - case lockfileSectionPLATFORMS: - parser.state = parserStatePlatform - case lockfileSectionRUBY: - parser.state = parserStateRuby - case lockfileSectionBUNDLED: - parser.state = parserStateBundledWith - default: - if isNotIndented(line) { - parser.state = "" - } - - if parser.state != "" { - parser.parseLineBasedOnState(line) - } - } - } -} - func ParseGemfileLock(pathToLockfile string) ([]PackageDetails, error) { - var parser gemfileLockfileParser - - bytes, err := os.ReadFile(pathToLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err) - } - - parser.parse(string(bytes)) - - return parser.dependencies, nil + return extract(pathToLockfile, gemfilelock.New(), BundlerEcosystem) } From 3bd0ebd8720a7870cf46ae502354a9e28b9a5677 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Fri, 22 Aug 2025 08:51:25 +1200 Subject: [PATCH 11/15] feat: replace `pubspec.lock` parser --- pkg/lockfile/parse-pubspec-lock.go | 86 +------------------------ pkg/lockfile/parse-pubspec-lock_test.go | 5 +- 2 files changed, 3 insertions(+), 88 deletions(-) diff --git a/pkg/lockfile/parse-pubspec-lock.go b/pkg/lockfile/parse-pubspec-lock.go index ef7df54b..6c4d70ad 100644 --- a/pkg/lockfile/parse-pubspec-lock.go +++ b/pkg/lockfile/parse-pubspec-lock.go @@ -1,93 +1,11 @@ package lockfile import ( - "fmt" - "os" - - "gopkg.in/yaml.v3" + "github.com/google/osv-scalibr/extractor/filesystem/language/dart/pubspec" ) -type PubspecLockDescription struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - Path string `yaml:"path"` - Ref string `yaml:"resolved-ref"` -} - -func (pld *PubspecLockDescription) UnmarshalYAML(value *yaml.Node) error { - var m struct { - Name string `yaml:"name"` - URL string `yaml:"url"` - Path string `yaml:"path"` - Ref string `yaml:"resolved-ref"` - } - - err := value.Decode(&m) - - if err == nil { - pld.Name = m.Name - pld.Path = m.Path - pld.URL = m.URL - pld.Ref = m.Ref - - return nil - } - - var str *string - - err = value.Decode(&str) - - if err != nil { - return fmt.Errorf("%w", err) - } - - pld.Path = *str - - return nil -} - -type PubspecLockPackage struct { - Source string `yaml:"source"` - Description PubspecLockDescription `yaml:"description"` - Version string `yaml:"version"` -} - -type PubspecLockfile struct { - Packages map[string]PubspecLockPackage `yaml:"packages,omitempty"` - Sdks map[string]string `yaml:"sdks"` -} - const PubEcosystem Ecosystem = "Pub" func ParsePubspecLock(pathToLockfile string) ([]PackageDetails, error) { - var parsedLockfile *PubspecLockfile - - lockfileContents, err := os.ReadFile(pathToLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not read %s: %w", pathToLockfile, err) - } - - err = yaml.Unmarshal(lockfileContents, &parsedLockfile) - - if err != nil { - return []PackageDetails{}, fmt.Errorf("could not parse %s: %w", pathToLockfile, err) - } - if parsedLockfile == nil { - return []PackageDetails{}, nil - } - - packages := make([]PackageDetails, 0, len(parsedLockfile.Packages)) - - for name, pkg := range parsedLockfile.Packages { - packages = append(packages, PackageDetails{ - Name: name, - Version: pkg.Version, - Commit: pkg.Description.Ref, - Ecosystem: PubEcosystem, - CompareAs: PubEcosystem, - }) - } - - return packages, nil + return extract(pathToLockfile, pubspec.New(), PubEcosystem) } diff --git a/pkg/lockfile/parse-pubspec-lock_test.go b/pkg/lockfile/parse-pubspec-lock_test.go index 9a0cbc53..9fe114eb 100644 --- a/pkg/lockfile/parse-pubspec-lock_test.go +++ b/pkg/lockfile/parse-pubspec-lock_test.go @@ -33,10 +33,7 @@ func TestParsePubspecLock_Empty(t *testing.T) { packages, err := lockfile.ParsePubspecLock("testdata/pub/empty.lock") - if err != nil { - t.Errorf("Got unexpected error: %v", err) - } - + expectErrContaining(t, err, "could not parse") expectPackages(t, packages, []lockfile.PackageDetails{}) } From 0b632660e7021d3a63ab95c3bd4f6bffbbaf6e38 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Wed, 3 Sep 2025 08:41:04 +1200 Subject: [PATCH 12/15] fix: use absolute path --- pkg/lockfile/extract.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/lockfile/extract.go b/pkg/lockfile/extract.go index d5244a3b..f0fed756 100644 --- a/pkg/lockfile/extract.go +++ b/pkg/lockfile/extract.go @@ -35,8 +35,13 @@ func extract(localPath string, extractor filesystem.Extractor, ecosystem Ecosyst func extractWithExtractor(ctx context.Context, localPath string, info fs.FileInfo, ext filesystem.Extractor) ([]*extractor.Package, error) { // Create a scan input centered at the system root directory, // to give access to the full filesystem for each extractor. - rootDir := getRootDir(localPath) - si, err := createScanInput(localPath, rootDir, info) + absPath, err := filepath.Abs(localPath) + if err != nil { + return nil, err + } + + rootDir := getRootDir(absPath) + si, err := createScanInput(absPath, rootDir, info) if err != nil { return nil, err } From 41bf9e65646e90706d93963daa160d620e621051 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Wed, 27 Aug 2025 13:55:36 +1200 Subject: [PATCH 13/15] feat: support new smart db --- main.go | 24 +- pkg/database/config.go | 2 + pkg/database/smart.go | 325 +++++++++++++++++++++++++++ pkg/database/smart_test.go | 436 +++++++++++++++++++++++++++++++++++++ pkg/database/zip_test.go | 17 ++ 5 files changed, 800 insertions(+), 4 deletions(-) create mode 100644 pkg/database/smart.go create mode 100644 pkg/database/smart_test.go diff --git a/main.go b/main.go index c0f04009..852b0912 100644 --- a/main.go +++ b/main.go @@ -33,10 +33,15 @@ func makeAPIDBConfig() database.Config { } } -func makeEcosystemDBConfig(ecosystem internal.Ecosystem) database.Config { +func makeEcosystemDBConfig(ecosystem internal.Ecosystem, beSmart bool) database.Config { + typ := "zip" + if beSmart { + typ = "smart" + } + return database.Config{ Name: string(ecosystem), - Type: "zip", + Type: typ, URL: fmt.Sprintf("https://osv-vulnerabilities.storage.googleapis.com/%s/all.zip", ecosystem), } } @@ -158,6 +163,15 @@ func describeDB(db database.DB) string { switch tt := db.(type) { case *database.APIDB: return "using batches of " + color.YellowString("%d", tt.BatchSize) + case *database.SmartDB: + count := tt.VulnerabilitiesCount + + return fmt.Sprintf( + "%s %s, including withdrawn - last updated %s", + color.YellowString("%d", count), + reporter.Form(count, "vulnerability", "vulnerabilities"), + tt.UpdatedAt, + ) case *database.ZipDB: count := tt.VulnerabilitiesCount @@ -372,6 +386,7 @@ func (files lockfileAndConfigOrErrs) adjustExtraDatabases( removeConfigDatabases bool, addDefaultAPIDatabase bool, addEcosystemDatabases bool, + beSmart bool, ) { for _, file := range files { if file.err != nil { @@ -391,7 +406,7 @@ func (files lockfileAndConfigOrErrs) adjustExtraDatabases( ecosystems := collectEcosystems([]lockfileAndConfigOrErr{file}) for _, ecosystem := range ecosystems { - extraDBConfigs = append(extraDBConfigs, makeEcosystemDBConfig(ecosystem)) + extraDBConfigs = append(extraDBConfigs, makeEcosystemDBConfig(ecosystem, beSmart)) } } @@ -508,6 +523,7 @@ func run(args []string, stdout, stderr io.Writer) int { useDatabases := cli.Bool("use-dbs", true, "Use the databases from osv.dev to check for known vulnerabilities") useAPI := cli.Bool("use-api", false, "Use the osv.dev API to check for known vulnerabilities") batchSize := cli.Int("batch-size", 1000, "The number of packages to include in each batch when using the api database") + beSmart := cli.Bool("be-smart", false, "Use smart database mode for faster incremental updates") cli.Var(&globalIgnores, "ignore", `ID of an OSV to ignore when determining exit codes. This flag can be passed multiple times to ignore different vulnerabilities`) @@ -589,7 +605,7 @@ This flag can be passed multiple times to ignore different vulnerabilities`) files := readAllLockfiles(r, pathsToLocksWithParseAs, cli.Args(), loadLocalConfig, &config) - files.adjustExtraDatabases(*noConfigDatabases, *useAPI, *useDatabases) + files.adjustExtraDatabases(*noConfigDatabases, *useAPI, *useDatabases, *beSmart) dbs, errored := loadDatabases( r, diff --git a/pkg/database/config.go b/pkg/database/config.go index 1ece9ef0..d295be50 100644 --- a/pkg/database/config.go +++ b/pkg/database/config.go @@ -29,6 +29,8 @@ var ErrUnsupportedDatabaseType = errors.New("unsupported database source type") // Load initializes a new OSV database based on the given Config func Load(config Config, offline bool, batchSize int) (DB, error) { switch config.Type { + case "smart": + return NewSmartDB(config, offline) case "zip": return NewZippedDB(config, offline) case "api": diff --git a/pkg/database/smart.go b/pkg/database/smart.go new file mode 100644 index 00000000..963001aa --- /dev/null +++ b/pkg/database/smart.go @@ -0,0 +1,325 @@ +package database + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/csv" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" +) + +type SmartDB struct { + // memDB + DirDB + + name string + identifier string + ArchiveURL string + WorkingDirectory string + Offline bool + UpdatedAt string +} + +func (db *SmartDB) Name() string { return db.name } +func (db *SmartDB) Identifier() string { return db.identifier } + +func (db *SmartDB) cachePath() string { + hash := sha256.Sum256([]byte(db.ArchiveURL)) + fileName := fmt.Sprintf("osv-detector-%x-db", hash) + + return filepath.Join(os.TempDir(), fileName) +} + +func (db *SmartDB) cacheFile(name string, content []byte) error { + //nolint:gosec // being world readable is fine + return os.WriteFile(filepath.Join(db.cachePath(), name), content, 0644) +} + +func (db *SmartDB) loadLastChecked() (time.Time, error) { + b, err := os.ReadFile(filepath.Join(db.cachePath(), "last_checked")) + + if err != nil { + return time.Time{}, err + } + + tim, err := time.Parse(time.RFC3339, string(b)) + + if err != nil { + return time.Time{}, err + } + + return tim, nil +} + +func (db *SmartDB) updateLastChecked(lastChecked time.Time) error { + db.UpdatedAt = lastChecked.Format(http.TimeFormat) + + return db.cacheFile("last_checked", []byte(lastChecked.Format(time.RFC3339))) +} + +func (db *SmartDB) writeZipFile(zipFile *zip.File) error { + dst, err := os.OpenFile(filepath.Join(db.cachePath(), zipFile.Name), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, zipFile.Mode()) + if err != nil { + return err + } + + defer dst.Close() + + z, err := zipFile.Open() + if err != nil { + return err + } + + defer z.Close() + + _, err = io.Copy(dst, z) + + return err +} + +func (db *SmartDB) populateFromZip() (*time.Time, error) { + err := os.MkdirAll(db.cachePath(), 0744) + + if err != nil { + return nil, err + } + + zdb := &ZipDB{ + name: db.name, + ArchiveURL: db.ArchiveURL, + WorkingDirectory: db.WorkingDirectory, + Offline: db.Offline, + } + + body, err := zdb.fetchZip() + + if err != nil { + return nil, err + } + + zipReader, err := zip.NewReader(bytes.NewReader(body), int64(len(body))) + if err != nil { + return nil, fmt.Errorf("could not read OSV database archive: %w", err) + } + + // Read each file from the archive and write it to the db directory + for _, zipFile := range zipReader.File { + if !strings.HasPrefix(zipFile.Name, db.WorkingDirectory) { + continue + } + + if !strings.HasSuffix(zipFile.Name, ".json") { + continue + } + + err = db.writeZipFile(zipFile) + + if err != nil { + return nil, err + } + } + + tim, err := time.Parse(http.TimeFormat, zdb.UpdatedAt) + + if err != nil { + return nil, err + } + + return &tim, nil +} + +type modifiedIDRow struct { + id string + modified time.Time +} + +func parseModifiedIDRow(columns []string) (*modifiedIDRow, error) { + modified, err := time.Parse(time.RFC3339, columns[0]) + + if err != nil { + return nil, err + } + + return &modifiedIDRow{id: columns[1], modified: modified}, nil +} + +func (db *SmartDB) fetchModifiedIDs(since time.Time) ([]modifiedIDRow, error) { + url := strings.TrimSuffix(db.ArchiveURL, "/all.zip") + "/modified_id.csv" + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + req.Header.Add("If-Modified-Since", since.Format(http.TimeFormat)) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotModified { + return []modifiedIDRow{}, nil + } else if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%w (%s)", ErrUnexpectedStatusCode, resp.Status) + } + + i := 0 + r := csv.NewReader(resp.Body) + + var rows []modifiedIDRow + + for { + i++ + record, err := r.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, fmt.Errorf("%w", err) + } + + row, err := parseModifiedIDRow(record) + if err != nil { + return nil, fmt.Errorf("row %d: %w", i, err) + } + + // the modified ids are sorted in reverse chronological order so once we hit + // a row that was modified before our "since" time, we can stop completely + if row.modified.Before(since) { + break + } + + rows = append(rows, *row) + } + + return rows, nil +} + +func (db *SmartDB) updateAdvisory(id string) error { + url := fmt.Sprintf("%s/%s.json", strings.TrimSuffix(db.ArchiveURL, "/all.zip"), id) + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) + if err != nil { + return err + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%w (%s)", ErrUnexpectedStatusCode, resp.Status) + } + + var body []byte + + body, err = io.ReadAll(resp.Body) + + if err != nil { + return err + } + + err = db.cacheFile(id+".json", body) + + return err +} + +func (db *SmartDB) updateModifiedAdvisories(since time.Time) error { + modifiedIDs, err := db.fetchModifiedIDs(since) + + if err != nil { + return err + } + + for _, row := range modifiedIDs { + err = db.updateAdvisory(row.id) + + if err != nil { + return err + } + } + + return nil +} + +func (db *SmartDB) populate() (*time.Time, error) { + lastChecked, err := db.loadLastChecked() + + // if there's an error, assumingly because the database does not already exist + // then extract it from a zip file, and use the zips updated date as the timestamp + if err != nil { + return db.populateFromZip() + } + + // if we're offline, then we can only work with the database on-hand, and don't + // want to change the last checked time + if db.Offline { + return &lastChecked, nil + } + + // otherwise, update all the advisories that have changed since our last check + err = db.updateModifiedAdvisories(lastChecked) + if err != nil { + return nil, err + } + + lastChecked = time.Now().UTC() + + return &lastChecked, nil +} + +// load fetches a zip archive of the OSV database and loads known vulnerabilities +// from it (which are assumed to be in json files following the OSV spec). +// +// Internally, the archive is cached along with the date that it was fetched +// so that a new version of the archive is only downloaded if it has been +// modified, per HTTP caching standards. +func (db *SmartDB) load() error { + lastChecked, err := db.populate() + + if err != nil { + return err + } + + if err = db.updateLastChecked(*lastChecked); err != nil { + return err + } + + db.DirDB = DirDB{ + name: db.name, + LocalPath: "file:///" + db.cachePath(), + WorkingDirectory: "", + Offline: db.Offline, + } + + return db.DirDB.load() +} + +func NewSmartDB(config Config, offline bool) (*SmartDB, error) { + db := &SmartDB{ + name: config.Name, + identifier: config.Identifier(), + ArchiveURL: config.URL, + WorkingDirectory: config.WorkingDirectory, + Offline: offline, + } + if err := db.load(); err != nil { + return nil, fmt.Errorf("unable to fetch OSV database: %w", err) + } + + return db, nil +} diff --git a/pkg/database/smart_test.go b/pkg/database/smart_test.go new file mode 100644 index 00000000..a54dbb0b --- /dev/null +++ b/pkg/database/smart_test.go @@ -0,0 +1,436 @@ +package database_test + +import ( + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/g-rath/osv-detector/pkg/database" +) + +func dirPath(url string) string { + hash := sha256.Sum256([]byte(url)) + fileName := fmt.Sprintf("osv-detector-%x-db", hash) + + return filepath.Join(os.TempDir(), fileName) +} + +func dirWrite(t *testing.T, url string, lastModified string, osvs map[string]database.OSV) { + t.Helper() + + err := os.Mkdir(dirPath(url), 0744) + + if err != nil { + t.Fatalf("unexpected error creating directory: %v", err) + } + + for _, osv := range osvs { + f, err := os.OpenFile(dirPath(url)+"/"+osv.ID+".json", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, os.ModePerm) + + if err != nil { + t.Fatalf("unexpected error creating file: %v", err) + } + + err = json.NewEncoder(f).Encode(osv) + if err != nil { + t.Fatalf("unexpected error encoding file: %v", err) + } + } + + err = os.WriteFile(dirPath(url)+"/last_checked", []byte(lastModified), 0600) + + if err != nil { + t.Fatalf("error creating last_checked file: %v", err) + } +} + +func expectLastCheckedTime(t *testing.T, url string, expected time.Time) { + t.Helper() + + expectedTime := expected.UTC().Format(time.RFC3339) + checkedTime, err := os.ReadFile(dirPath(url) + "/last_checked") + + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if string(checkedTime) != expectedTime { + t.Errorf("last checked time got = \"%s\", want = \"%s\"", string(checkedTime), expectedTime) + } +} + +func TestNewSmartDB_Offline_WithoutCache(t *testing.T) { + t.Parallel() + + ts := createZipServer(t, func(_ http.ResponseWriter, _ *http.Request) { + t.Errorf("a server request was made when running offline") + }) + + _, err := database.NewSmartDB(database.Config{URL: ts.URL}, true) + + if !errors.Is(err, database.ErrOfflineDatabaseNotFound) { + t.Errorf("expected \"%v\" error but got \"%v\"", database.ErrOfflineDatabaseNotFound, err) + } +} + +func TestNewSmartDB_Offline_WithZipCache(t *testing.T) { + t.Parallel() + + date := time.Now().UTC() + osvs := []database.OSV{ + withDefaultAffected("GHSA-1"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), + withDefaultAffected("GHSA-4"), + withDefaultAffected("GHSA-5"), + } + + ts := createZipServer(t, func(_ http.ResponseWriter, _ *http.Request) { + t.Errorf("a server request was made when running offline") + }) + + cacheWrite(t, database.Cache{ + URL: ts.URL, + ETag: "", + Date: date.Format(http.TimeFormat), + Body: zipOSVs(t, map[string]database.OSV{ + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), + "GHSA-4.json": withDefaultAffected("GHSA-4"), + "GHSA-5.json": withDefaultAffected("GHSA-5"), + }), + }) + + db, err := database.NewSmartDB(database.Config{URL: ts.URL}, true) + + if err != nil { + t.Fatalf("unexpected error \"%v\"", err) + } + + if db.UpdatedAt != date.Format(http.TimeFormat) { + t.Errorf("db.UpdatedAt got = \"%s\", want = \"%s\"", db.UpdatedAt, date) + } + + expectDBToHaveOSVs(t, db, osvs) + expectLastCheckedTime(t, ts.URL, date) +} + +func TestNewSmartDB_Offline_WithDirCache(t *testing.T) { + t.Parallel() + + date := time.Date(2022, time.June, 17, 22, 28, 13, 0, time.UTC) + osvs := []database.OSV{ + withDefaultAffected("GHSA-1"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), + withDefaultAffected("GHSA-4"), + withDefaultAffected("GHSA-5"), + } + + ts := createZipServer(t, func(_ http.ResponseWriter, _ *http.Request) { + t.Errorf("a server request was made when running offline") + }) + + cacheWrite(t, database.Cache{ + URL: ts.URL, + ETag: "", + Date: date.Format(http.TimeFormat), + Body: zipOSVs(t, map[string]database.OSV{ + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), + "GHSA-4.json": withDefaultAffected("GHSA-4"), + "GHSA-5.json": withDefaultAffected("GHSA-5"), + }), + }) + + db, err := database.NewSmartDB(database.Config{URL: ts.URL}, true) + + if err != nil { + t.Fatalf("unexpected error \"%v\"", err) + } + + if db.UpdatedAt != date.Format(http.TimeFormat) { + t.Errorf("db.UpdatedAt got = \"%s\", want = \"%s\"", db.UpdatedAt, date) + } + + expectDBToHaveOSVs(t, db, osvs) + + // when offline, the "last checked" time should not be changed + expectLastCheckedTime(t, ts.URL, date) +} + +func TestNewSmartDB_BadZip(t *testing.T) { + t.Parallel() + + ts := createZipServer(t, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("this is not a zip")) + }) + + _, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + + if err == nil { + t.Errorf("expected an error but did not get one") + } +} + +func TestNewSmartDB_UnsupportedProtocol(t *testing.T) { + t.Parallel() + + _, err := database.NewSmartDB(database.Config{URL: "file://hello-world"}, false) + + if err == nil { + t.Errorf("expected an error but did not get one") + } +} + +func TestNewSmartDB_Online_WithoutCache(t *testing.T) { + t.Parallel() + + osvs := []database.OSV{ + withDefaultAffected("GHSA-1"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), + withDefaultAffected("GHSA-4"), + withDefaultAffected("GHSA-5"), + } + + ts := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/modified_id.csv" { + t.Errorf("unexpected request for modified_id.csv") + + _, _ = w.Write([]byte("")) + + return + } + + _, _ = w.Write(zipOSVs(t, map[string]database.OSV{ + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), + "GHSA-4.json": withDefaultAffected("GHSA-4"), + "GHSA-5.json": withDefaultAffected("GHSA-5"), + })) + }) + + db, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + + if err != nil { + t.Fatalf("unexpected error \"%v\"", err) + } + + expectDBToHaveOSVs(t, db, osvs) +} + +func TestNewSmartDB_Online_WithoutCache_NotFound(t *testing.T) { + t.Parallel() + + ts := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/modified_id.csv" { + t.Errorf("unexpected request for modified_id.csv") + + _, _ = w.Write([]byte("")) + + return + } + + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write(zipOSVs(t, map[string]database.OSV{})) + }) + + _, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + + if err == nil { + t.Errorf("expected an error but did not get one") + } else if !errors.Is(err, database.ErrUnexpectedStatusCode) { + t.Errorf("expected %v error but got %v", database.ErrUnexpectedStatusCode, err) + } +} + +func toJSON(t *testing.T, v interface{}) []byte { + t.Helper() + + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + + return b +} + +func TestNewSmartDB_Online_WithExistingDirDB_UpToDate(t *testing.T) { + t.Parallel() + + date := time.Now().Add(-14 * time.Hour).UTC() + osvs := []database.OSV{ + withDefaultAffected("GHSA-1"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), + withDefaultAffected("GHSA-4"), + withDefaultAffected("GHSA-5"), + } + + ts := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/modified_id.csv" { + _, _ = w.Write([]byte(strings.Join([]string{ + date.Add(-02*time.Hour).Format(time.RFC3339) + ",GHSA-4", + date.Add(-38*time.Minute).Format(time.RFC3339) + ",GHSA-2", + date.Add(-06*time.Hour).Format(time.RFC3339) + ",GHSA-1", + date.Add(-03*time.Minute).Format(time.RFC3339) + ",GHSA-3", + date.Add(-14*time.Hour).Format(time.RFC3339) + ",GHSA-5", + }, "\n"))) + + return + } + + t.Errorf("unexpected request for %s", r.URL.Path) + + _, _ = w.Write([]byte("{}")) + }) + + dirWrite(t, ts.URL, date.Format(time.RFC3339), map[string]database.OSV{ + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), + "GHSA-4.json": withDefaultAffected("GHSA-4"), + "GHSA-5.json": withDefaultAffected("GHSA-5"), + }) + + db, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + + if err != nil { + t.Fatalf("unexpected error \"%v\"", err) + } + + if now := time.Now().UTC().Format(http.TimeFormat); db.UpdatedAt != now { + t.Errorf("db.UpdatedAt got = \"%s\", want = \"%s\"", db.UpdatedAt, now) + } + + expectDBToHaveOSVs(t, db, osvs) + + // when online, the "last checked" time should be now + expectLastCheckedTime(t, ts.URL, time.Now().UTC()) +} + +func TestNewSmartDB_Online_WithExistingDirDB_NotModified(t *testing.T) { + t.Parallel() + + date := time.Now().Add(-14 * time.Hour).UTC() + osvs := []database.OSV{ + withDefaultAffected("GHSA-1"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), + withDefaultAffected("GHSA-4"), + withDefaultAffected("GHSA-5"), + } + + ts := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/modified_id.csv" { + if dateHeader := r.Header.Get("If-Modified-Since"); dateHeader != date.Format(http.TimeFormat) { + t.Errorf("incorrect Date header: got = \"%s\", want = \"%s\"", dateHeader, date) + } + + w.WriteHeader(http.StatusNotModified) + + return + } + + t.Errorf("unexpected request for %s", r.URL.Path) + + _, _ = w.Write([]byte("{}")) + }) + + dirWrite(t, ts.URL, date.Format(time.RFC3339), map[string]database.OSV{ + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), + "GHSA-4.json": withDefaultAffected("GHSA-4"), + "GHSA-5.json": withDefaultAffected("GHSA-5"), + }) + + db, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + + if err != nil { + t.Fatalf("unexpected error \"%v\"", err) + } + + if now := time.Now().UTC().Format(http.TimeFormat); db.UpdatedAt != now { + t.Errorf("db.UpdatedAt got = \"%s\", want = \"%s\"", db.UpdatedAt, now) + } + + expectDBToHaveOSVs(t, db, osvs) + + // when online, the "last checked" time should be now + expectLastCheckedTime(t, ts.URL, time.Now().UTC()) +} + +func TestNewSmartDB_Online_WithExistingDirDB_Outdated(t *testing.T) { + t.Parallel() + + date := time.Now().Add(-14 * time.Hour).UTC() + osvs := []database.OSV{ + withSummary("GHSA-1", "this summary has changed"), + withDefaultAffected("GHSA-2"), + withDefaultAffected("GHSA-3"), + withSummary("GHSA-4", "and so has this one!"), + withDefaultAffected("GHSA-5"), + } + + ts := createZipServer(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/modified_id.csv": + _, _ = w.Write([]byte(strings.Join([]string{ + date.Add(+02*time.Hour).Format(time.RFC3339) + ",GHSA-4", + date.Add(+38*time.Minute).Format(time.RFC3339) + ",GHSA-2", + date.Add(+06*time.Hour).Format(time.RFC3339) + ",GHSA-1", + date.Add(-03*time.Minute).Format(time.RFC3339) + ",GHSA-3", + date.Add(-14*time.Hour).Format(time.RFC3339) + ",GHSA-5", + }, "\n"))) + + return + case "/GHSA-1.json": + _, _ = w.Write(toJSON(t, withSummary("GHSA-1", "this summary has changed"))) + case "/GHSA-2.json": + _, _ = w.Write(toJSON(t, withDefaultAffected("GHSA-2"))) + case "/GHSA-3.json": + t.Errorf("unexpected request for GHSA-3.json") + _, _ = w.Write(toJSON(t, withDefaultAffected("GHSA-3"))) + case "/GHSA-4.json": + _, _ = w.Write(toJSON(t, withSummary("GHSA-4", "and so has this one!"))) + case "/GHSA-5.json": + t.Errorf("unexpected request for GHSA-5.json") + _, _ = w.Write(toJSON(t, withDefaultAffected("GHSA-5"))) + } + }) + + dirWrite(t, ts.URL, date.Format(time.RFC3339), map[string]database.OSV{ + "GHSA-1.json": withDefaultAffected("GHSA-1"), + "GHSA-2.json": withDefaultAffected("GHSA-2"), + "GHSA-3.json": withDefaultAffected("GHSA-3"), + "GHSA-4.json": withDefaultAffected("GHSA-4"), + "GHSA-5.json": withDefaultAffected("GHSA-5"), + }) + + db, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + + if err != nil { + t.Fatalf("unexpected error \"%v\"", err) + } + + if now := time.Now().UTC().Format(http.TimeFormat); db.UpdatedAt != now { + t.Errorf("db.UpdatedAt got = \"%s\", want = \"%s\"", db.UpdatedAt, now) + } + + expectDBToHaveOSVs(t, db, osvs) + + // when online, the "last checked" time should be now + expectLastCheckedTime(t, ts.URL, time.Now().UTC()) +} diff --git a/pkg/database/zip_test.go b/pkg/database/zip_test.go index b7278bb3..df448a96 100644 --- a/pkg/database/zip_test.go +++ b/pkg/database/zip_test.go @@ -34,6 +34,22 @@ func withDefaultAffected(id string) database.OSV { } } +func withSummary(id string, summary string) database.OSV { + return database.OSV{ + ID: id, + Summary: summary, + Affected: []database.Affected{ + { + Package: database.Package{ + Name: "mine", + Ecosystem: "PyPi", + }, + Versions: database.Versions{}, + }, + }, + } +} + func expectDBToHaveOSVs( t *testing.T, db interface { @@ -99,6 +115,7 @@ func createZipServer(t *testing.T, handler http.HandlerFunc) *httptest.Server { ts.Close() _ = os.Remove(cachePath(ts.URL)) + _ = os.RemoveAll(dirPath(ts.URL)) }) return ts From 662bc76da6bbc5d0c4125f1f23f22f1460ef00f4 Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Wed, 3 Sep 2025 13:49:18 +1200 Subject: [PATCH 14/15] feat: store databases in user cache directory by default --- pkg/database/config.go | 2 ++ pkg/database/smart.go | 16 +++++++++++++++- pkg/database/smart_test.go | 20 ++++++++++---------- pkg/database/zip.go | 34 +++++++++++++++++++++++++++++++++- pkg/database/zip_test.go | 24 ++++++++++++------------ 5 files changed, 72 insertions(+), 24 deletions(-) diff --git a/pkg/database/config.go b/pkg/database/config.go index d295be50..4eaf3cbc 100644 --- a/pkg/database/config.go +++ b/pkg/database/config.go @@ -10,6 +10,8 @@ type Config struct { Type string `yaml:"type"` URL string `yaml:"url"` WorkingDirectory string `yaml:"working-directory"` + + CacheDirectory string `yaml:"-"` } // Identifier returns a unique string that can be used to check if a loaded diff --git a/pkg/database/smart.go b/pkg/database/smart.go index 963001aa..d451f71f 100644 --- a/pkg/database/smart.go +++ b/pkg/database/smart.go @@ -26,6 +26,8 @@ type SmartDB struct { WorkingDirectory string Offline bool UpdatedAt string + + cacheDirectory string } func (db *SmartDB) Name() string { return db.name } @@ -35,7 +37,7 @@ func (db *SmartDB) cachePath() string { hash := sha256.Sum256([]byte(db.ArchiveURL)) fileName := fmt.Sprintf("osv-detector-%x-db", hash) - return filepath.Join(os.TempDir(), fileName) + return filepath.Join(db.cacheDirectory, fileName) } func (db *SmartDB) cacheFile(name string, content []byte) error { @@ -96,6 +98,7 @@ func (db *SmartDB) populateFromZip() (*time.Time, error) { name: db.name, ArchiveURL: db.ArchiveURL, WorkingDirectory: db.WorkingDirectory, + cacheDirectory: db.cacheDirectory, Offline: db.Offline, } @@ -310,11 +313,22 @@ func (db *SmartDB) load() error { } func NewSmartDB(config Config, offline bool) (*SmartDB, error) { + if config.CacheDirectory == "" { + d, err := setupCacheDirectory() + + if err != nil { + return nil, err + } + + config.CacheDirectory = d + } + db := &SmartDB{ name: config.Name, identifier: config.Identifier(), ArchiveURL: config.URL, WorkingDirectory: config.WorkingDirectory, + cacheDirectory: config.CacheDirectory, Offline: offline, } if err := db.load(); err != nil { diff --git a/pkg/database/smart_test.go b/pkg/database/smart_test.go index a54dbb0b..2152bb52 100644 --- a/pkg/database/smart_test.go +++ b/pkg/database/smart_test.go @@ -73,7 +73,7 @@ func TestNewSmartDB_Offline_WithoutCache(t *testing.T) { t.Errorf("a server request was made when running offline") }) - _, err := database.NewSmartDB(database.Config{URL: ts.URL}, true) + _, err := database.NewSmartDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, true) if !errors.Is(err, database.ErrOfflineDatabaseNotFound) { t.Errorf("expected \"%v\" error but got \"%v\"", database.ErrOfflineDatabaseNotFound, err) @@ -109,7 +109,7 @@ func TestNewSmartDB_Offline_WithZipCache(t *testing.T) { }), }) - db, err := database.NewSmartDB(database.Config{URL: ts.URL}, true) + db, err := database.NewSmartDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, true) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -152,7 +152,7 @@ func TestNewSmartDB_Offline_WithDirCache(t *testing.T) { }), }) - db, err := database.NewSmartDB(database.Config{URL: ts.URL}, true) + db, err := database.NewSmartDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, true) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -175,7 +175,7 @@ func TestNewSmartDB_BadZip(t *testing.T) { _, _ = w.Write([]byte("this is not a zip")) }) - _, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + _, err := database.NewSmartDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err == nil { t.Errorf("expected an error but did not get one") @@ -185,7 +185,7 @@ func TestNewSmartDB_BadZip(t *testing.T) { func TestNewSmartDB_UnsupportedProtocol(t *testing.T) { t.Parallel() - _, err := database.NewSmartDB(database.Config{URL: "file://hello-world"}, false) + _, err := database.NewSmartDB(database.Config{CacheDirectory: os.TempDir(), URL: "file://hello-world"}, false) if err == nil { t.Errorf("expected an error but did not get one") @@ -221,7 +221,7 @@ func TestNewSmartDB_Online_WithoutCache(t *testing.T) { })) }) - db, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + db, err := database.NewSmartDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -246,7 +246,7 @@ func TestNewSmartDB_Online_WithoutCache_NotFound(t *testing.T) { _, _ = w.Write(zipOSVs(t, map[string]database.OSV{})) }) - _, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + _, err := database.NewSmartDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err == nil { t.Errorf("expected an error but did not get one") @@ -304,7 +304,7 @@ func TestNewSmartDB_Online_WithExistingDirDB_UpToDate(t *testing.T) { "GHSA-5.json": withDefaultAffected("GHSA-5"), }) - db, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + db, err := database.NewSmartDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -356,7 +356,7 @@ func TestNewSmartDB_Online_WithExistingDirDB_NotModified(t *testing.T) { "GHSA-5.json": withDefaultAffected("GHSA-5"), }) - db, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + db, err := database.NewSmartDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -419,7 +419,7 @@ func TestNewSmartDB_Online_WithExistingDirDB_Outdated(t *testing.T) { "GHSA-5.json": withDefaultAffected("GHSA-5"), }) - db, err := database.NewSmartDB(database.Config{URL: ts.URL}, false) + db, err := database.NewSmartDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err != nil { t.Fatalf("unexpected error \"%v\"", err) diff --git a/pkg/database/zip.go b/pkg/database/zip.go index 79ee555c..5a22cb37 100644 --- a/pkg/database/zip.go +++ b/pkg/database/zip.go @@ -11,6 +11,7 @@ import ( "io" "net/http" "os" + "path" "path/filepath" "strings" ) @@ -24,6 +25,7 @@ type ZipDB struct { WorkingDirectory string Offline bool UpdatedAt string + cacheDirectory string } func (db *ZipDB) Name() string { return db.name } @@ -44,7 +46,7 @@ func (db *ZipDB) cachePath() string { hash := sha256.Sum256([]byte(db.ArchiveURL)) fileName := fmt.Sprintf("osv-detector-%x-db.json", hash) - return filepath.Join(os.TempDir(), fileName) + return filepath.Join(db.cacheDirectory, fileName) } func (db *ZipDB) fetchZip() ([]byte, error) { @@ -192,11 +194,22 @@ func (db *ZipDB) load() error { } func NewZippedDB(config Config, offline bool) (*ZipDB, error) { + if config.CacheDirectory == "" { + d, err := setupCacheDirectory() + + if err != nil { + return nil, err + } + + config.CacheDirectory = d + } + db := &ZipDB{ name: config.Name, identifier: config.Identifier(), ArchiveURL: config.URL, WorkingDirectory: config.WorkingDirectory, + cacheDirectory: config.CacheDirectory, Offline: offline, } if err := db.load(); err != nil { @@ -205,3 +218,22 @@ func NewZippedDB(config Config, offline bool) (*ZipDB, error) { return db, nil } + +// setupCacheDirectory attempts to set up the directory the detector should use +// to cache things like local databases, attempting to use the user cache directory +// if possible or otherwise falling back to the temp directory +func setupCacheDirectory() (string, error) { + localDBPath, err := os.UserCacheDir() + + if err != nil { + localDBPath = os.TempDir() + } + + altPath := path.Join(localDBPath, "osv-detector") + err = os.MkdirAll(altPath, 0750) + if err == nil { + return altPath, nil + } + + return "", err +} diff --git a/pkg/database/zip_test.go b/pkg/database/zip_test.go index df448a96..c3330603 100644 --- a/pkg/database/zip_test.go +++ b/pkg/database/zip_test.go @@ -36,7 +36,7 @@ func withDefaultAffected(id string) database.OSV { func withSummary(id string, summary string) database.OSV { return database.OSV{ - ID: id, + ID: id, Summary: summary, Affected: []database.Affected{ { @@ -157,7 +157,7 @@ func TestNewZippedDB_Offline_WithoutCache(t *testing.T) { t.Errorf("a server request was made when running offline") }) - _, err := database.NewZippedDB(database.Config{URL: ts.URL}, true) + _, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, true) if !errors.Is(err, database.ErrOfflineDatabaseNotFound) { t.Errorf("expected \"%v\" error but got \"%v\"", database.ErrOfflineDatabaseNotFound, err) @@ -193,7 +193,7 @@ func TestNewZippedDB_Offline_WithCache(t *testing.T) { }), }) - db, err := database.NewZippedDB(database.Config{URL: ts.URL}, true) + db, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, true) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -213,7 +213,7 @@ func TestNewZippedDB_BadZip(t *testing.T) { _, _ = w.Write([]byte("this is not a zip")) }) - _, err := database.NewZippedDB(database.Config{URL: ts.URL}, false) + _, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err == nil { t.Errorf("expected an error but did not get one") @@ -223,7 +223,7 @@ func TestNewZippedDB_BadZip(t *testing.T) { func TestNewZippedDB_UnsupportedProtocol(t *testing.T) { t.Parallel() - _, err := database.NewZippedDB(database.Config{URL: "file://hello-world"}, false) + _, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: "file://hello-world"}, false) if err == nil { t.Errorf("expected an error but did not get one") @@ -251,7 +251,7 @@ func TestNewZippedDB_Online_WithoutCache(t *testing.T) { })) }) - db, err := database.NewZippedDB(database.Config{URL: ts.URL}, false) + db, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -268,7 +268,7 @@ func TestNewZippedDB_Online_WithoutCache_NotFound(t *testing.T) { _, _ = w.Write(zipOSVs(t, map[string]database.OSV{})) }) - _, err := database.NewZippedDB(database.Config{URL: ts.URL}, false) + _, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err == nil { t.Errorf("expected an error but did not get one") @@ -306,7 +306,7 @@ func TestNewZippedDB_Online_WithCache(t *testing.T) { }), }) - db, err := database.NewZippedDB(database.Config{URL: ts.URL}, false) + db, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -357,7 +357,7 @@ func TestNewZippedDB_Online_WithOldCache(t *testing.T) { }), }) - db, err := database.NewZippedDB(database.Config{URL: ts.URL}, false) + db, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -389,7 +389,7 @@ func TestNewZippedDB_Online_WithBadCache(t *testing.T) { cacheWriteBad(t, ts.URL, "this is not json!") - db, err := database.NewZippedDB(database.Config{URL: ts.URL}, false) + db, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -413,7 +413,7 @@ func TestNewZippedDB_FileChecks(t *testing.T) { })) }) - db, err := database.NewZippedDB(database.Config{URL: ts.URL}, false) + db, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL}, false) if err != nil { t.Fatalf("unexpected error \"%v\"", err) @@ -435,7 +435,7 @@ func TestNewZippedDB_WorkingDirectory(t *testing.T) { })) }) - db, err := database.NewZippedDB(database.Config{URL: ts.URL, WorkingDirectory: "reviewed"}, false) + db, err := database.NewZippedDB(database.Config{CacheDirectory: os.TempDir(), URL: ts.URL, WorkingDirectory: "reviewed"}, false) if err != nil { t.Fatalf("unexpected error \"%v\"", err) From 54da9dd0f6de3623e23ae5ad6a3a0c62d5064bcc Mon Sep 17 00:00:00 2001 From: Gareth Jones <3151613+G-Rath@users.noreply.github.com> Date: Mon, 8 Sep 2025 11:23:15 +1200 Subject: [PATCH 15/15] perf: fetch modified advisories in parallel --- pkg/database/smart.go | 70 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 56 insertions(+), 14 deletions(-) diff --git a/pkg/database/smart.go b/pkg/database/smart.go index d451f71f..a299e508 100644 --- a/pkg/database/smart.go +++ b/pkg/database/smart.go @@ -154,7 +154,7 @@ func parseModifiedIDRow(columns []string) (*modifiedIDRow, error) { return &modifiedIDRow{id: columns[1], modified: modified}, nil } -func (db *SmartDB) fetchModifiedIDs(since time.Time) ([]modifiedIDRow, error) { +func (db *SmartDB) fetchModifiedIDs(since time.Time) ([]string, error) { url := strings.TrimSuffix(db.ArchiveURL, "/all.zip") + "/modified_id.csv" req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil) @@ -172,7 +172,7 @@ func (db *SmartDB) fetchModifiedIDs(since time.Time) ([]modifiedIDRow, error) { defer resp.Body.Close() if resp.StatusCode == http.StatusNotModified { - return []modifiedIDRow{}, nil + return nil, nil } else if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("%w (%s)", ErrUnexpectedStatusCode, resp.Status) } @@ -180,7 +180,7 @@ func (db *SmartDB) fetchModifiedIDs(since time.Time) ([]modifiedIDRow, error) { i := 0 r := csv.NewReader(resp.Body) - var rows []modifiedIDRow + var ids []string for { i++ @@ -203,10 +203,10 @@ func (db *SmartDB) fetchModifiedIDs(since time.Time) ([]modifiedIDRow, error) { break } - rows = append(rows, *row) + ids = append(ids, row.id) } - return rows, nil + return ids, nil } func (db *SmartDB) updateAdvisory(id string) error { @@ -241,22 +241,64 @@ func (db *SmartDB) updateAdvisory(id string) error { return err } -func (db *SmartDB) updateModifiedAdvisories(since time.Time) error { - modifiedIDs, err := db.fetchModifiedIDs(since) +func (db *SmartDB) downloadModifiedAdvisories(ids []string) error { + conLimit := 200 - if err != nil { - return err + if len(ids) == 0 { + return nil + } + + // buffered channel which controls the number of concurrent operations + semaphoreChan := make(chan struct{}, conLimit) + resultsChan := make(chan *result) + + defer func() { + close(semaphoreChan) + close(resultsChan) + }() + + for i, id := range ids { + go func(i int, id string) { + // read from the buffered semaphore channel, which will block if we're + // already got as many goroutines as our concurrency limit allows + // + // when one of those routines finish they'll read from this channel, + // freeing up a slot to unblock this send + semaphoreChan <- struct{}{} + + // use an empty OSV as we're reusing the result struct + result := &result{i, OSV{}, db.updateAdvisory(id)} + + resultsChan <- result + + // read from the buffered semaphore to free up a slot to allow + // another goroutine to start, since this one is wrapping up + <-semaphoreChan + }(i, id) } - for _, row := range modifiedIDs { - err = db.updateAdvisory(row.id) + var errs []error - if err != nil { - return err + for { + result := <-resultsChan + errs = append(errs, result.err) + + if len(errs) == len(ids) { + break } } - return nil + return errors.Join(errs...) +} + +func (db *SmartDB) updateModifiedAdvisories(since time.Time) error { + modifiedIDs, err := db.fetchModifiedIDs(since) + + if err != nil { + return err + } + + return db.downloadModifiedAdvisories(modifiedIDs) } func (db *SmartDB) populate() (*time.Time, error) {