feat: introduce a smarter database for osv.dev zips

__snapshots__/main_test.snap

@@ -713,7 +713,7 @@ testdata/locks-empty/composer.lock: found 0 packages
 ---
 
 [TestRun_ParseAsGlobal/#02 - 2]
-Error, could not parse testdata/locks-empty/Gemfile.lock: unexpected end of JSON input
+Error, could not parse testdata/locks-empty/Gemfile.lock: EOF
 Error, could not parse testdata/locks-empty/yarn.lock: invalid character '#' looking for beginning of value
 
 ---
@@ -743,7 +743,7 @@ testdata/locks-insecure/my-package-lock.json: found 1 package
 ---
 
 [TestRun_ParseAsGlobal/#03 - 2]
-Error, could not parse testdata/locks-empty/Gemfile.lock: unexpected end of JSON input
+Error, could not parse testdata/locks-empty/Gemfile.lock: EOF
 Error, could not parse testdata/locks-empty/yarn.lock: invalid character '#' looking for beginning of value
 
 ---
@@ -857,7 +857,7 @@ testdata/locks-empty/composer.lock: found 0 packages
 ---
 
 [TestRun_ParseAsSpecific/#04 - 2]
-Error, could not parse testdata/locks-empty/Gemfile.lock: unexpected end of JSON input
+Error, could not parse testdata/locks-empty/Gemfile.lock: EOF
 Error, could not parse testdata/locks-empty/yarn.lock: invalid character '#' looking for beginning of value
 
 ---
@@ -887,7 +887,7 @@ testdata/locks-insecure/my-package-lock.json: found 1 package
 ---
 
 [TestRun_ParseAsSpecific/#05 - 2]
-Error, could not parse testdata/locks-empty/Gemfile.lock: unexpected end of JSON input
+Error, could not parse testdata/locks-empty/Gemfile.lock: EOF
 Error, could not parse testdata/locks-empty/yarn.lock: invalid character '#' looking for beginning of value
 
 ---

go.mod

@@ -8,24 +8,39 @@ require (
 	github.com/gkampitakis/go-snaps v0.5.15
 	github.com/google/go-cmp v0.7.0
 	github.com/google/osv-scalibr v0.3.3-0.20250919162947-bcadd5b8f946
-	github.com/tidwall/jsonc v0.3.2
 	golang.org/x/mod v0.28.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
+	github.com/CycloneDX/cyclonedx-go v0.9.2 // indirect
+	github.com/anchore/go-struct-converter v0.0.0-20230627203149-c72ef8859ca9 // indirect
 	github.com/gkampitakis/ciinfo v0.3.2 // indirect
 	github.com/gkampitakis/go-diff v1.3.2 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.6.2 // indirect
+	github.com/go-git/go-git/v5 v5.16.2 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/goccy/go-yaml v1.18.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect
 	github.com/maruel/natural v1.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/rogpeppe/go-internal v1.13.1 // indirect
+	github.com/ossf/osv-schema/bindings/go v0.0.0-20250805051309-c463400aa925 // indirect
+	github.com/package-url/packageurl-go v0.1.3 // indirect
+	github.com/rogpeppe/go-internal v1.14.1 // indirect
+	github.com/spdx/tools-golang v0.5.5 // indirect
 	github.com/tidwall/gjson v1.18.0 // indirect
+	github.com/tidwall/jsonc v0.3.2 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
+	golang.org/x/net v0.41.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
+	google.golang.org/protobuf v1.36.8 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 )

internal/reporter/reporter.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"strings"
 
 	"github.com/fatih/color"
 	"github.com/g-rath/osv-detector/pkg/database"
@@ -30,7 +31,9 @@ func New(stdout io.Writer, stderr io.Writer, outputAsJSON bool) *Reporter {
 // PrintErrorf writes the given message to stderr, regardless of if the reporter
 // is outputting as JSON or not
 func (r *Reporter) PrintErrorf(msg string, a ...any) {
-	fmt.Fprintf(r.stderr, msg, a...)
+	// todo: this is a hack to make the lockfile/extractor error output more like the original
+	//  there's no real reason to be doing it other than that there isn't a reason not to...
+	fmt.Fprint(r.stderr, strings.Replace(fmt.Sprintf(msg, a...), " could not extract:", "", 1))
 }
 
 // PrintTextf writes the given message to stdout, _unless_ the reporter is set

main.go

@@ -33,10 +33,15 @@ func makeAPIDBConfig() database.Config {
 	}
 }
 
-func makeEcosystemDBConfig(ecosystem internal.Ecosystem) database.Config {
+func makeEcosystemDBConfig(ecosystem internal.Ecosystem, beSmart bool) database.Config {
+	typ := "zip"
+	if beSmart {
+		typ = "smart"
+	}
+
 	return database.Config{
 		Name: string(ecosystem),
-		Type: "zip",
+		Type: typ,
 		URL:  fmt.Sprintf("https://osv-vulnerabilities.storage.googleapis.com/%s/all.zip", ecosystem),
 	}
 }
@@ -158,6 +163,15 @@ func describeDB(db database.DB) string {
 	switch tt := db.(type) {
 	case *database.APIDB:
 		return "using batches of " + color.YellowString("%d", tt.BatchSize)
+	case *database.SmartDB:
+		count := tt.VulnerabilitiesCount
+
+		return fmt.Sprintf(
+			"%s %s, including withdrawn - last updated %s",
+			color.YellowString("%d", count),
+			reporter.Form(count, "vulnerability", "vulnerabilities"),
+			tt.UpdatedAt,
+		)
 	case *database.ZipDB:
 		count := tt.VulnerabilitiesCount
 
@@ -372,6 +386,7 @@ func (files lockfileAndConfigOrErrs) adjustExtraDatabases(
 	removeConfigDatabases bool,
 	addDefaultAPIDatabase bool,
 	addEcosystemDatabases bool,
+	beSmart bool,
 ) {
 	for _, file := range files {
 		if file.err != nil {
@@ -391,7 +406,7 @@ func (files lockfileAndConfigOrErrs) adjustExtraDatabases(
 			ecosystems := collectEcosystems([]lockfileAndConfigOrErr{file})
 
 			for _, ecosystem := range ecosystems {
-				extraDBConfigs = append(extraDBConfigs, makeEcosystemDBConfig(ecosystem))
+				extraDBConfigs = append(extraDBConfigs, makeEcosystemDBConfig(ecosystem, beSmart))
 			}
 		}
 
@@ -508,6 +523,7 @@ func run(args []string, stdout, stderr io.Writer) int {
 	useDatabases := cli.Bool("use-dbs", true, "Use the databases from osv.dev to check for known vulnerabilities")
 	useAPI := cli.Bool("use-api", false, "Use the osv.dev API to check for known vulnerabilities")
 	batchSize := cli.Int("batch-size", 1000, "The number of packages to include in each batch when using the api database")
+	beSmart := cli.Bool("be-smart", false, "Use smart database mode for faster incremental updates")
 
 	cli.Var(&globalIgnores, "ignore", `ID of an OSV to ignore when determining exit codes.
 This flag can be passed multiple times to ignore different vulnerabilities`)
@@ -589,7 +605,7 @@ This flag can be passed multiple times to ignore different vulnerabilities`)
 
 	files := readAllLockfiles(r, pathsToLocksWithParseAs, cli.Args(), loadLocalConfig, &config)
 
-	files.adjustExtraDatabases(*noConfigDatabases, *useAPI, *useDatabases)
+	files.adjustExtraDatabases(*noConfigDatabases, *useAPI, *useDatabases, *beSmart)
 
 	dbs, errored := loadDatabases(
 		r,

pkg/database/config.go

@@ -10,6 +10,8 @@ type Config struct {
 	Type             string `yaml:"type"`
 	URL              string `yaml:"url"`
 	WorkingDirectory string `yaml:"working-directory"`
+
+	CacheDirectory string `yaml:"-"`
 }
 
 // Identifier returns a unique string that can be used to check if a loaded
@@ -29,6 +31,8 @@ var ErrUnsupportedDatabaseType = errors.New("unsupported database source type")
 // Load initializes a new OSV database based on the given Config
 func Load(config Config, offline bool, batchSize int) (DB, error) {
 	switch config.Type {
+	case "smart":
+		return NewSmartDB(config, offline)
 	case "zip":
 		return NewZippedDB(config, offline)
 	case "api":