feat: add marketplace phase2 vector retrieval backend

cmd/anyclaw-registry/main.go

@@ -9,7 +9,9 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
 	"syscall"
+	"time"
 
 	"github.com/1024XEngineer/anyclaw/pkg/marketregistry"
 )
@@ -42,21 +44,42 @@ func serve(args []string) error {
 	dbDriver := fs.String("db-driver", "sqlite", "database/sql driver name")
 	dbDSN := fs.String("db-dsn", "", "database DSN; defaults to <data-dir>/registry.db for sqlite")
 	adminToken := fs.String("admin-token", os.Getenv("ANYCLAW_REGISTRY_ADMIN_TOKEN"), "admin bearer token; defaults to ANYCLAW_REGISTRY_ADMIN_TOKEN")
+	requireAdminToken := fs.Bool("require-admin-token", envBool("ANYCLAW_REGISTRY_REQUIRE_ADMIN_TOKEN", false), "fail startup when admin token is empty")
 	seed := fs.Bool("seed", true, "seed fixture artifacts when the registry is empty")
 	if err := fs.Parse(args); err != nil {
 		return err
 	}
 
+	vectorCfg := marketregistry.VectorConfig{
+		Enabled:         envBool("ANYCLAW_MARKET_VECTOR_ENABLED", false),
+		FailOpen:        envBool("ANYCLAW_MARKET_VECTOR_FAIL_OPEN", true),
+		Provider:        os.Getenv("ANYCLAW_MARKET_EMBEDDING_PROVIDER"),
+		Model:           firstNonEmpty(os.Getenv("ANYCLAW_MARKET_EMBEDDING_MODEL"), os.Getenv("ANYCLAW_MARKET_EMBEDDING_DASHSCOPE_MODEL")),
+		QueryModel:      firstNonEmpty(os.Getenv("ANYCLAW_MARKET_EMBEDDING_QUERY_MODEL"), os.Getenv("ANYCLAW_MARKET_EMBEDDING_DASHSCOPE_QUERY_MODEL")),
+		APIKey:          firstNonEmpty(os.Getenv("ANYCLAW_MARKET_EMBEDDING_API_KEY"), os.Getenv("ANYCLAW_MARKET_EMBEDDING_DASHSCOPE_API_KEY")),
+		BaseURL:         os.Getenv("ANYCLAW_MARKET_EMBEDDING_BASE_URL"),
+		SecretKey:       os.Getenv("ANYCLAW_MARKET_EMBEDDING_SECRET_KEY"),
+		QueryTimeout:    time.Duration(envInt("ANYCLAW_MARKET_VECTOR_TIMEOUT_MS", 12000)) * time.Millisecond,
+		WorkerPoll:      time.Duration(envInt("ANYCLAW_MARKET_VECTOR_WORKER_POLL_MS", 5000)) * time.Millisecond,
+		MaxJobAttempts:  envInt("ANYCLAW_MARKET_VECTOR_MAX_JOB_ATTEMPTS", 3),
+		HybridTopK:      envInt("ANYCLAW_MARKET_VECTOR_TOP_K", 25),
+		HybridCandidate: envInt("ANYCLAW_MARKET_VECTOR_CANDIDATE_LIMIT", 200),
+		QueryCacheTTL:   time.Duration(envInt("ANYCLAW_MARKET_QUERY_CACHE_TTL_SECONDS", 600)) * time.Second,
+		QueryCacheSize:  envInt("ANYCLAW_MARKET_QUERY_CACHE_SIZE", 512),
+	}
+
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()
 
 	server, err := marketregistry.NewServer(ctx, marketregistry.ServerConfig{
-		Addr:       *addr,
-		DataDir:    *dataDir,
-		DBDriver:   *dbDriver,
-		DBDSN:      *dbDSN,
-		AdminToken: *adminToken,
-		Seed:       *seed,
+		Addr:              *addr,
+		DataDir:           *dataDir,
+		DBDriver:          *dbDriver,
+		DBDSN:             *dbDSN,
+		AdminToken:        *adminToken,
+		RequireAdminToken: *requireAdminToken,
+		Seed:              *seed,
+		Vector:            vectorCfg,
 	})
 	if err != nil {
 		return err
@@ -72,5 +95,41 @@ func serve(args []string) error {
 }
 
 func printUsage() {
-	fmt.Println("Usage: anyclaw-registry serve [--addr :8791] [--data-dir .anyclaw-registry] [--db-driver sqlite] [--db-dsn path-or-url] [--admin-token token] [--seed=true]")
+	fmt.Println("Usage: anyclaw-registry serve [--addr :8791] [--data-dir .anyclaw-registry] [--db-driver sqlite] [--db-dsn path-or-url] [--admin-token token] [--require-admin-token=true] [--seed=true]")
+}
+
+func envBool(name string, fallback bool) bool {
+	value := os.Getenv(name)
+	if value == "" {
+		return fallback
+	}
+	switch value {
+	case "1", "true", "TRUE", "True", "yes", "YES", "on", "ON":
+		return true
+	case "0", "false", "FALSE", "False", "no", "NO", "off", "OFF":
+		return false
+	default:
+		return fallback
+	}
+}
+
+func envInt(name string, fallback int) int {
+	value := os.Getenv(name)
+	if value == "" {
+		return fallback
+	}
+	n, err := strconv.Atoi(value)
+	if err != nil {
+		return fallback
+	}
+	return n
+}
+
+func firstNonEmpty(values ...string) string {
+	for _, value := range values {
+		if value != "" {
+			return value
+		}
+	}
+	return ""
 }

cmd/anyclaw-registry/main_test.go

@@ -11,7 +11,7 @@ import (
 func TestRunDefaultsToServeAndRequiresAdminToken(t *testing.T) {
 	t.Setenv("ANYCLAW_REGISTRY_ADMIN_TOKEN", "")
 
-	err := run([]string{"serve", "--data-dir", t.TempDir(), "--seed=false"})
+	err := run([]string{"serve", "--data-dir", t.TempDir(), "--seed=false", "--require-admin-token=true"})
 	if err == nil || !strings.Contains(err.Error(), "admin token is required") {
 		t.Fatalf("expected missing admin token error, got %v", err)
 	}

pkg/capability/markettools/marketplace.go

@@ -16,6 +16,13 @@ type Options struct {
 	AuditLogger tools.AuditLogger
 }
 
+type retrievalMetaPayload struct {
+	SearchMode           string         `json:"search_mode,omitempty"`
+	VectorApplied        *bool          `json:"vector_applied,omitempty"`
+	VectorFallbackReason string         `json:"vector_fallback_reason,omitempty"`
+	CandidateCounts      map[string]int `json:"candidate_counts,omitempty"`
+}
+
 func Register(registry *tools.Registry, opts Options) {
 	if registry == nil || opts.Bridge == nil {
 		return
@@ -26,10 +33,11 @@ func Register(registry *tools.Registry, opts Options) {
 		InputSchema: map[string]any{
 			"type": "object",
 			"properties": map[string]any{
-				"query":  map[string]string{"type": "string", "description": "Capability need or search query"},
-				"kind":   map[string]string{"type": "string", "description": "Optional kind: agent, skill, or cli"},
-				"source": map[string]string{"type": "string", "description": "Optional source: local, cloud, or all"},
-				"limit":  map[string]string{"type": "number", "description": "Maximum results"},
+				"query":       map[string]string{"type": "string", "description": "Capability need or search query"},
+				"kind":        map[string]string{"type": "string", "description": "Optional kind: agent, skill, or cli"},
+				"source":      map[string]string{"type": "string", "description": "Optional source: local, cloud, or all"},
+				"search_mode": map[string]string{"type": "string", "description": "Optional search mode: auto, lexical, or hybrid"},
+				"limit":       map[string]string{"type": "number", "description": "Maximum results"},
 			},
 			"required": []string{"query"},
 		},
@@ -99,22 +107,31 @@ func searchArtifacts(ctx context.Context, opts Options, input map[string]any) (s
 	query := stringValue(input["query"])
 	kind := marketplace.NormalizeKind(stringValue(input["kind"]))
 	source := marketplace.NormalizeSource(stringValue(input["source"]))
+	searchMode := marketplace.NormalizeSearchMode(stringValue(input["search_mode"]))
 	limit := intValue(input["limit"], 5)
-	result, err := opts.Bridge.Search(ctx, marketbridge.SearchRequest{Query: query, Kind: kind, Source: source, Limit: limit})
+	result, err := opts.Bridge.Search(ctx, marketbridge.SearchRequest{
+		Query:      query,
+		Kind:       kind,
+		Source:     source,
+		SearchMode: searchMode,
+		Limit:      limit,
+	})
 	if err != nil {
 		return "", err
 	}
 	route := marketplace.RouteCapabilityNeed(query, result.Local, result.Cloud, limit)
 	return marshalJSON(map[string]any{
-		"query":       query,
-		"kind":        kind,
-		"source":      firstNonEmpty(string(source), "all"),
-		"route":       route,
-		"local_count": len(result.Local),
-		"cloud_count": len(result.Cloud),
-		"local":       marketplace.BuildCapabilityIndex(result.Local),
-		"cloud":       marketplace.BuildCapabilityIndex(result.Cloud),
-		"cloud_error": result.CloudErr,
+		"query":           query,
+		"kind":            kind,
+		"source":          firstNonEmpty(string(source), "all"),
+		"route":           route,
+		"local_count":     len(result.Local),
+		"cloud_count":     len(result.Cloud),
+		"local":           marketplace.BuildCapabilityIndex(result.Local),
+		"cloud":           marketplace.BuildCapabilityIndex(result.Cloud),
+		"cloud_artifacts": result.Cloud,
+		"cloud_error":     result.CloudErr,
+		"retrieval_meta":  buildRetrievalMetaPayload(result.RetrievalMeta),
 	})
 }
 
@@ -235,3 +252,22 @@ func marshalJSON(value any) (string, error) {
 	}
 	return string(data), nil
 }
+
+func buildRetrievalMetaPayload(meta *marketplace.RetrievalMeta) *retrievalMetaPayload {
+	if meta == nil {
+		return nil
+	}
+	payload := &retrievalMetaPayload{
+		SearchMode:           string(meta.SearchMode),
+		VectorApplied:        meta.VectorApplied,
+		VectorFallbackReason: meta.VectorFallbackReason,
+	}
+	if meta.CandidateCounts != nil {
+		payload.CandidateCounts = map[string]int{
+			"lexical": meta.CandidateCounts.Lexical,
+			"vector":  meta.CandidateCounts.Vector,
+			"merged":  meta.CandidateCounts.Merged,
+		}
+	}
+	return payload
+}

pkg/capability/markettools/marketplace_test.go

@@ -53,6 +53,12 @@ func TestSearchToolRoutesMissingCapabilityToCloud(t *testing.T) {
 	if !strings.Contains(out, `"action": "install_from_market"`) || !strings.Contains(out, "cloud.skill.release-notes") {
 		t.Fatalf("output = %s, want cloud install route", out)
 	}
+	if !strings.Contains(out, `"retrieval_meta"`) || !strings.Contains(out, `"search_mode": "lexical"`) {
+		t.Fatalf("output = %s, want retrieval meta", out)
+	}
+	if !strings.Contains(out, `"cloud_artifacts"`) || !strings.Contains(out, `"final_score": 0.91`) {
+		t.Fatalf("output = %s, want full cloud artifacts with final_score", out)
+	}
 }
 
 func TestInstallToolAskReturnsConfirmationWithoutInstalling(t *testing.T) {
@@ -244,11 +250,16 @@ func testMarketRegistryServer(t *testing.T, id, kind, risk, trust string, permis
 					"tags":           []string{"release notes", "changelog", "code review", "pull request"},
 					"hit_signals":    []string{"release notes", "code review"},
 					"score":          0.91,
+					"final_score":    0.91,
+					"match_signals":  []string{"lexical", "trust"},
 				}},
-				"total":  1,
-				"limit":  10,
-				"offset": 0,
-			}})
+				"total":          1,
+				"limit":          10,
+				"offset":         0,
+				"retrieval_meta": map[string]any{"search_mode": "lexical", "vector_applied": false},
+			},
+				"meta": map[string]any{"search_mode": "lexical", "vector_applied": false},
+			})
 		default:
 			http.NotFound(w, r)
 		}

pkg/marketplace/bridge/bridge.go

@@ -30,16 +30,18 @@ type Bridge interface {
 }
 
 type SearchRequest struct {
-	Query  string
-	Kind   marketplace.ArtifactKind
-	Source marketplace.SourceKind
-	Limit  int
+	Query      string
+	Kind       marketplace.ArtifactKind
+	Source     marketplace.SourceKind
+	SearchMode marketplace.SearchMode
+	Limit      int
 }
 
 type SearchResult struct {
-	Local    []marketplace.Artifact
-	Cloud    []marketplace.Artifact
-	CloudErr string
+	Local         []marketplace.Artifact
+	Cloud         []marketplace.Artifact
+	CloudErr      string
+	RetrievalMeta *marketplace.RetrievalMeta
 }
 
 type ListResult struct {
@@ -107,18 +109,30 @@ func (b *DefaultBridge) Search(ctx context.Context, req SearchRequest) (SearchRe
 	}
 	var cloud []marketplace.Artifact
 	var cloudErr string
+	var retrievalMeta *marketplace.RetrievalMeta
 	if req.Source != marketplace.SourceLocal && b.registry != nil {
-		result, err := b.registry.List(ctx, marketplace.Filter{Kind: req.Kind, Query: req.Query, Limit: limit})
+		result, err := b.registry.List(ctx, marketplace.Filter{
+			Kind:       req.Kind,
+			Query:      req.Query,
+			SearchMode: req.SearchMode,
+			Limit:      limit,
+		})
 		if err != nil {
 			cloudErr = err.Error()
 		} else {
 			cloud = result.Items
+			retrievalMeta = result.RetrievalMeta
 		}
 	}
 	if req.Source == marketplace.SourceCloud {
 		local = nil
 	}
-	return SearchResult{Local: local, Cloud: cloud, CloudErr: cloudErr}, nil
+	return SearchResult{
+		Local:         local,
+		Cloud:         cloud,
+		CloudErr:      cloudErr,
+		RetrievalMeta: retrievalMeta,
+	}, nil
 }
 
 func (b *DefaultBridge) List(ctx context.Context, filter marketplace.Filter) (ListResult, error) {

pkg/marketplace/capability_index.go

@@ -6,16 +6,23 @@ func BuildCapabilityIndex(items []Artifact) []CapabilityIndexItem {
 	out := make([]CapabilityIndexItem, 0, len(items))
 	for _, item := range items {
 		out = append(out, CapabilityIndexItem{
-			ArtifactID:   item.ID,
-			Kind:         item.Kind,
-			Name:         firstNonEmpty(item.DisplayName, item.Name),
-			Source:       item.Source,
-			Status:       string(item.Status),
-			Capabilities: artifactCapabilityTerms(item),
-			Permissions:  append([]string(nil), item.Permissions...),
-			RiskLevel:    item.RiskLevel,
-			TrustLevel:   item.TrustLevel,
-			Score:        item.Score,
+			ID:            item.ID,
+			ArtifactID:    item.ID,
+			Kind:          item.Kind,
+			Name:          firstNonEmpty(item.DisplayName, item.Name),
+			Description:   item.Description,
+			Source:        item.Source,
+			Status:        string(item.Status),
+			Capabilities:  artifactCapabilityTerms(item),
+			Permissions:   append([]string(nil), item.Permissions...),
+			RiskLevel:     item.RiskLevel,
+			TrustLevel:    item.TrustLevel,
+			Compatibility: item.Compatibility,
+			Installed:     item.Installed,
+			Score:         item.Score,
+			FinalScore:    item.FinalScore,
+			VectorScore:   item.VectorScore,
+			MatchSignals:  append([]string(nil), item.MatchSignals...),
 		})
 	}
 	return out

pkg/marketplace/registry/client.go

@@ -110,6 +110,9 @@ func (c *Client) List(ctx context.Context, filter marketplace.Filter) (marketpla
 	if filter.Query != "" {
 		values.Set("q", filter.Query)
 	}
+	if filter.SearchMode != "" {
+		values.Set("search_mode", string(filter.SearchMode))
+	}
 	if filter.Risk != "" {
 		values.Set("risk", filter.Risk)
 	}
@@ -149,10 +152,11 @@ func (c *Client) List(ctx context.Context, filter marketplace.Filter) (marketpla
 		items = append(items, convertArtifact(item))
 	}
 	return marketplace.ListResult{
-		Items:  items,
-		Total:  envelope.Data.Total,
-		Limit:  envelope.Data.Limit,
-		Offset: envelope.Data.Offset,
+		Items:         items,
+		Total:         envelope.Data.Total,
+		Limit:         envelope.Data.Limit,
+		Offset:        envelope.Data.Offset,
+		RetrievalMeta: firstRetrievalMeta(envelope.Data.RetrievalMeta, &envelope.Meta),
 	}, nil
 }
 
@@ -381,3 +385,16 @@ type remoteStatusError struct {
 func (e remoteStatusError) Error() string {
 	return fmt.Sprintf("marketplace registry returned HTTP %d", e.StatusCode)
 }
+
+func firstRetrievalMeta(values ...*remoteRetrievalMeta) *marketplace.RetrievalMeta {
+	for _, value := range values {
+		if value == nil {
+			continue
+		}
+		if value.SearchMode == "" && value.VectorApplied == nil && value.VectorFallbackReason == "" && value.CandidateCounts == nil {
+			continue
+		}
+		return convertRetrievalMeta(value)
+	}
+	return nil
+}