feat(sync): support direct cloud upload

cmd/keystone-edge/main.go

@@ -17,7 +17,6 @@ import (
 
 	"github.com/joho/godotenv"
 
-	"archebase.com/keystone-edge/internal/cloud"
 	"archebase.com/keystone-edge/internal/config"
 	"archebase.com/keystone-edge/internal/logger"
 	"archebase.com/keystone-edge/internal/server"
@@ -115,46 +114,8 @@ func main() {
 
 	// Initialize cloud sync worker
 	var syncWorker *services.SyncWorker
-	if cfg.Sync.Enabled && cfg.Sync.AuthEndpoint != "" && cfg.Sync.GatewayEndpoint != "" && s3Client != nil {
-		authClient := cloud.NewAuthClient(cloud.AuthClientConfig{
-			Endpoint:      cfg.Sync.AuthEndpoint,
-			UseTLS:        cfg.Sync.CloudUseTLS,
-			TLSCAFile:     cfg.Sync.CloudTLSCAFile,
-			TLSServerName: cfg.Sync.CloudTLSServerName,
-			APIKey:        cfg.Sync.APIKey,
-			RefreshBefore: 60 * time.Second,
-		})
-
-		gatewayClient := cloud.NewGatewayClient(cloud.GatewayClientConfig{
-			Endpoint:       cfg.Sync.GatewayEndpoint,
-			UseTLS:         cfg.Sync.CloudUseTLS,
-			TLSCAFile:      cfg.Sync.CloudTLSCAFile,
-			TLSServerName:  cfg.Sync.CloudTLSServerName,
-			RequestTimeout: time.Duration(cfg.Sync.RequestTimeoutSec) * time.Second,
-		}, authClient)
-		// Close gateway client before auth client (LIFO defer order).
-		defer func() {
-			if err := authClient.Close(); err != nil {
-				logger.Printf("[SYNC] Failed to close auth client: %v", err)
-			}
-		}()
-		defer func() {
-			if err := gatewayClient.Close(); err != nil {
-				logger.Printf("[SYNC] Failed to close gateway client: %v", err)
-			}
-		}()
-
-		uploader, err := cloud.NewUploader(gatewayClient, s3Client, cfg.Storage.Bucket, cloud.UploaderConfig{
-			RequestTimeout:  time.Duration(cfg.Sync.RequestTimeoutSec) * time.Second,
-			OSSTimeout:      time.Duration(cfg.Sync.OSSTimeoutSec) * time.Second,
-			PersistRootDir:  cfg.Sync.PersistRootDir,
-			MaxRestartCount: uint32(cfg.Sync.MaxRestartCount), //nolint:gosec // non-negative guaranteed by config.Validate()
-		})
-		if err != nil {
-			logger.Fatalf("[SYNC] Failed to initialise uploader: %v", err)
-		}
-
-		syncWorker = services.NewSyncWorker(db.DB, uploader, s3Client, cfg.Storage.Bucket, services.SyncWorkerConfig{
+	if cfg.Sync.Enabled && cfg.Sync.DPConfigPath != "" && s3Client != nil {
+		syncWorker = services.NewSyncWorker(db.DB, nil, s3Client, cfg.Storage.Bucket, services.SyncWorkerConfig{
 			BatchSize:       cfg.Sync.BatchSize,
 			MaxConcurrent:   cfg.Sync.MaxConcurrent,
 			MaxRetries:      cfg.Sync.MaxRetries,
@@ -166,9 +127,9 @@ func main() {
 		}, &cfg.Sync)
 
 		syncWorker.Start()
-		logger.Printf("[SYNC] Cloud sync worker started: auth=%s gateway=%s auto_scan=%t", cfg.Sync.AuthEndpoint, cfg.Sync.GatewayEndpoint, cfg.Sync.AutoScanEnabled)
+		logger.Printf("[SYNC] Cloud sync worker started: dp_config=%s auto_scan=%t", cfg.Sync.DPConfigPath, cfg.Sync.AutoScanEnabled)
 	} else {
-		logger.Println("[SYNC] Cloud sync disabled (KEYSTONE_SYNC_ENABLED=false or missing endpoints)")
+		logger.Println("[SYNC] Cloud sync disabled (KEYSTONE_SYNC_ENABLED=false, missing KEYSTONE_SYNC_DP_CONFIG, or S3 unavailable)")
 	}
 
 	// Initialize and start HTTP server

docker/.env.example

@@ -42,15 +42,7 @@ KEYSTONE_MINIO_USE_SSL=false
 KEYSTONE_SYNC_ENABLED=true
 KEYSTONE_SYNC_BATCH_SIZE=10
 KEYSTONE_SYNC_MAX_RETRIES=5
-KEYSTONE_CLOUD_AUTH_ENDPOINT=127.0.0.1:50051
-KEYSTONE_CLOUD_GATEWAY_ENDPOINT=127.0.0.1:50053
-KEYSTONE_CLOUD_USE_TLS=false
-# Optional: custom CA bundle for TLS verification (PEM).
-# KEYSTONE_CLOUD_TLS_CA_FILE=/etc/ssl/certs/your-ca.pem
-# Optional: override TLS server name (SNI / verification), useful when endpoint is an IP.
-# KEYSTONE_CLOUD_TLS_SERVER_NAME=cloud.example.com
-# API key issued by the data-platform (base64url, no padding).
-KEYSTONE_CLOUD_API_KEY=your-api-key-here
+KEYSTONE_SYNC_DP_CONFIG=~/.archebase/config.json
 KEYSTONE_SYNC_WORKER_INTERVAL=15
 KEYSTONE_SYNC_REQUEST_TIMEOUT=30
 KEYSTONE_SYNC_OSS_TIMEOUT=120