From b2b9aa24d3ea965e4305a202550c03e8260c6e6d Mon Sep 17 00:00:00 2001 From: Matee Ullah Malik Date: Tue, 8 Jul 2025 14:16:35 +0500 Subject: [PATCH 1/3] Refactor: reformat whole project --- pkg/crypto/hash.go | 1 - pkg/errors/utils_test.go | 1 - pkg/lumera/validator.go | 2 +- pkg/net/credentials/address_helper.go | 79 +++++++++---------- pkg/net/credentials/address_helper_test.go | 4 +- pkg/net/credentials/alts/common/common.go | 8 +- pkg/net/credentials/alts/common/utils_test.go | 10 +-- .../credentials/alts/conn/aes128gcm_test.go | 6 +- pkg/net/credentials/alts/conn/counter.go | 1 - pkg/net/credentials/alts/conn/record.go | 2 +- .../credentials/alts/handshake/common_test.go | 6 +- .../alts/handshake/handshake_test.go | 6 +- .../alts/testutil/fake_handshaker.go | 16 ++-- pkg/net/grpc/internal/leakcheck/leakcheck.go | 2 +- .../internal/leakcheck/leakcheck_enabled.go | 1 + pkg/testutil/accounts.go | 10 +-- pkg/testutil/net.go | 2 +- pkg/utils/heap.go | 4 +- sdk/adapters/supernodeservice/adapter.go | 2 +- .../node/supernode/server/mock_keyring.go | 1 - .../services/common/supernode/metrics.go | 4 +- .../services/common/supernode/service.go | 3 +- supernode/services/common/supernode/types.go | 4 +- .../securegrpc/secure_connection_test.go | 10 +-- 24 files changed, 90 insertions(+), 95 deletions(-) diff --git a/pkg/crypto/hash.go b/pkg/crypto/hash.go index e4f8dd12..5871506e 100644 --- a/pkg/crypto/hash.go +++ b/pkg/crypto/hash.go @@ -1,2 +1 @@ package crypto - diff --git a/pkg/errors/utils_test.go b/pkg/errors/utils_test.go index ab8d5545..0fdb56b7 100644 --- a/pkg/errors/utils_test.go +++ b/pkg/errors/utils_test.go @@ -108,4 +108,3 @@ func TestRecover(t *testing.T) { } } - diff --git a/pkg/lumera/validator.go b/pkg/lumera/validator.go index cfc38caa..8df272c4 100644 --- a/pkg/lumera/validator.go +++ b/pkg/lumera/validator.go @@ -4,8 +4,8 @@ import ( "context" "fmt" - authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" sntypes "github.com/LumeraProtocol/lumera/x/supernode/v1/types" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" ) type SecureKeyExchangeValidator struct { diff --git a/pkg/net/credentials/address_helper.go b/pkg/net/credentials/address_helper.go index 4745df5d..fd0ac87a 100644 --- a/pkg/net/credentials/address_helper.go +++ b/pkg/net/credentials/address_helper.go @@ -2,28 +2,28 @@ package credentials import ( "fmt" - "strings" "net" "strconv" + "strings" ) // LumeraAddress represents the components of a Lumera address type LumeraAddress struct { - Identity string - Host string - Port uint16 + Identity string + Host string + Port uint16 } type LumeraAddresses []LumeraAddress // String returns the address in the format "identity@host:port" func (a LumeraAddress) String() string { - return fmt.Sprintf("%s@%s:%d", a.Identity, a.Host, a.Port) + return fmt.Sprintf("%s@%s:%d", a.Identity, a.Host, a.Port) } // HostPort returns just the "host:port" portion func (a LumeraAddress) HostPort() string { - return fmt.Sprintf("%s:%d", a.Host, a.Port) + return fmt.Sprintf("%s:%d", a.Host, a.Port) } // ExtractIdentity extracts the identity part from an address in the format "identity@address" @@ -31,13 +31,13 @@ func (a LumeraAddress) HostPort() string { // If requireIdentity is true, an error is returned when identity is not found func ExtractIdentity(address string, requireIdentity ...bool) (string, string, error) { parts := strings.SplitN(address, "@", 2) - + // Check if identity is required identityRequired := false if len(requireIdentity) > 0 { identityRequired = requireIdentity[0] } - + if len(parts) != 2 { // Not in Lumera format, return empty identity and original address if identityRequired { @@ -45,18 +45,18 @@ func ExtractIdentity(address string, requireIdentity ...bool) (string, string, e } return "", address, nil } - + identity := parts[0] standardAddress := parts[1] - + if identity == "" { return "", "", fmt.Errorf("empty identity found in address: %s", address) } - + if standardAddress == "" { return "", "", fmt.Errorf("missing address in: %s", address) } - + return identity, standardAddress, nil } @@ -64,37 +64,36 @@ func ExtractIdentity(address string, requireIdentity ...bool) (string, string, e // and returns the components as a LumeraAddress struct. // Returns an error if any component, including the port, is missing. func ParseLumeraAddress(address string) (LumeraAddress, error) { - var result LumeraAddress - - // Extract identity and the remainder (host:port) - identity, remainder, err := ExtractIdentity(address, true) - if err != nil { - return result, fmt.Errorf("failed to extract identity: %w", err) - } - result.Identity = identity - - // Split the remainder into host and port - host, portStr, err := net.SplitHostPort(remainder) - if err != nil { - // If port is missing or any other format error, return an error - return result, fmt.Errorf("invalid host:port format: %w", err) - } + var result LumeraAddress + + // Extract identity and the remainder (host:port) + identity, remainder, err := ExtractIdentity(address, true) + if err != nil { + return result, fmt.Errorf("failed to extract identity: %w", err) + } + result.Identity = identity + + // Split the remainder into host and port + host, portStr, err := net.SplitHostPort(remainder) + if err != nil { + // If port is missing or any other format error, return an error + return result, fmt.Errorf("invalid host:port format: %w", err) + } if host == "" { return result, fmt.Errorf("missing host in address: %s", address) } - - result.Host = host - - // Parse the port string to uint16 - portInt, err := strconv.ParseUint(portStr, 10, 16) - if err != nil { - return result, fmt.Errorf("invalid port number: %w", err) - } - result.Port = uint16(portInt) - - return result, nil -} + result.Host = host + + // Parse the port string to uint16 + portInt, err := strconv.ParseUint(portStr, 10, 16) + if err != nil { + return result, fmt.Errorf("invalid port number: %w", err) + } + result.Port = uint16(portInt) + + return result, nil +} // IsLumeraAddressFormat checks if the address is in Lumera format (contains @) func IsLumeraAddressFormat(address string) bool { @@ -104,4 +103,4 @@ func IsLumeraAddressFormat(address string) bool { // FormatAddressWithIdentity creates a properly formatted address with identity@address func FormatAddressWithIdentity(identity, address string) string { return fmt.Sprintf("%s@%s", identity, address) -} \ No newline at end of file +} diff --git a/pkg/net/credentials/address_helper_test.go b/pkg/net/credentials/address_helper_test.go index 6a5f8f36..7f1665a7 100644 --- a/pkg/net/credentials/address_helper_test.go +++ b/pkg/net/credentials/address_helper_test.go @@ -1,8 +1,8 @@ package credentials import ( - "testing" "reflect" + "testing" ) func TestExtractIdentity(t *testing.T) { @@ -378,4 +378,4 @@ func TestFormatAddressWithIdentity(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/pkg/net/credentials/alts/common/common.go b/pkg/net/credentials/alts/common/common.go index 6b1835d5..27804eb3 100644 --- a/pkg/net/credentials/alts/common/common.go +++ b/pkg/net/credentials/alts/common/common.go @@ -24,10 +24,10 @@ const ( // Record protocol using XChaCha20-Poly1305 with rekeying RecordProtocolXChaCha20Poly1305ReKey = "ALTSRP_XCHACHA20_POLY1305_REKEY" - // Key sizes for different protocols - KeySizeAESGCM = 16 - KeySizeAESGCMReKey = 44 // 32 bytes key + 12 bytes counter mask - KeySizeXChaCha20Poly1305ReKey = 56 // 32 bytes key + 24 bytes nonce + // Key sizes for different protocols + KeySizeAESGCM = 16 + KeySizeAESGCMReKey = 44 // 32 bytes key + 12 bytes counter mask + KeySizeXChaCha20Poly1305ReKey = 56 // 32 bytes key + 24 bytes nonce ) // ALTSRecordCrypto is the interface for gRPC ALTS record protocol. diff --git a/pkg/net/credentials/alts/common/utils_test.go b/pkg/net/credentials/alts/common/utils_test.go index 2c69d3e4..d69826bd 100644 --- a/pkg/net/credentials/alts/common/utils_test.go +++ b/pkg/net/credentials/alts/common/utils_test.go @@ -4,13 +4,13 @@ import ( "bytes" "encoding/binary" "errors" + "github.com/cosmos/gogoproto/proto" + "github.com/stretchr/testify/require" "io" "net" "strings" "testing" "time" - "github.com/cosmos/gogoproto/proto" - "github.com/stretchr/testify/require" "github.com/LumeraProtocol/lumera/x/lumeraid/securekeyx" lumeraidtypes "github.com/LumeraProtocol/lumera/x/lumeraid/types" @@ -231,10 +231,10 @@ func TestReceiveHandshakeMessageFailures(t *testing.T) { func TestParseValidHandshakeMessage(t *testing.T) { // Create a valid handshake message. validHandshake := &lumeraidtypes.HandshakeInfo{ - Address: "127.0.0.1:8080", - PeerType: int32(securekeyx.Supernode), + Address: "127.0.0.1:8080", + PeerType: int32(securekeyx.Supernode), PublicKey: []byte("public-key"), - Curve: "curve-name", + Curve: "curve-name", } handshakeBytes, err := proto.Marshal(validHandshake) if err != nil { diff --git a/pkg/net/credentials/alts/conn/aes128gcm_test.go b/pkg/net/credentials/alts/conn/aes128gcm_test.go index 628e67a0..5188a0ce 100644 --- a/pkg/net/credentials/alts/conn/aes128gcm_test.go +++ b/pkg/net/credentials/alts/conn/aes128gcm_test.go @@ -2,11 +2,11 @@ package conn import ( "bytes" - "testing" - "runtime" "fmt" - "time" + "runtime" "strconv" + "testing" + "time" . "github.com/LumeraProtocol/supernode/pkg/net/credentials/alts/common" ) diff --git a/pkg/net/credentials/alts/conn/counter.go b/pkg/net/credentials/alts/conn/counter.go index 83dc7517..66ec7239 100644 --- a/pkg/net/credentials/alts/conn/counter.go +++ b/pkg/net/credentials/alts/conn/counter.go @@ -42,4 +42,3 @@ func (c *Counter) Inc() { c.invalid = true } } - diff --git a/pkg/net/credentials/alts/conn/record.go b/pkg/net/credentials/alts/conn/record.go index 85bebaa1..af97d3df 100644 --- a/pkg/net/credentials/alts/conn/record.go +++ b/pkg/net/credentials/alts/conn/record.go @@ -63,7 +63,7 @@ type Conn struct { // NewConn creates a new secure channel instance given the other party role and // handshaking result. -var NewConn = func (c net.Conn, side Side, recordProtocol string, key, protected []byte) (net.Conn, error) { +var NewConn = func(c net.Conn, side Side, recordProtocol string, key, protected []byte) (net.Conn, error) { newCrypto := protocols[recordProtocol] if newCrypto == nil { return nil, fmt.Errorf("negotiated unknown next_protocol %q", recordProtocol) diff --git a/pkg/net/credentials/alts/handshake/common_test.go b/pkg/net/credentials/alts/handshake/common_test.go index ec85bf1a..735b7a8d 100644 --- a/pkg/net/credentials/alts/handshake/common_test.go +++ b/pkg/net/credentials/alts/handshake/common_test.go @@ -3,9 +3,9 @@ package handshake import ( "testing" - "github.com/stretchr/testify/assert" - . "github.com/LumeraProtocol/supernode/pkg/net/credentials/alts/common" "github.com/LumeraProtocol/lumera/x/lumeraid/securekeyx" + . "github.com/LumeraProtocol/supernode/pkg/net/credentials/alts/common" + "github.com/stretchr/testify/assert" ) func TestNewAuthInfo(t *testing.T) { @@ -15,4 +15,4 @@ func TestNewAuthInfo(t *testing.T) { assert.Equal(t, ClientSide, autInfo.(*AuthInfo).Side, "Side should match") assert.Equal(t, securekeyx.Simplenode, autInfo.(*AuthInfo).RemotePeerType, "RemotePeerType should match") assert.Equal(t, "cosmos1", autInfo.(*AuthInfo).RemoteIdentity, "RemoteIdentity should match") -} \ No newline at end of file +} diff --git a/pkg/net/credentials/alts/handshake/handshake_test.go b/pkg/net/credentials/alts/handshake/handshake_test.go index 8cd8844e..5b4db595 100644 --- a/pkg/net/credentials/alts/handshake/handshake_test.go +++ b/pkg/net/credentials/alts/handshake/handshake_test.go @@ -17,12 +17,12 @@ import ( lumeraidmocks "github.com/LumeraProtocol/lumera/x/lumeraid/mocks" "github.com/LumeraProtocol/lumera/x/lumeraid/securekeyx" lumeraidtypes "github.com/LumeraProtocol/lumera/x/lumeraid/types" + sntypes "github.com/LumeraProtocol/lumera/x/supernode/v1/types" . "github.com/LumeraProtocol/supernode/pkg/net/credentials/alts/common" "github.com/LumeraProtocol/supernode/pkg/net/credentials/alts/conn" "github.com/LumeraProtocol/supernode/pkg/net/credentials/alts/testutil" - authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" - sntypes "github.com/LumeraProtocol/lumera/x/supernode/v1/types" . "github.com/LumeraProtocol/supernode/pkg/testutil" + authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" ) const defaultTestTimeout = 100 * time.Second @@ -331,7 +331,7 @@ func TestHandshakerConcurrentHandshakes(t *testing.T) { SupernodeAccount: serverAddr, }, nil). Times(1) - + serverMockValidator := lumeraidmocks.NewMockKeyExchangerValidator(ctrl) serverMockValidator.EXPECT(). GetSupernodeBySupernodeAddress(gomock.Any(), serverAddr). diff --git a/pkg/net/credentials/alts/testutil/fake_handshaker.go b/pkg/net/credentials/alts/testutil/fake_handshaker.go index 7eec82f2..3995b478 100644 --- a/pkg/net/credentials/alts/testutil/fake_handshaker.go +++ b/pkg/net/credentials/alts/testutil/fake_handshaker.go @@ -3,12 +3,12 @@ package testutil import ( "bytes" "fmt" - "time" "net" + "time" - "github.com/cosmos/gogoproto/proto" lumeraidtypes "github.com/LumeraProtocol/lumera/x/lumeraid/types" . "github.com/LumeraProtocol/supernode/pkg/net/credentials/alts/common" + "github.com/cosmos/gogoproto/proto" ) // FakeHandshakeConn implements a fake handshake connection @@ -52,11 +52,11 @@ func (c *FakeHandshakeConn) Close() error { // FakeHandshaker simulates a peer in the handshake process type FakeHandshaker struct { - conn *FakeHandshakeConn - signature []byte - pubKey []byte - peerType int32 - curve string + conn *FakeHandshakeConn + signature []byte + pubKey []byte + peerType int32 + curve string } func NewFakeHandshaker(conn *FakeHandshakeConn) *FakeHandshaker { @@ -101,4 +101,4 @@ func (h *FakeHandshaker) SimulateHandshake(isClient bool) error { func (h *FakeHandshaker) SimulateError(err error) { h.conn.handshakeErr = err -} \ No newline at end of file +} diff --git a/pkg/net/grpc/internal/leakcheck/leakcheck.go b/pkg/net/grpc/internal/leakcheck/leakcheck.go index 8d2a39d8..04e9b336 100644 --- a/pkg/net/grpc/internal/leakcheck/leakcheck.go +++ b/pkg/net/grpc/internal/leakcheck/leakcheck.go @@ -16,8 +16,8 @@ import ( "sync/atomic" "time" - "github.com/LumeraProtocol/supernode/pkg/net/grpc/mem" "github.com/LumeraProtocol/supernode/pkg/net/grpc/internal" + "github.com/LumeraProtocol/supernode/pkg/net/grpc/mem" ) // failTestsOnLeakedBuffers is a special flag that will cause tests to fail if diff --git a/pkg/net/grpc/internal/leakcheck/leakcheck_enabled.go b/pkg/net/grpc/internal/leakcheck/leakcheck_enabled.go index 60642ad4..07f6ba41 100644 --- a/pkg/net/grpc/internal/leakcheck/leakcheck_enabled.go +++ b/pkg/net/grpc/internal/leakcheck/leakcheck_enabled.go @@ -1,4 +1,5 @@ //go:build checkbuffers + package leakcheck func init() { diff --git a/pkg/testutil/accounts.go b/pkg/testutil/accounts.go index 53991270..f180ae3d 100644 --- a/pkg/testutil/accounts.go +++ b/pkg/testutil/accounts.go @@ -1,17 +1,17 @@ package testutil import ( - "testing" "crypto/ecdh" "github.com/stretchr/testify/require" + "testing" - "github.com/cosmos/go-bip39" "github.com/cosmos/cosmos-sdk/codec" codectypes "github.com/cosmos/cosmos-sdk/codec/types" + cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec" "github.com/cosmos/cosmos-sdk/crypto/hd" "github.com/cosmos/cosmos-sdk/crypto/keyring" - cryptocodec "github.com/cosmos/cosmos-sdk/crypto/codec" cryptotypes "github.com/cosmos/cosmos-sdk/crypto/types" + "github.com/cosmos/go-bip39" "github.com/LumeraProtocol/lumera/x/lumeraid/securekeyx" ) @@ -27,9 +27,9 @@ type TestAccount struct { Address string PubKey cryptotypes.PubKey } - + // setupTestKeyExchange creates a key exchange instance for testing -func SetupTestKeyExchange(t *testing.T, kb keyring.Keyring, addr string, +func SetupTestKeyExchange(t *testing.T, kb keyring.Keyring, addr string, peerType securekeyx.PeerType, validator securekeyx.KeyExchangerValidator) *securekeyx.SecureKeyExchange { ke, err := securekeyx.NewSecureKeyExchange(kb, addr, peerType, ecdh.P256(), validator) require.NoError(t, err) diff --git a/pkg/testutil/net.go b/pkg/testutil/net.go index 306ff8c1..f1994b93 100644 --- a/pkg/testutil/net.go +++ b/pkg/testutil/net.go @@ -16,4 +16,4 @@ func GetFreePortInRange(start, end int) (int, error) { } } return 0, fmt.Errorf("no free port found in range %d-%d", start, end) -} \ No newline at end of file +} diff --git a/pkg/utils/heap.go b/pkg/utils/heap.go index 5655c3ec..321288af 100644 --- a/pkg/utils/heap.go +++ b/pkg/utils/heap.go @@ -10,10 +10,10 @@ type DistanceEntry struct { str string } -//MaxHeap to efficiently sort XOR distances +// MaxHeap to efficiently sort XOR distances type MaxHeap []DistanceEntry -//Len returns the length of heap +// Len returns the length of heap func (h MaxHeap) Len() int { return len(h) } diff --git a/sdk/adapters/supernodeservice/adapter.go b/sdk/adapters/supernodeservice/adapter.go index 34a5817f..0885c14f 100644 --- a/sdk/adapters/supernodeservice/adapter.go +++ b/sdk/adapters/supernodeservice/adapter.go @@ -325,7 +325,7 @@ func toSdkSupernodeStatus(resp *supernode.StatusResponse) *SupernodeStatusrespon TaskCount: service.TaskCount, }) } - + // Convert AvailableServices data result.AvailableServices = make([]string, len(resp.AvailableServices)) copy(result.AvailableServices, resp.AvailableServices) diff --git a/supernode/node/supernode/server/mock_keyring.go b/supernode/node/supernode/server/mock_keyring.go index 8e97e51d..85cb9910 100644 --- a/supernode/node/supernode/server/mock_keyring.go +++ b/supernode/node/supernode/server/mock_keyring.go @@ -12,7 +12,6 @@ import ( types0 "github.com/cosmos/cosmos-sdk/types" signing "github.com/cosmos/cosmos-sdk/types/tx/signing" gomock "go.uber.org/mock/gomock" - ) // MockKeyring is a mock of Keyring interface. diff --git a/supernode/services/common/supernode/metrics.go b/supernode/services/common/supernode/metrics.go index 4940a08f..180dd8f4 100644 --- a/supernode/services/common/supernode/metrics.go +++ b/supernode/services/common/supernode/metrics.go @@ -29,7 +29,7 @@ func (m *MetricsCollector) CollectCPUMetrics(ctx context.Context) (usage, remain usageFloat := percentages[0] remainingFloat := 100 - usageFloat - + return fmt.Sprintf("%.2f", usageFloat), fmt.Sprintf("%.2f", remainingFloat), nil } @@ -43,4 +43,4 @@ func (m *MetricsCollector) CollectMemoryMetrics(ctx context.Context) (total, use } return vmem.Total, vmem.Used, vmem.Available, vmem.UsedPercent, nil -} \ No newline at end of file +} diff --git a/supernode/services/common/supernode/service.go b/supernode/services/common/supernode/service.go index 09290c80..70265cdd 100644 --- a/supernode/services/common/supernode/service.go +++ b/supernode/services/common/supernode/service.go @@ -6,11 +6,10 @@ import ( "github.com/LumeraProtocol/supernode/pkg/logtrace" ) - // SupernodeStatusService provides centralized status information // by collecting system metrics and aggregating task information from registered services type SupernodeStatusService struct { - taskProviders []TaskProvider // List of registered services that provide task information + taskProviders []TaskProvider // List of registered services that provide task information metrics *MetricsCollector // System metrics collector for CPU and memory stats } diff --git a/supernode/services/common/supernode/types.go b/supernode/services/common/supernode/types.go index a3aeed8c..4942b4c2 100644 --- a/supernode/services/common/supernode/types.go +++ b/supernode/services/common/supernode/types.go @@ -29,7 +29,7 @@ type ServiceTasks struct { type TaskProvider interface { // GetServiceName returns the unique name identifier for this service GetServiceName() string - + // GetRunningTasks returns a list of currently active task IDs GetRunningTasks() []string -} \ No newline at end of file +} diff --git a/tests/integration/securegrpc/secure_connection_test.go b/tests/integration/securegrpc/secure_connection_test.go index 61049909..7a44ccb9 100644 --- a/tests/integration/securegrpc/secure_connection_test.go +++ b/tests/integration/securegrpc/secure_connection_test.go @@ -12,23 +12,23 @@ import ( "testing" "time" - "github.com/stretchr/testify/require" "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" "google.golang.org/grpc/grpclog" "google.golang.org/grpc/health" "google.golang.org/grpc/health/grpc_health_v1" + lumeraidmocks "github.com/LumeraProtocol/lumera/x/lumeraid/mocks" "github.com/LumeraProtocol/lumera/x/lumeraid/securekeyx" + sntypes "github.com/LumeraProtocol/lumera/x/supernode/v1/types" pb "github.com/LumeraProtocol/supernode/gen/supernode/tests/integration/securegrpc" + snkeyring "github.com/LumeraProtocol/supernode/pkg/keyring" ltc "github.com/LumeraProtocol/supernode/pkg/net/credentials" "github.com/LumeraProtocol/supernode/pkg/net/credentials/alts/conn" "github.com/LumeraProtocol/supernode/pkg/net/grpc/client" "github.com/LumeraProtocol/supernode/pkg/net/grpc/server" - snkeyring "github.com/LumeraProtocol/supernode/pkg/keyring" - lumeraidmocks "github.com/LumeraProtocol/lumera/x/lumeraid/mocks" "github.com/LumeraProtocol/supernode/pkg/testutil" authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" - sntypes "github.com/LumeraProtocol/lumera/x/supernode/v1/types" ) func waitForServerReady(address string, timeout time.Duration) error { @@ -98,7 +98,7 @@ func TestSecureGRPCConnection(t *testing.T) { SupernodeAccount: serverAddress, }, nil). Times(1) - + serverMockValidator := lumeraidmocks.NewMockKeyExchangerValidator(ctrl) serverMockValidator.EXPECT(). GetSupernodeBySupernodeAddress(gomock.Any(), serverAddress). From e03265b3cae561e0c7d5c79786b7c8a143335f7f Mon Sep 17 00:00:00 2001 From: Matee Ullah Malik Date: Wed, 9 Jul 2025 14:14:33 +0500 Subject: [PATCH 2/3] Implement adaptive chunk size calculation for file downloads --- sdk/adapters/supernodeservice/adapter.go | 45 ++++++++++++++++- .../server/cascade/cascade_action_server.go | 50 ++++++++++++++++++- 2 files changed, 91 insertions(+), 4 deletions(-) diff --git a/sdk/adapters/supernodeservice/adapter.go b/sdk/adapters/supernodeservice/adapter.go index 0885c14f..9b400d71 100644 --- a/sdk/adapters/supernodeservice/adapter.go +++ b/sdk/adapters/supernodeservice/adapter.go @@ -36,6 +36,45 @@ func NewCascadeAdapter(ctx context.Context, conn *grpc.ClientConn, logger log.Lo } } +// calculateOptimalChunkSize returns an optimal chunk size based on file size +// to balance throughput and memory usage +func calculateOptimalChunkSize(fileSize int64) int { + const ( + minChunkSize = 64 * 1024 // 64 KB minimum + maxChunkSize = 4 * 1024 * 1024 // 4 MB maximum for 1GB+ files + smallFileThreshold = 1024 * 1024 // 1 MB + mediumFileThreshold = 50 * 1024 * 1024 // 50 MB + largeFileThreshold = 500 * 1024 * 1024 // 500 MB + ) + + var chunkSize int + + switch { + case fileSize <= smallFileThreshold: + // For small files (up to 1MB), use 64KB chunks + chunkSize = minChunkSize + case fileSize <= mediumFileThreshold: + // For medium files (1MB-50MB), use 256KB chunks + chunkSize = 256 * 1024 + case fileSize <= largeFileThreshold: + // For large files (50MB-500MB), use 1MB chunks + chunkSize = 1024 * 1024 + default: + // For very large files (500MB+), use 4MB chunks for optimal throughput + chunkSize = maxChunkSize + } + + // Ensure chunk size is within bounds + if chunkSize < minChunkSize { + chunkSize = minChunkSize + } + if chunkSize > maxChunkSize { + chunkSize = maxChunkSize + } + + return chunkSize +} + func (a *cascadeAdapter) CascadeSupernodeRegister(ctx context.Context, in *CascadeSupernodeRegisterRequest, opts ...grpc.CallOption) (*CascadeSupernodeRegisterResponse, error) { // Create the client stream ctx = net.AddCorrelationID(ctx) @@ -63,8 +102,10 @@ func (a *cascadeAdapter) CascadeSupernodeRegister(ctx context.Context, in *Casca } totalBytes := fileInfo.Size() - // Define chunk size - const chunkSize = 1024 // 1 KB + // Define adaptive chunk size based on file size + chunkSize := calculateOptimalChunkSize(totalBytes) + + a.logger.Debug(ctx, "Calculated optimal chunk size", "fileSize", totalBytes, "chunkSize", chunkSize) // Keep track of how much data we've processed bytesRead := int64(0) diff --git a/supernode/node/action/server/cascade/cascade_action_server.go b/supernode/node/action/server/cascade/cascade_action_server.go index 319be25a..ab203836 100644 --- a/supernode/node/action/server/cascade/cascade_action_server.go +++ b/supernode/node/action/server/cascade/cascade_action_server.go @@ -24,6 +24,45 @@ func NewCascadeActionServer(factory cascadeService.CascadeServiceFactory) *Actio return &ActionServer{factory: factory} } +// calculateOptimalChunkSize returns an optimal chunk size based on file size +// to balance throughput and memory usage +func calculateOptimalChunkSize(fileSize int64) int { + const ( + minChunkSize = 64 * 1024 // 64 KB minimum + maxChunkSize = 4 * 1024 * 1024 // 4 MB maximum for 1GB+ files + smallFileThreshold = 1024 * 1024 // 1 MB + mediumFileThreshold = 50 * 1024 * 1024 // 50 MB + largeFileThreshold = 500 * 1024 * 1024 // 500 MB + ) + + var chunkSize int + + switch { + case fileSize <= smallFileThreshold: + // For small files (up to 1MB), use 64KB chunks + chunkSize = minChunkSize + case fileSize <= mediumFileThreshold: + // For medium files (1MB-50MB), use 256KB chunks + chunkSize = 256 * 1024 + case fileSize <= largeFileThreshold: + // For large files (50MB-500MB), use 1MB chunks + chunkSize = 1024 * 1024 + default: + // For very large files (500MB+), use 4MB chunks for optimal throughput + chunkSize = maxChunkSize + } + + // Ensure chunk size is within bounds + if chunkSize < minChunkSize { + chunkSize = minChunkSize + } + if chunkSize > maxChunkSize { + chunkSize = maxChunkSize + } + + return chunkSize +} + func (server *ActionServer) Desc() *grpc.ServiceDesc { return &pb.CascadeService_ServiceDesc } @@ -209,8 +248,15 @@ func (server *ActionServer) Download(req *pb.DownloadRequest, stream pb.CascadeS } logtrace.Info(ctx, "streaming artefact file in chunks", fields) - // Split and stream the file in 1024 byte chunks - const chunkSize = 1024 + // Calculate optimal chunk size based on file size + chunkSize := calculateOptimalChunkSize(int64(len(restoredFile))) + + logtrace.Info(ctx, "calculated optimal chunk size for download", logtrace.Fields{ + "file_size": len(restoredFile), + "chunk_size": chunkSize, + }) + + // Split and stream the file using adaptive chunk size for i := 0; i < len(restoredFile); i += chunkSize { end := i + chunkSize if end > len(restoredFile) { From d94a518ec1c13c5c1bf8b8cb596a04e8984eda19 Mon Sep 17 00:00:00 2001 From: Matee Ullah Malik Date: Fri, 11 Jul 2025 15:09:27 +0500 Subject: [PATCH 3/3] feat : implement index file structure, update signature verification, and improve file handling --- .../server/cascade/cascade_action_server.go | 7 ++ supernode/node/supernode/server/server.go | 8 +- supernode/services/cascade/download.go | 69 ++++++++-- supernode/services/cascade/helper.go | 99 +++++++++++---- supernode/services/cascade/metadata.go | 32 +++++ supernode/services/cascade/register.go | 2 +- supernode/services/cascade/register_test.go | 25 +++- tests/system/e2e_cascade_test.go | 119 +++++++++++++----- tests/system/test.txt | 5 + 9 files changed, 290 insertions(+), 76 deletions(-) create mode 100644 tests/system/test.txt diff --git a/supernode/node/action/server/cascade/cascade_action_server.go b/supernode/node/action/server/cascade/cascade_action_server.go index ab203836..1f896fb1 100644 --- a/supernode/node/action/server/cascade/cascade_action_server.go +++ b/supernode/node/action/server/cascade/cascade_action_server.go @@ -154,6 +154,13 @@ func (server *ActionServer) Register(stream pb.CascadeService_RegisterServer) er fields[logtrace.FieldActionID] = metadata.GetActionId() logtrace.Info(ctx, "metadata received from action-sdk", fields) + // Ensure all data is written to disk before calculating hash + if err := tempFile.Sync(); err != nil { + fields[logtrace.FieldError] = err.Error() + logtrace.Error(ctx, "failed to sync temp file", fields) + return fmt.Errorf("failed to sync temp file: %w", err) + } + hash := hasher.Sum(nil) hashHex := hex.EncodeToString(hash) fields[logtrace.FieldHashHex] = hashHex diff --git a/supernode/node/supernode/server/server.go b/supernode/node/supernode/server/server.go index e4ad906c..157775d8 100644 --- a/supernode/node/supernode/server/server.go +++ b/supernode/node/supernode/server/server.go @@ -62,8 +62,12 @@ func (server *Server) Run(ctx context.Context) error { // Custom server options opts := grpcserver.DefaultServerOptions() - //TODO : Defaul ServerOptions needs to be updated to hanlde larger files - //EXAMPLE: opts.GracefulShutdownTime = 60 * time.Second + opts.MaxRecvMsgSize = 2 * 1024 * 1024 * 1024 // 2 GB + opts.MaxSendMsgSize = 2 * 1024 * 1024 * 1024 // 2 GB + opts.InitialWindowSize = 32 * 1024 * 1024 // 32 MB + opts.InitialConnWindowSize = 32 * 1024 * 1024 // 32 MB + opts.WriteBufferSize = 1024 * 1024 // 1 MB + opts.ReadBufferSize = 1024 * 1024 // 1 MB for _, address := range addresses { addr := net.JoinHostPort(strings.TrimSpace(address), strconv.Itoa(server.config.Port)) diff --git a/supernode/services/cascade/download.go b/supernode/services/cascade/download.go index fab26fba..dbc1ba4d 100644 --- a/supernode/services/cascade/download.go +++ b/supernode/services/cascade/download.go @@ -1,6 +1,7 @@ package cascade import ( + "bytes" "context" "fmt" "os" @@ -77,26 +78,29 @@ func (task *CascadeRegistrationTask) downloadArtifacts(ctx context.Context, acti logtrace.Info(ctx, "started downloading the artifacts", fields) var layout codec.Layout - for _, rqID := range metadata.RqIdsIds { - rqIDFile, err := task.P2PClient.Retrieve(ctx, rqID) - if err != nil || len(rqIDFile) == 0 { + + for _, indexID := range metadata.RqIdsIds { + indexFile, err := task.P2PClient.Retrieve(ctx, indexID) + if err != nil || len(indexFile) == 0 { continue } - layout, _, _, err = parseRQMetadataFile(rqIDFile) - - if len(layout.Blocks) == 0 { - logtrace.Info(ctx, "no symbols found in RQ metadata", fields) + // Parse index file to get layout IDs + indexData, err := task.parseIndexFile(indexFile) + if err != nil { + logtrace.Info(ctx, "failed to parse index file", fields) continue } - //if len(layout.Blocks) < int(float64(len(metadata.RqIdsIds))*requiredSymbolPercent/100) { - // logtrace.Info(ctx, "not enough symbols found in RQ metadata", fields) - // continue - //} + // Try to retrieve layout files using layout IDs from index file + layout, err = task.retrieveLayoutFromIndex(ctx, indexData, fields) + if err != nil { + logtrace.Info(ctx, "failed to retrieve layout from index", fields) + continue + } - if err == nil { - logtrace.Info(ctx, "layout file retrieved", fields) + if len(layout.Blocks) > 0 { + logtrace.Info(ctx, "layout file retrieved via index", fields) break } } @@ -190,6 +194,45 @@ func (task *CascadeRegistrationTask) streamDownloadEvent(eventType SupernodeEven return } +// parseIndexFile parses compressed index file to extract IndexFile structure +func (task *CascadeRegistrationTask) parseIndexFile(data []byte) (IndexFile, error) { + decompressed, err := utils.ZstdDecompress(data) + if err != nil { + return IndexFile{}, errors.Errorf("decompress index file: %w", err) + } + + // Parse decompressed data: base64IndexFile.signature.counter + parts := bytes.Split(decompressed, []byte{SeparatorByte}) + if len(parts) < 2 { + return IndexFile{}, errors.New("invalid index file format") + } + + // Decode the base64 index file + return decodeIndexFile(string(parts[0])) +} + +// retrieveLayoutFromIndex retrieves layout file using layout IDs from index file +func (task *CascadeRegistrationTask) retrieveLayoutFromIndex(ctx context.Context, indexData IndexFile, fields logtrace.Fields) (codec.Layout, error) { + // Try to retrieve layout files using layout IDs from index file + for _, layoutID := range indexData.LayoutIDs { + layoutFile, err := task.P2PClient.Retrieve(ctx, layoutID) + if err != nil || len(layoutFile) == 0 { + continue + } + + layout, _, _, err := parseRQMetadataFile(layoutFile) + if err != nil { + continue + } + + if len(layout.Blocks) > 0 { + return layout, nil + } + } + + return codec.Layout{}, errors.New("no valid layout found in index") +} + func (task *CascadeRegistrationTask) CleanupDownload(ctx context.Context, actionID string) error { if actionID == "" { return errors.New("actionID is empty") diff --git a/supernode/services/cascade/helper.go b/supernode/services/cascade/helper.go index 0ae132dd..f6cf44a4 100644 --- a/supernode/services/cascade/helper.go +++ b/supernode/services/cascade/helper.go @@ -80,8 +80,8 @@ func (task *CascadeRegistrationTask) verifyDataHash(ctx context.Context, dh []by return nil } -func (task *CascadeRegistrationTask) encodeInput(ctx context.Context, path string, dataSize int, f logtrace.Fields) (*adaptors.EncodeResult, error) { - resp, err := task.RQ.EncodeInput(ctx, task.ID(), path, dataSize) +func (task *CascadeRegistrationTask) encodeInput(ctx context.Context, actionID string, path string, dataSize int, f logtrace.Fields) (*adaptors.EncodeResult, error) { + resp, err := task.RQ.EncodeInput(ctx, actionID, path, dataSize) if err != nil { return nil, task.wrapErr(ctx, "failed to encode data", err, f) } @@ -91,42 +91,56 @@ func (task *CascadeRegistrationTask) encodeInput(ctx context.Context, path strin func (task *CascadeRegistrationTask) verifySignatureAndDecodeLayout(ctx context.Context, encoded string, creator string, encodedMeta codec.Layout, f logtrace.Fields) (codec.Layout, string, error) { - file, sig, err := extractSignatureAndFirstPart(encoded) + // Extract index file and creator signature from encoded data + // The signatures field contains: Base64(index_file).creators_signature + indexFileB64, creatorSig, err := extractIndexFileAndSignature(encoded) if err != nil { - return codec.Layout{}, "", task.wrapErr(ctx, "failed to extract signature and first part", err, f) + return codec.Layout{}, "", task.wrapErr(ctx, "failed to extract index file and creator signature", err, f) } - logtrace.Info(ctx, "signature and first part have been extracted", f) - // Decode the base64-encoded signature - sigBytes, err := base64.StdEncoding.DecodeString(sig) + // Verify creator signature on index file + creatorSigBytes, err := base64.StdEncoding.DecodeString(creatorSig) if err != nil { - return codec.Layout{}, "", task.wrapErr(ctx, "failed to decode signature from base64", err, f) + return codec.Layout{}, "", task.wrapErr(ctx, "failed to decode creator signature from base64", err, f) } - // Log the verification attempt for the node creator - logtrace.Info(ctx, "verifying signature from node creator", logtrace.Fields{ - "creator": creator, - "taskID": task.ID(), - }) + if err := task.LumeraClient.Verify(ctx, creator, []byte(indexFileB64), creatorSigBytes); err != nil { + return codec.Layout{}, "", task.wrapErr(ctx, "failed to verify creator signature", err, f) + } + logtrace.Info(ctx, "creator signature successfully verified", f) - // Pass the decoded signature bytes for verification - if err := task.LumeraClient.Verify(ctx, creator, []byte(file), sigBytes); err != nil { - return codec.Layout{}, "", task.wrapErr(ctx, "failed to verify node creator signature", err, f) + // Decode index file to get the layout signature + indexFile, err := decodeIndexFile(indexFileB64) + if err != nil { + return codec.Layout{}, "", task.wrapErr(ctx, "failed to decode index file", err, f) } - logtrace.Info(ctx, "node creator signature successfully verified", f) + // Verify layout signature on the actual layout + layoutSigBytes, err := base64.StdEncoding.DecodeString(indexFile.LayoutSignature) + if err != nil { + return codec.Layout{}, "", task.wrapErr(ctx, "failed to decode layout signature from base64", err, f) + } - layout, err := decodeMetadataFile(file) + layoutJSON, err := json.Marshal(encodedMeta) if err != nil { - return codec.Layout{}, "", task.wrapErr(ctx, "failed to decode metadata file", err, f) + return codec.Layout{}, "", task.wrapErr(ctx, "failed to marshal layout", err, f) + } + layoutB64 := utils.B64Encode(layoutJSON) + if err := task.LumeraClient.Verify(ctx, creator, layoutB64, layoutSigBytes); err != nil { + return codec.Layout{}, "", task.wrapErr(ctx, "failed to verify layout signature", err, f) } + logtrace.Info(ctx, "layout signature successfully verified", f) - return layout, sig, nil + return encodedMeta, indexFile.LayoutSignature, nil } func (task *CascadeRegistrationTask) generateRQIDFiles(ctx context.Context, meta actiontypes.CascadeMetadata, sig, creator string, encodedMeta codec.Layout, f logtrace.Fields) (GenRQIdentifiersFilesResponse, error) { - res, err := GenRQIdentifiersFiles(ctx, GenRQIdentifiersFilesRequest{ + // The signatures field contains: Base64(index_file).creators_signature + // This full format will be used for ID generation to match chain expectations + + // Generate layout files + layoutRes, err := GenRQIdentifiersFiles(ctx, GenRQIdentifiersFilesRequest{ Metadata: encodedMeta, CreatorSNAddress: creator, RqMax: uint32(meta.RqIdsMax), @@ -135,10 +149,24 @@ func (task *CascadeRegistrationTask) generateRQIDFiles(ctx context.Context, meta }) if err != nil { return GenRQIdentifiersFilesResponse{}, - task.wrapErr(ctx, "failed to generate RQID Files", err, f) + task.wrapErr(ctx, "failed to generate layout files", err, f) } - logtrace.Info(ctx, "rq symbols, rq-ids and rqid-files have been generated", f) - return res, nil + + // Generate index files using full signatures format for ID generation (matches chain expectation) + indexIDs, indexFiles, err := GenIndexFiles(ctx, layoutRes.RedundantMetadataFiles, sig, meta.Signatures, uint32(meta.RqIdsIc), uint32(meta.RqIdsMax)) + if err != nil { + return GenRQIdentifiersFilesResponse{}, + task.wrapErr(ctx, "failed to generate index files", err, f) + } + + // Store layout files and index files separately in P2P + allFiles := append(layoutRes.RedundantMetadataFiles, indexFiles...) + + // Return index IDs (sent to chain) and all files (stored in P2P) + return GenRQIdentifiersFilesResponse{ + RQIDs: indexIDs, + RedundantMetadataFiles: allFiles, + }, nil } func (task *CascadeRegistrationTask) storeArtefacts(ctx context.Context, actionID string, idFiles [][]byte, symbolsDir string, f logtrace.Fields) error { @@ -262,3 +290,26 @@ func parseRQMetadataFile(data []byte) (layout codec.Layout, signature string, co return layout, signature, counter, nil } + +// extractIndexFileAndSignature extracts index file and creator signature from signatures field +// data is expected to be in format: Base64(index_file).creators_signature +func extractIndexFileAndSignature(data string) (indexFileB64 string, creatorSignature string, err error) { + parts := strings.Split(data, ".") + if len(parts) < 2 { + return "", "", errors.New("invalid signatures format") + } + return parts[0], parts[1], nil +} + +// decodeIndexFile decodes base64 encoded index file +func decodeIndexFile(data string) (IndexFile, error) { + var indexFile IndexFile + decodedData, err := utils.B64Decode([]byte(data)) + if err != nil { + return indexFile, errors.Errorf("failed to decode index file: %w", err) + } + if err := json.Unmarshal(decodedData, &indexFile); err != nil { + return indexFile, errors.Errorf("failed to unmarshal index file: %w", err) + } + return indexFile, nil +} diff --git a/supernode/services/cascade/metadata.go b/supernode/services/cascade/metadata.go index 3add83cd..f1999cc5 100644 --- a/supernode/services/cascade/metadata.go +++ b/supernode/services/cascade/metadata.go @@ -18,6 +18,13 @@ const ( SeparatorByte byte = 46 // separator in dd_and_fingerprints.signature i.e. '.' ) +// IndexFile represents the structure of the index file +type IndexFile struct { + Version int `json:"version"` + LayoutIDs []string `json:"layout_ids"` + LayoutSignature string `json:"layout_signature"` +} + type GenRQIdentifiersFilesRequest struct { Metadata codec.Layout RqMax uint32 @@ -93,3 +100,28 @@ func GetIDFiles(ctx context.Context, encMetadataFileWithSignature []byte, ic uin return ids, idFiles, nil } + +// GenIndexFiles generates index files and their IDs from layout files using full signatures format +func GenIndexFiles(ctx context.Context, layoutFiles [][]byte, layoutSignature string, signaturesFormat string, ic uint32, max uint32) (indexIDs []string, indexFiles [][]byte, err error) { + // Create layout IDs from layout files + layoutIDs := make([]string, len(layoutFiles)) + for i, layoutFile := range layoutFiles { + hash, err := utils.Blake3Hash(layoutFile) + if err != nil { + return nil, nil, errors.Errorf("hash layout file: %w", err) + } + layoutIDs[i] = base58.Encode(hash) + } + + // Use the full signatures format that matches what was sent during RequestAction + // The chain expects this exact format for ID generation + indexFileWithSignatures := []byte(signaturesFormat) + + // Generate index file IDs using full signatures format + indexIDs, indexFiles, err = GetIDFiles(ctx, indexFileWithSignatures, ic, max) + if err != nil { + return nil, nil, errors.Errorf("get index ID files: %w", err) + } + + return indexIDs, indexFiles, nil +} diff --git a/supernode/services/cascade/register.go b/supernode/services/cascade/register.go index a0cccba5..939fded0 100644 --- a/supernode/services/cascade/register.go +++ b/supernode/services/cascade/register.go @@ -90,7 +90,7 @@ func (task *CascadeRegistrationTask) Register( task.streamEvent(SupernodeEventTypeDataHashVerified, "data-hash has been verified", "", send) /* 6. Encode the raw data ------------------------------------------------------ */ - encResp, err := task.encodeInput(ctx, req.FilePath, req.DataSize, fields) + encResp, err := task.encodeInput(ctx, req.ActionID, req.FilePath, req.DataSize, fields) if err != nil { return err } diff --git a/supernode/services/cascade/register_test.go b/supernode/services/cascade/register_test.go index 2fe64359..ab5daed1 100644 --- a/supernode/services/cascade/register_test.go +++ b/supernode/services/cascade/register_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/base64" "encoding/hex" + "encoding/json" "os" "testing" @@ -75,10 +76,12 @@ func TestCascadeRegistrationTask_Register(t *testing.T) { }, }, nil) - // 3. Signature verification + // 3. Signature verification - layout signature on layout file + // Expect two verification calls: creator signature and layout signature lc.EXPECT(). Verify(gomock.Any(), "creator1", gomock.Any(), gomock.Any()). - Return(nil) + Return(nil). + Times(2) // 4. Finalize lc.EXPECT(). @@ -253,9 +256,19 @@ func TestCascadeRegistrationTask_Register(t *testing.T) { func encodedCascadeMetadata(hash string, t *testing.T) []byte { t.Helper() - // Fake encoded layout and signature - fakeLayout := base64.StdEncoding.EncodeToString([]byte(`{"blocks":[{"block_id":1,"hash":"abc"}]}`)) - fakeSig := base64.StdEncoding.EncodeToString([]byte("fakesignature")) + // Fake layout signature for new index file format + fakeLayoutSig := base64.StdEncoding.EncodeToString([]byte("fakelayoutsignature")) + + // Create index file structure + indexFile := map[string]any{ + "layout_ids": []string{"layout_id_1", "layout_id_2"}, + "layout_signature": fakeLayoutSig, + } + indexFileJSON, _ := json.Marshal(indexFile) + fakeIndexFile := base64.StdEncoding.EncodeToString(indexFileJSON) + + // Fake creators signature - this is what the chain uses for index ID generation + fakeCreatorsSig := base64.StdEncoding.EncodeToString([]byte("fakecreatorssignature")) metadata := &actiontypes.CascadeMetadata{ DataHash: hash, @@ -263,7 +276,7 @@ func encodedCascadeMetadata(hash string, t *testing.T) []byte { RqIdsIc: 2, RqIdsMax: 4, RqIdsIds: []string{"id1", "id2"}, - Signatures: fakeLayout + "." + fakeSig, + Signatures: fakeIndexFile + "." + fakeCreatorsSig, } bytes, err := proto.Marshal(metadata) diff --git a/tests/system/e2e_cascade_test.go b/tests/system/e2e_cascade_test.go index 1181e039..f52cefea 100644 --- a/tests/system/e2e_cascade_test.go +++ b/tests/system/e2e_cascade_test.go @@ -17,6 +17,8 @@ import ( "github.com/LumeraProtocol/supernode/pkg/codec" "github.com/LumeraProtocol/supernode/pkg/keyring" "github.com/LumeraProtocol/supernode/pkg/lumera" + "github.com/LumeraProtocol/supernode/pkg/utils" + "github.com/cosmos/btcutil/base58" "lukechampine.com/blake3" "github.com/LumeraProtocol/supernode/sdk/action" @@ -77,7 +79,6 @@ func TestCascadeE2E(t *testing.T) { sut.ModifyGenesisJSON(t, SetActionParams(t)) // Reset and start the blockchain - sut.ResetChain(t) sut.StartChain(t) cli := NewLumeradCLI(t, sut, true) // --------------------------------------- @@ -97,9 +98,10 @@ func TestCascadeE2E(t *testing.T) { // Register the supernode with the network registerCmd := []string{ "tx", "supernode", "register-supernode", - valAddr, // validator address - "localhost:" + port, // IP address with unique port - addr, // supernode account + valAddr, + "localhost:" + port, + addr, + "--p2p-port", p2pPort, "--from", nodeKey, } @@ -246,13 +248,13 @@ func TestCascadeE2E(t *testing.T) { // --------------------------------------- t.Log("Step 4: Creating test file for RaptorQ encoding") - // Create a test file with sample data in a temporary directory + // Use test file from tests/system directory + testFileName := "test.txt" + testFileFullpath := filepath.Join(testFileName) - testFileName := "testfile.txt" - testFileFullpath := filepath.Join(t.TempDir(), testFileName) - testData := []byte("This is test data for RaptorQ encoding in the Lumera nasaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaasassasetwork") - err = os.WriteFile(testFileFullpath, testData, 0644) - require.NoError(t, err, "Failed to write test file") + // Verify test file exists + _, err = os.Stat(testFileFullpath) + require.NoError(t, err, "Test file not found: %s", testFileFullpath) // Read the file into memory for processing file, err := os.Open(testFileFullpath) @@ -283,26 +285,83 @@ func TestCascadeE2E(t *testing.T) { me, err := json.Marshal(metadataFile) require.NoError(t, err, "Failed to marshal metadata to JSON") - // Step 1: Encode the metadata JSON as base64 string - // This becomes the first part of our signature format - regularbase64EncodedData := base64.StdEncoding.EncodeToString(me) - t.Logf("Base64 encoded RQ IDs file length: %d", len(regularbase64EncodedData)) + // Step 1: Encode the metadata JSON as base64 string for layout signature + layoutBase64 := base64.StdEncoding.EncodeToString(me) + t.Logf("Base64 encoded layout file length: %d", len(layoutBase64)) - // Step 2: Sign the base64-encoded string with user key instead of testkey1 - signedMetaData, err := keyring.SignBytes(keplrKeyring, userKeyName, []byte(regularbase64EncodedData)) - require.NoError(t, err, "Failed to sign metadata") + // Step 2: Sign the layout with user key (layout signature) + layoutSignature, err := keyring.SignBytes(keplrKeyring, userKeyName, []byte(layoutBase64)) + require.NoError(t, err, "Failed to sign layout") + layoutSignatureB64 := base64.StdEncoding.EncodeToString(layoutSignature) - // Step 3: Encode the resulting signature as base64 - signedbase64EncodedData := base64.StdEncoding.EncodeToString(signedMetaData) - t.Logf("Base64 signed RQ IDs file length: %d", len(signedbase64EncodedData)) + // Step 3: Generate real layout files and IDs (production flow) + // Create redundant layout files with counters and signatures + const ic = uint32(121) + const maxFiles = uint32(50) - // Step 4: Format according to the expected verification pattern: Base64(rq_ids).signature - // This format is expected by VerifySignature in the CascadeActionHandler.RegisterAction method - // - regularbase64EncodedData: The base64-encoded metadata - // - signedbase64EncodedData: The base64-encoded signature of the above - signatureFormat := fmt.Sprintf("%s.%s", regularbase64EncodedData, signedbase64EncodedData) + // Create layout file with signature (base64Layout.signature) + layoutWithSig := fmt.Sprintf("%s.%s", layoutBase64, layoutSignatureB64) + + // Generate layout IDs + layoutIDs := make([]string, maxFiles) + for i := range maxFiles { + counter := ic + uint32(i) + // Create layout file content: base64Layout.signature.counter + layoutFileContent := fmt.Sprintf("%s.%d", layoutWithSig, counter) + + // Compress and hash to get layout ID + compressedData, err := utils.ZstdCompress([]byte(layoutFileContent)) + require.NoError(t, err, "Failed to compress layout file") + + hash, err := utils.Blake3Hash(compressedData) + require.NoError(t, err, "Failed to hash layout file") + + layoutIDs[i] = base58.Encode(hash) + } + t.Logf("Generated %d real layout IDs", len(layoutIDs)) + + // Create index file structure with l layout IDs + indexFile := map[string]any{ + "layout_ids": layoutIDs, + "layout_signature": layoutSignatureB64, + } + + // Step 4: Marshal and encode index file + indexFileJSON, err := json.Marshal(indexFile) + require.NoError(t, err, "Failed to marshal index file") + indexFileBase64 := base64.StdEncoding.EncodeToString(indexFileJSON) + t.Logf("Base64 encoded index file length: %d", len(indexFileBase64)) + + // Step 5: Sign the index file with user key (creator signature) + creatorSignature, err := keyring.SignBytes(keplrKeyring, userKeyName, []byte(indexFileBase64)) + require.NoError(t, err, "Failed to sign index file") + creatorSignatureB64 := base64.StdEncoding.EncodeToString(creatorSignature) + + // Step 6: Format Base64(index_file).creators_signature + // The chain uses this format to regenerate and validate index IDs + signatureFormat := fmt.Sprintf("%s.%s", indexFileBase64, creatorSignatureB64) t.Logf("Signature format prepared with length: %d bytes", len(signatureFormat)) + // Step 7: Generate index file IDs (what the supernode should send to chain) + // Supernode creates index IDs using: Base58(BLAKE3(zstd(rq_ids_signature.counter))) + // where rq_ids_signature is the signatureFormat we just created + indexFileIDs := make([]string, maxFiles) + for i := range maxFiles { + counter := ic + uint32(i) + // Create index file content: rq_ids_signature.counter + indexFileContent := fmt.Sprintf("%s.%d", signatureFormat, counter) + + // Compress and hash to get index file ID + compressedData, err := utils.ZstdCompress([]byte(indexFileContent)) + require.NoError(t, err, "Failed to compress index file") + + hash, err := utils.Blake3Hash(compressedData) + require.NoError(t, err, "Failed to hash index file") + + indexFileIDs[i] = base58.Encode(hash) + } + t.Logf("Generated %d index file IDs for chain verification", len(indexFileIDs)) + // Data hash with blake3 hash, err := Blake3Hash(data) b64EncodedHash := base64.StdEncoding.EncodeToString(hash) @@ -381,7 +440,7 @@ func TestCascadeE2E(t *testing.T) { // --------------------------------------- // Step 8: Extract action ID and start cascade // --------------------------------------- - time.Sleep(30 * time.Second) + time.Sleep(40 * time.Second) t.Log("Step 8: Extracting action ID and creating cascade request") @@ -558,7 +617,7 @@ func TestCascadeE2E(t *testing.T) { t.Log("Test completed successfully!") - time.Sleep(10 * time.Second) + time.Sleep(120 * time.Second) outputFileBaseDir := filepath.Join(".") // Try to download the file using the action ID @@ -567,7 +626,7 @@ func TestCascadeE2E(t *testing.T) { t.Logf("Download response: %s", dtaskID) require.NoError(t, err, "Failed to download cascade data using action ID") - time.Sleep(10 * time.Second) // Wait to ensure all events are processed + time.Sleep(4 * time.Second) // --------------------------------------- // Step 11: Validate downloaded files exist @@ -625,8 +684,8 @@ func SetActionParams(t *testing.T) GenesisMutator { "denom": "ulume" }, "expiration_duration": "24h0m0s", - "fee_per_byte": { - "amount": "100", + "fee_per_kbyte": { + "amount": "0", "denom": "ulume" }, "foundation_fee_share": "0.000000000000000000", diff --git a/tests/system/test.txt b/tests/system/test.txt new file mode 100644 index 00000000..586dce57 --- /dev/null +++ b/tests/system/test.txt @@ -0,0 +1,5 @@ +This is a test file for cascade e2e testing. +Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. +Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. +Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. +Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. \ No newline at end of file