diff --git a/differ/differ_test.go b/differ/differ_test.go index cd790b4..d51894c 100644 --- a/differ/differ_test.go +++ b/differ/differ_test.go @@ -12,8 +12,6 @@ package differ import ( "crypto/sha512" "fmt" - "github.com/couchbase/gomemcached" - "github.com/stretchr/testify/assert" "io/ioutil" "math/rand" "os" @@ -22,11 +20,24 @@ import ( "time" "xdcrDiffer/dcp" fdp "xdcrDiffer/fileDescriptorPool" + + "github.com/couchbase/gomemcached" + xdcrBase "github.com/couchbase/goxdcr/base" + hlv "github.com/couchbase/goxdcr/hlv" + xdcrLog "github.com/couchbase/goxdcr/log" + "github.com/stretchr/testify/assert" ) const MaxUint64 = ^uint64(0) const MinUint = 0 +const ( + eventingKey = "_eventing" + ftsKey = "_fts" + queryKey = "_query" + indexingKey = "_indexing" +) + var randomOnce sync.Once func randomString(l int) string { @@ -43,16 +54,18 @@ func randInt(min int, max int) int { // serialize mutation into []byte // format: -// keyLen - 2 bytes -// key - length specified by keyLen -// seqno - 8 bytes -// revId - 8 bytes -// cas - 8 bytes -// flags - 4 bytes -// expiry - 4 bytes -// opCode - 1 bytes -// hash - 64 bytes -func genTestData(regularMutation, colFilters bool) (key string, seqno, revId, cas uint64, flags, expiry uint32, opCode gomemcached.CommandCode, hash [64]byte, ret []byte, colId uint32, filterIds []uint8) { +// +// keyLen - 2 bytes +// key - length specified by keyLen +// seqno - 8 bytes +// revId - 8 bytes +// cas - 8 bytes +// flags - 4 bytes +// expiry - 4 bytes +// opCode - 1 bytes +// hash - 64 bytes +func genTestData(withXattrs, regularMutation, colFilters bool) (key string, seqno, revId, cas uint64, flags, expiry uint32, opCode gomemcached.CommandCode, hash [64]byte, ret []byte, colId uint32, filterIds []uint8, err error) { + var mutationToSerialize dcp.Mutation randomOnce.Do(func() { rand.Seed(time.Now().UTC().UnixNano()) }) @@ -80,31 +93,97 @@ func genTestData(regularMutation, colFilters bool) (key string, seqno, revId, ca } //dataSlice := createDataByteSlice(key, seqno, revId, cas, flags, expiry, opCode, hash, colId, filterIds) - mutationToSerialize := dcp.Mutation{ - Vbno: 0, - Key: []byte(key), - Seqno: seqno, - RevId: revId, - Cas: cas, - Flags: flags, - Expiry: expiry, - OpCode: opCode, - Value: []byte(key), - Datatype: 0, - ColId: 0, - ColFiltersMatched: filterIds, + if !withXattrs { + mutationToSerialize = dcp.Mutation{ + Vbno: 0, + Key: []byte(key), + Seqno: seqno, + RevId: revId, + Cas: cas, + Flags: flags, + Expiry: expiry, + OpCode: opCode, + Value: []byte(key), + Datatype: 0, + ColId: 0, + ColFiltersMatched: filterIds, + } + } else { + xattrIterator := &xdcrBase.XattrIterator{} + xattrKeysToExclude := genXattrKeysToExclude() + ret, err = genXattrsWithBody(key) + if err != nil { + return + } + mutationToSerialize = dcp.Mutation{ + Vbno: 0, + Key: []byte(key), + Seqno: seqno, + RevId: revId, + Cas: cas, + Flags: flags, + Expiry: expiry, + OpCode: opCode, + Value: ret, + Datatype: 0, + ColId: 0, + ColFiltersMatched: filterIds, + XattrIterator: xattrIterator, + XattrKeysForNoCompare: xattrKeysToExclude, + } } - dataSlice := mutationToSerialize.Serialize() - return key, seqno, revId, cas, flags, expiry, opCode, hash, dataSlice, colId, filterIds + ret, err = mutationToSerialize.Serialize() + return key, seqno, revId, cas, flags, expiry, opCode, hash, ret, colId, filterIds, err +} + +func genXattrsWithBody(body string) ([]byte, error) { + var keySize int = randInt(12, 64) + var valueSize int = randInt(15, 70) + var numOfKeyValuePairs int = randInt(5, 10) + var netSize int = (numOfKeyValuePairs * (keySize + valueSize + 6)) + len(body) + 4 + var xattrsPlusBody []byte = make([]byte, netSize) + xattrComposer := xdcrBase.NewXattrComposer(xattrsPlusBody) + for i := 0; i < numOfKeyValuePairs; i++ { + key := randomString(keySize) + value := randomString(valueSize) + err := xattrComposer.WriteKV([]byte(key), []byte(value)) + if err != nil { + return nil, err + } + } + xattrsPlusBody, _ = xattrComposer.FinishAndAppendDocValue([]byte(body)) + return xattrsPlusBody, nil } +func genXattrKeysToExclude() map[string]bool { + keys := []string{eventingKey, ftsKey, indexingKey, queryKey} + randomLen := rand.Int() % 4 + xattrKeysToExclude := make(map[string]bool) + for i := 0; i < randomLen; i++ { + xattrKeysToExclude[keys[i]] = true + } + return xattrKeysToExclude +} func genMultipleRecords(numOfRecords int) []byte { var retSlice []byte for i := 0; i < numOfRecords; i++ { - _, _, _, _, _, _, _, _, record, _, _ := genTestData(true, false) - retSlice = append(retSlice, record...) + if i%2 == 0 { + _, _, _, _, _, _, _, _, record, _, _, err := genTestData(false, true, false) + if err != nil { + i-- + } else { + retSlice = append(retSlice, record...) + } + } else { + _, _, _, _, _, _, _, _, record, _, _, err := genTestData(true, true, false) + if err != nil { + i-- + } else { + retSlice = append(retSlice, record...) + } + } } return retSlice @@ -154,7 +233,12 @@ func genMismatchedFiles(numOfRecords, mismatchCnt int, fileName1, fileName2 stri defer f2.Close() for i := 0; i < mismatchCnt; i++ { - key, seqno, revId, cas, flags, expiry, opCode, _, oneData, colId, _ := genTestData(true, false) + withXattrs := i%2 == 0 + key, seqno, revId, cas, flags, expiry, opCode, _, oneData, colId, _, er := genTestData(withXattrs, true, false) + if er != nil { + i-- + continue + } mismatchedDataMut := &dcp.Mutation{ Vbno: 0, Key: []byte(key), @@ -169,8 +253,11 @@ func genMismatchedFiles(numOfRecords, mismatchCnt int, fileName1, fileName2 stri ColId: colId, ColFiltersMatched: nil, } - mismatchedData := mismatchedDataMut.Serialize() - + mismatchedData, err := mismatchedDataMut.Serialize() + if err != nil { + i-- + continue + } _, err = f1.Write(oneData) if err != nil { return mismatchedKeyNames, err @@ -204,16 +291,21 @@ func verifyMisMatch(mismatchKeys []string, differ *FilesDiffer) bool { } func TestLoader(t *testing.T) { + testLogger := xdcrLog.NewLogger("testLogger", xdcrLog.DefaultLoggerContext) assert := assert.New(t) var outputFileTemp string = "/tmp/xdcrDiffer.tmp" defer os.Remove(outputFileTemp) - - key, seqno, _, _, _, _, _, _, data, _, _ := genTestData(true, false) + withXattrs := rand.Int()%2 == 0 + key, seqno, _, _, _, _, _, _, data, _, _, er := genTestData(withXattrs, true, false) + if er != nil { + fmt.Printf("An error occured while running TestLoader. err: %v", er) + return + } err := ioutil.WriteFile(outputFileTemp, data, 0644) assert.Nil(err) - differ := NewFilesDiffer(outputFileTemp, "", nil, nil, nil) + differ := NewFilesDiffer(outputFileTemp, "", nil, nil, nil, testLogger) err = differ.file1.LoadFileIntoBuffer() assert.Nil(err) @@ -225,16 +317,20 @@ func TestLoader(t *testing.T) { } func TestLoaderWithColFilters(t *testing.T) { + testLogger := xdcrLog.NewLogger("testLogger", xdcrLog.DefaultLoggerContext) assert := assert.New(t) var outputFileTemp string = "/tmp/xdcrDiffer.tmp" defer os.Remove(outputFileTemp) - - key, _, _, _, _, _, _, _, data, _, filterIds := genTestData(true, true) - + withXattrs := rand.Int()%2 == 0 + key, _, _, _, _, _, _, _, data, _, filterIds, er := genTestData(withXattrs, true, true) + if er != nil { + fmt.Printf("An error occured while running TestLoader. err: %v", er) + return + } err := ioutil.WriteFile(outputFileTemp, data, 0644) assert.Nil(err) - differ := NewFilesDiffer(outputFileTemp, "", nil, nil, nil) + differ := NewFilesDiffer(outputFileTemp, "", nil, nil, nil, testLogger) err = differ.file1.LoadFileIntoBuffer() assert.Nil(err) @@ -246,6 +342,7 @@ func TestLoaderWithColFilters(t *testing.T) { } func TestLoadSameFile(t *testing.T) { + testLogger := xdcrLog.NewLogger("testLogger", xdcrLog.DefaultLoggerContext) fmt.Println("============== Test case start: TestLoadSameFile =================") assert := assert.New(t) @@ -259,7 +356,7 @@ func TestLoadSameFile(t *testing.T) { err := genSameFiles(entries, file1, file2) assert.Equal(nil, err) - differ := NewFilesDiffer(file1, file2, nil, nil, nil) + differ := NewFilesDiffer(file1, file2, nil, nil, nil, testLogger) assert.NotNil(differ) srcDiffMap, tgtDiffMap, _, _, _ := differ.Diff() @@ -274,6 +371,7 @@ func TestLoadSameFile(t *testing.T) { // But now that is incorrect and the test is no longer valid func Disabled_TestLoadMismatchedFilesOnly(t *testing.T) { fmt.Println("============== Test case start: TestLoadMismatchedFilesOnly =================") + testLogger := xdcrLog.NewLogger("testLogger", xdcrLog.DefaultLoggerContext) assert := assert.New(t) file1 := "/tmp/test1.bin" @@ -287,7 +385,7 @@ func Disabled_TestLoadMismatchedFilesOnly(t *testing.T) { keys, err := genMismatchedFiles(entries, numMismatch, file1, file2) assert.Nil(err) - differ := NewFilesDiffer(file1, file2, nil, nil, nil) + differ := NewFilesDiffer(file1, file2, nil, nil, nil, testLogger) assert.NotNil(differ) srcDiffMap, tgtDiffMap, _, _, _ := differ.Diff() @@ -310,7 +408,7 @@ func Disabled_TestLoadMismatchedFilesOnly(t *testing.T) { func Disabled_TestLoadMismatchedFilesAndUneven(t *testing.T) { fmt.Println("============== Test case start: TestLoadMismatchedFilesAndUneven =================") assert := assert.New(t) - + testLogger := xdcrLog.NewLogger("testLogger", xdcrLog.DefaultLoggerContext) file1 := "/tmp/test1.bin" file2 := "/tmp/test2.bin" defer os.Remove(file1) @@ -331,7 +429,7 @@ func Disabled_TestLoadMismatchedFilesAndUneven(t *testing.T) { assert.Nil(err) f.Close() - differ := NewFilesDiffer(file1, file2, nil, nil, nil) + differ := NewFilesDiffer(file1, file2, nil, nil, nil, testLogger) assert.NotNil(differ) srcDiffMap, tgtDiffMap, _, _, _ := differ.Diff() @@ -351,7 +449,7 @@ func Disabled_TestLoadMismatchedFilesAndUneven(t *testing.T) { func TestLoadSameFileWPool(t *testing.T) { fmt.Println("============== Test case start: TestLoadSameFileWPool =================") assert := assert.New(t) - + testLogger := xdcrLog.NewLogger("testLogger", xdcrLog.DefaultLoggerContext) fileDescPool := fdp.NewFileDescriptorPool(50) file1 := "/tmp/test1.bin" @@ -364,7 +462,7 @@ func TestLoadSameFileWPool(t *testing.T) { err := genSameFiles(entries, file1, file2) assert.Equal(nil, err) - differ, err := NewFilesDifferWithFDPool(file1, file2, fileDescPool, nil, nil, nil) + differ, err := NewFilesDifferWithFDPool(file1, file2, fileDescPool, nil, nil, nil, testLogger) assert.NotNil(differ) assert.Nil(err) @@ -377,10 +475,134 @@ func TestLoadSameFileWPool(t *testing.T) { func TestNoFilePool(t *testing.T) { fmt.Println("============== Test case start: TestNoFilePool =================") + testLogger := xdcrLog.NewLogger("testLogger", xdcrLog.DefaultLoggerContext) assert := assert.New(t) - differDriver := NewDifferDriver("", "", "", "", 2, 2, 0, nil, nil, nil) + differDriver := NewDifferDriver("", "", "", "", 2, 2, 0, nil, nil, nil, "", "", nil, nil, testLogger) assert.NotNil(differDriver) assert.Nil(differDriver.fileDescPool) fmt.Println("============== Test case end: TestNoFilePool =================") } +func Test_compareHlv(t *testing.T) { + sourcePruningWindow.duration = time.Duration(5) * time.Nanosecond + targetPruningWindow.duration = time.Duration(5) * time.Nanosecond + sourceBucketUUID := hlv.DocumentSourceId(randomString(10)) + targetBucketUUID := hlv.DocumentSourceId(randomString(10)) + type args struct { + hlv1 *hlv.HLV + hlv2 *hlv.HLV + cas1 uint64 + cas2 uint64 + bucketUUID1 hlv.DocumentSourceId + bucketUUID2 hlv.DocumentSourceId + } + tests := []struct { + name string + args args + same bool + }{ + //Test1 : both source and target HLVs are nil + { + name: "HLVs Absent", + args: args{ + hlv1: nil, + hlv2: nil, + cas1: rand.Uint64(), + cas2: rand.Uint64(), + bucketUUID1: sourceBucketUUID, + bucketUUID2: targetBucketUUID, + }, + same: true, + }, + //Test2 : HLV is present at the source + { + name: "HLV present at Source", + args: args{ + hlv1: generateHLV(sourceBucketUUID, 10, 10, targetBucketUUID, 10, nil, nil), + hlv2: nil, + cas1: 10, + cas2: 10, + bucketUUID1: sourceBucketUUID, + bucketUUID2: targetBucketUUID, + }, + same: true, + }, + //Test3 : HLV is present at the target + { + name: "HLV present at Target", + args: args{ + hlv1: nil, + hlv2: generateHLV(targetBucketUUID, 20, 20, sourceBucketUUID, 20, nil, nil), + cas1: 20, + cas2: 20, + bucketUUID1: sourceBucketUUID, + bucketUUID2: targetBucketUUID, + }, + same: true, + }, + //Test4 : both the HLVs are present - with one of them outdated(This case involves implicit construction of HLV) + { + name: "HLV present at both source and target but outdated at source", + args: args{ + hlv1: generateHLV(sourceBucketUUID, 30, 20, targetBucketUUID, 20, hlv.VersionsMap{}, nil), + hlv2: generateHLV(targetBucketUUID, 30, 30, sourceBucketUUID, 30, hlv.VersionsMap{targetBucketUUID: 20}, nil), + cas1: 30, + cas2: 30, + bucketUUID1: sourceBucketUUID, + bucketUUID2: targetBucketUUID, + }, + same: true, + }, + //Test5 : both the HLVs are present - with PV pruned in one of them + { + name: "HLV present at both source and target with Missing PVs(Pruned version at Target) ", + args: args{ + hlv1: generateHLV(sourceBucketUUID, 30, 20, sourceBucketUUID, 20, hlv.VersionsMap{hlv.DocumentSourceId(randomString(10)): 1}, nil), + hlv2: generateHLV(targetBucketUUID, 30, 30, sourceBucketUUID, 30, hlv.VersionsMap{}, nil), + cas1: 30, + cas2: 30, + bucketUUID1: sourceBucketUUID, + bucketUUID2: targetBucketUUID, + }, + same: true, + }, + //Test6: HLV cvCASs missmatch + { + name: "HLV cvCAS mismatch", + args: args{ + hlv1: generateHLV(sourceBucketUUID, 40, 40, sourceBucketUUID, 40, nil, nil), + hlv2: generateHLV(targetBucketUUID, 20, 20, sourceBucketUUID, 20, nil, nil), + cas1: 40, + cas2: 20, + bucketUUID1: sourceBucketUUID, + bucketUUID2: targetBucketUUID, + }, + same: false, + }, + //Test7: PVs present on both source and target with pruning and data mismatch + { + name: "PV pruning+mismatch", + args: args{ + hlv1: generateHLV(sourceBucketUUID, 30, 30, sourceBucketUUID, 30, hlv.VersionsMap{targetBucketUUID: 10, hlv.DocumentSourceId(randomString(10)): 1}, nil), + hlv2: generateHLV(targetBucketUUID, 30, 30, sourceBucketUUID, 30, hlv.VersionsMap{targetBucketUUID: 20}, nil), + cas1: 30, + cas2: 30, + bucketUUID1: sourceBucketUUID, + bucketUUID2: targetBucketUUID, + }, + same: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := compareHlv(tt.args.hlv1, tt.args.hlv2, tt.args.cas1, tt.args.cas2, tt.args.bucketUUID1, tt.args.bucketUUID2); got != tt.same { + t.Errorf("compareHlv() = %v, want %v", got, tt.same) + } + }) + } +} + +func generateHLV(source hlv.DocumentSourceId, cas uint64, cvCas uint64, src hlv.DocumentSourceId, ver uint64, pv hlv.VersionsMap, mv hlv.VersionsMap) *hlv.HLV { + HLV, _ := hlv.NewHLV(source, cas, cvCas, src, ver, pv, mv) + return HLV +}