From 54ee85b3340aae3e5b68afa777314a59bd3f8abe Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Fri, 18 Sep 2020 17:39:38 -0400
Subject: [PATCH 01/25] Add rawSeqStore_t member to ZSTD_matchstate_t

---
 lib/compress/zstd_compress_internal.h | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index b161a208cf0..28aedd6818b 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -131,6 +131,19 @@ typedef struct {
     U32 lowLimit;           /* below that point, no more valid data */
 } ZSTD_window_t;
 
+typedef struct {
+    U32 offset;
+    U32 litLength;
+    U32 matchLength;
+} rawSeq;
+
+typedef struct {
+  rawSeq* seq;     /* The start of the sequences */
+  size_t pos;      /* The position where reading stopped. <= size. */
+  size_t size;     /* The number of sequences. <= capacity. */
+  size_t capacity; /* The capacity starting from `seq` pointer */
+} rawSeqStore_t;
+
 typedef struct ZSTD_matchState_t ZSTD_matchState_t;
 struct ZSTD_matchState_t {
     ZSTD_window_t window;   /* State for window round buffer management */
@@ -150,6 +163,7 @@ struct ZSTD_matchState_t {
                                * dedicated dictionary search structure.
                                */
     optState_t opt;         /* optimal parser state */
+    rawSeqStore_t ldmSeqStore; /* raw seq store containing LDMs */
     const ZSTD_matchState_t* dictMatchState;
     ZSTD_compressionParameters cParams;
 };
@@ -183,19 +197,6 @@ typedef struct {
     U32 windowLog;          /* Window log for the LDM */
 } ldmParams_t;
 
-typedef struct {
-    U32 offset;
-    U32 litLength;
-    U32 matchLength;
-} rawSeq;
-
-typedef struct {
-  rawSeq* seq;     /* The start of the sequences */
-  size_t pos;      /* The position where reading stopped. <= size. */
-  size_t size;     /* The number of sequences. <= capacity. */
-  size_t capacity; /* The capacity starting from `seq` pointer */
-} rawSeqStore_t;
-
 typedef struct {
     int collectSequences;
     ZSTD_Sequence* seqStart;

From 13c08d3adc9f03d820fef1fbee4073a6092e5195 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Fri, 18 Sep 2020 17:46:19 -0400
Subject: [PATCH 02/25] Add call to use opt parser early when in long mode

---
 lib/compress/zstd_ldm.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index dbfce3dce7f..28ceec1d047 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -576,6 +576,15 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
     /* Input positions */
     BYTE const* ip = istart;
 
+    /* If compression strategy uses optimal parser, use LDMs only as candidates
+     * rather than accepting all LDMs and calling regular match finder on literal
+     * blocks in between.
+     */
+    if (cParams->strategy >= ZSTD_btopt) {
+        ms->ldmSeqStore = *rawSeqStore;
+        return blockCompressor(ms, seqStore, rep, src, srcSize);
+    }
+
     DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize);
     assert(rawSeqStore->pos <= rawSeqStore->size);
     assert(rawSeqStore->size <= rawSeqStore->capacity);

From e348c2d6d2120940de1fb108b1853f2731cb02a0 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Fri, 18 Sep 2020 17:53:10 -0400
Subject: [PATCH 03/25] convertSeqStoreToRanges() static function explanation

---
 lib/compress/zstd_ldm.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 28ceec1d047..7832f673d58 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -562,6 +562,22 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
     return sequence;
 }
 
+/**
+ * Converts the elements of a rawSeqStore into a series of ranges representing
+ * the beginning and end of a match. We store the start of a match in "litLength"
+ * and end of a match in "matchLength". So a rawSeqStore containing:
+ * (litLength: 1000, matchLength: 500)
+ * (litLength: 2000, matchLength: 1000)
+ * (litLength: 4000, matchLength: 1000)
+ * would be converted into:
+ * (matchStart: 1000, matchEnd: 3500)
+ * (matchStart: 3500, matchEnd: 4500)
+ * (matchStart: 8500, matchEnd: 9500)
+ */
+static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore) {
+
+}
+
 size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
     ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
     void const* src, size_t srcSize)

From 4ef590531cf6b430a60a3181531a956b5170b75b Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Fri, 18 Sep 2020 17:58:31 -0400
Subject: [PATCH 04/25] Implement convertSeqStoreToRanges()

---
 lib/compress/zstd_ldm.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 7832f673d58..5ea90122001 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -575,7 +575,18 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
  * (matchStart: 8500, matchEnd: 9500)
  */
 static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore) {
-
+    size_t i;
+    size_t currPos = 0;
+    for( ; i < rawSeqStore->size; ++i) {
+        size_t matchStart;
+        size_t matchEnd;
+        currPos += rawSeqStore->seq[i].litLength;
+        matchStart = currPos;
+        currPos += rawSeqStore->seq[i].matchLength;
+        matchEnd = currPos;
+        rawSeqStore->seq[i].litLength = matchStart;
+        rawSeqStore->seq[i].matchLength = matchEnd;
+    }
 }
 
 size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,

From addbdba5421ff5e3105ee57dd4e6940df7e16799 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Sat, 19 Sep 2020 18:37:21 -0400
Subject: [PATCH 05/25] Add getNextLdm() and callsite on first match

---
 lib/compress/zstd_ldm.c |  6 +++++-
 lib/compress/zstd_opt.c | 20 +++++++++++++++++++-
 2 files changed, 24 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 5ea90122001..aa096c1c994 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -569,8 +569,10 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
  * (litLength: 1000, matchLength: 500)
  * (litLength: 2000, matchLength: 1000)
  * (litLength: 4000, matchLength: 1000)
+ * 
  * would be converted into:
- * (matchStart: 1000, matchEnd: 3500)
+ * 
+ * (matchStart: 1000, matchEnd: 1500)
  * (matchStart: 3500, matchEnd: 4500)
  * (matchStart: 8500, matchEnd: 9500)
  */
@@ -608,6 +610,8 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
      * blocks in between.
      */
     if (cParams->strategy >= ZSTD_btopt) {
+        convertSeqStoreToRanges(rawSeqStore);
+        //rawSeqStore->pos = ??? /* represents the startOfLdmSeqStore */
         ms->ldmSeqStore = *rawSeqStore;
         return blockCompressor(ms, seqStore, rep, src, srcSize);
     }
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 5acc9e0b680..8d205e41704 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -763,7 +763,17 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
     case 6 : return ZSTD_insertBtAndGetAllMatches(matches, ms, nextToUpdate3, ip, iHighLimit, dictMode, rep, ll0, lengthToBeat, 6);
     }
 }
+/*********************************
+*  LDM functions
+*********************************/
+
+static void getNextLdm(U32* ldmStart, U32* ldmEnd, const rawSeqStore_t* const ldmSeqStore,
+                            U32 relativePos, U32 startBlockIdx) {
+    *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
+    *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
 
+    /* Handle block splitting */
+}
 
 /*-*******************************
 *  Optimal parser
@@ -813,6 +823,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1);
     U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
     U32 nextToUpdate3 = ms->nextToUpdate;
+    U32 const startBlockIdx = (U32)(istart - base);
+    U32 currLdmStart = 0;
+    U32 currLdmEnd = 0;
 
     ZSTD_optimal_t* const opt = optStatePtr->priceTable;
     ZSTD_match_t* const matches = optStatePtr->matchTable;
@@ -832,7 +845,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
         /* find first match */
         {   U32 const litlen = (U32)(ip - anchor);
             U32 const ll0 = !litlen;
-            U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
+            U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
+            if ((U32)(ip - base) >= currLdmEnd) {
+                getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore,
+                                &ms->ldmSeqStore + (U32)(ip - base), startBlockIdx);
+            }
+
             if (!nbMatches) { ip++; continue; }
 
             /* initialize opt[0] */

From 305d7a503c5cf6b15a16ec7caf277d3771a1933a Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Sat, 19 Sep 2020 18:40:45 -0400
Subject: [PATCH 06/25] Add getNextLdm() callsite for next looped matches

---
 lib/compress/zstd_opt.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 8d205e41704..96f4db8598a 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -959,6 +959,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                 continue;  /* skip unpromising positions; about ~+6% speed, -0.01 ratio */
             }
 
+            {   if ((U32)(inr - base) >= currLdmEnd) {
+                    getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore,
+                               &ms->ldmSeqStore + (U32)(inr - base), startBlockIdx);
+                }
+            }
+
             {   U32 const ll0 = (opt[cur].mlen != 0);
                 U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0;
                 U32 const previousPrice = opt[cur].price;

From c02b161a5d6a29c8066f8f11d0badc3e707540f4 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Sat, 19 Sep 2020 18:53:13 -0400
Subject: [PATCH 07/25] Add callsite logic for maybeAddLdm()

---
 lib/compress/zstd_opt.c | 40 ++++++++++++++++++++++++++++++----------
 1 file changed, 30 insertions(+), 10 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 96f4db8598a..f6c207f8cbf 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -767,14 +767,20 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
 *  LDM functions
 *********************************/
 
-static void getNextLdm(U32* ldmStart, U32* ldmEnd, const rawSeqStore_t* const ldmSeqStore,
+/* TODO: Increment by SBI */
+static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
                             U32 relativePos, U32 startBlockIdx) {
     *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
     *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
+    ldmSeqStore->pos++;
 
     /* Handle block splitting */
 }
 
+static void maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U328 ldmStart, U32* ldmEnd) {
+
+}
+
 /*-*******************************
 *  Optimal parser
 *********************************/
@@ -845,10 +851,18 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
         /* find first match */
         {   U32 const litlen = (U32)(ip - anchor);
             U32 const ll0 = !litlen;
-            U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
-            if ((U32)(ip - base) >= currLdmEnd) {
+            U32 const current = (U32)(ip - base);
+
+            /* Fetch next LDM if necessary */
+            if (current >= currLdmEnd) {
                 getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore,
-                                &ms->ldmSeqStore + (U32)(ip - base), startBlockIdx);
+                           ms->ldmSeqStore.capacity + current, startBlockIdx);
+            }
+
+            U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
+            
+            if (current >= currLdmStart) {
+                maybeAddLdm(matches, &nbMatches, &currLdmStart, &currLdmEnd);
             }
 
             if (!nbMatches) { ip++; continue; }
@@ -908,6 +922,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
         /* check further positions */
         for (cur = 1; cur <= last_pos; cur++) {
             const BYTE* const inr = ip + cur;
+            U32 const current = (U32)(inr - base);
             assert(cur < ZSTD_OPT_NUM);
             DEBUGLOG(7, "cPos:%zi==rPos:%u", inr-istart, cur)
 
@@ -959,17 +974,22 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                 continue;  /* skip unpromising positions; about ~+6% speed, -0.01 ratio */
             }
 
-            {   if ((U32)(inr - base) >= currLdmEnd) {
-                    getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore,
-                               &ms->ldmSeqStore + (U32)(inr - base), startBlockIdx);
-                }
-            }
-
             {   U32 const ll0 = (opt[cur].mlen != 0);
                 U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0;
                 U32 const previousPrice = opt[cur].price;
                 U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
+
+                if (current >= currLdmEnd) {
+                    getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore,
+                               ms->ldmSeqStore.capacity + current, startBlockIdx);
+                }
+                
                 U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
+
+                if (current >= currLdmStart) {
+                    maybeAddLdm(matches, &nbMatches, &currLdmStart, &currLdmEnd);
+                }
+
                 U32 matchNb;
                 if (!nbMatches) {
                     DEBUGLOG(7, "rPos:%u : no match found", cur);

From 8e819104bd92c6173719876beb30a2e2f768f52a Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Sun, 20 Sep 2020 11:46:28 -0400
Subject: [PATCH 08/25] Getting correct LDMs for single-threaded case

---
 lib/compress/zstd_ldm.c | 22 ++++++++++++++++++----
 lib/compress/zstd_opt.c | 25 +++++++++++++++----------
 2 files changed, 33 insertions(+), 14 deletions(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index aa096c1c994..451fc684596 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -576,18 +576,27 @@ static rawSeq maybeSplitSequence(rawSeqStore_t* rawSeqStore,
  * (matchStart: 3500, matchEnd: 4500)
  * (matchStart: 8500, matchEnd: 9500)
  */
+
+static void printSeqStore(rawSeqStore_t* rawSeqStore) {
+    printf("rawSeqStore: pos: %zu\n", rawSeqStore->pos);
+    for (int i = 0; i < rawSeqStore->size; ++i) {
+        printf("(of:%u ml:%u ll: %u)\n", rawSeqStore->seq[i].offset, rawSeqStore->seq[i].matchLength, rawSeqStore->seq[i].litLength);
+    }
+}
 static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore) {
     size_t i;
     size_t currPos = 0;
-    for( ; i < rawSeqStore->size; ++i) {
+    printf("Conversion...\n");
+    for(i = 0 ; i < rawSeqStore->size; ++i) {
         size_t matchStart;
         size_t matchEnd;
         currPos += rawSeqStore->seq[i].litLength;
         matchStart = currPos;
         currPos += rawSeqStore->seq[i].matchLength;
         matchEnd = currPos;
-        rawSeqStore->seq[i].litLength = matchStart;
-        rawSeqStore->seq[i].matchLength = matchEnd;
+        rawSeqStore->seq[i].matchLength = matchStart;
+        rawSeqStore->seq[i].litLength = matchEnd;
+        printf("(%u, %u)\n", matchStart, matchEnd);
     }
 }
 
@@ -610,10 +619,15 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
      * blocks in between.
      */
     if (cParams->strategy >= ZSTD_btopt) {
+        printf("start of this seqstore: %u\n", (U32)(istart - ms->window.base));
+        size_t cLen;
+        printSeqStore(rawSeqStore);
         convertSeqStoreToRanges(rawSeqStore);
+        rawSeqStore->capacity = (U32)(istart - ms->window.base);
         //rawSeqStore->pos = ??? /* represents the startOfLdmSeqStore */
         ms->ldmSeqStore = *rawSeqStore;
-        return blockCompressor(ms, seqStore, rep, src, srcSize);
+        cLen = blockCompressor(ms, seqStore, rep, src, srcSize);
+        return cLen;
     }
 
     DEBUGLOG(5, "ZSTD_ldm_blockCompress: srcSize=%zu", srcSize);
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index f6c207f8cbf..f1ec7c8a21b 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -769,15 +769,18 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
 
 /* TODO: Increment by SBI */
 static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
-                            U32 relativePos, U32 startBlockIdx) {
+                       const U32 ldmSeqStoreStartPos, U32 currPosInBlock) {
+    if (ldmSeqStore->pos >= ldmSeqStore->size)
+        return;
+    printf("getNextLdm(): start of seqStore: %u, startBlockIdx: %u ", ldmSeqStoreStartPos, currPosInBlock);
     *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
     *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
+    printf("-- range: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd, ldmSeqStore->pos);
     ldmSeqStore->pos++;
-
     /* Handle block splitting */
 }
 
-static void maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U328 ldmStart, U32* ldmEnd) {
+static void maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32* ldmStart, U32* ldmEnd) {
 
 }
 
@@ -830,6 +833,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
     U32 nextToUpdate3 = ms->nextToUpdate;
     U32 const startBlockIdx = (U32)(istart - base);
+    
     U32 currLdmStart = 0;
     U32 currLdmEnd = 0;
 
@@ -840,6 +844,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     /* init */
     DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
                 (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
+    printf("ZSTD_compressBlock_opt_generic: current=%u, sbi=%u\n", (U32)(ip - base), startBlockIdx);
     assert(optLevel <= 2);
     ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
     ip += (ip==prefixStart);
@@ -854,14 +859,14 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
             U32 const current = (U32)(ip - base);
 
             /* Fetch next LDM if necessary */
-            if (current >= currLdmEnd) {
+            if (ms->ldmSeqStore.size > 0 && current >= currLdmEnd + startBlockIdx) {
                 getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore,
-                           ms->ldmSeqStore.capacity + current, startBlockIdx);
+                           ms->ldmSeqStore.capacity, startBlockIdx);
             }
 
             U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
             
-            if (current >= currLdmStart) {
+            if (current >= currLdmStart + startBlockIdx) {
                 maybeAddLdm(matches, &nbMatches, &currLdmStart, &currLdmEnd);
             }
 
@@ -979,12 +984,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                 U32 const previousPrice = opt[cur].price;
                 U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
 
-                if (current >= currLdmEnd) {
+                if (ms->ldmSeqStore.size > 0 && current >= currLdmEnd + startBlockIdx) {
                     getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore,
-                               ms->ldmSeqStore.capacity + current, startBlockIdx);
+                               ms->ldmSeqStore.capacity, (U32)(ip - istart));
                 }
-                
-                U32 const nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
+
+                U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
 
                 if (current >= currLdmStart) {
                     maybeAddLdm(matches, &nbMatches, &currLdmStart, &currLdmEnd);

From 0eb0c0a5d7700c4b01d562c020382ba0162a773c Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Sun, 20 Sep 2020 12:22:38 -0400
Subject: [PATCH 09/25] Implement maybeAddLdm(), working roundtrip for
 --single-thread

---
 lib/compress/zstd_opt.c | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index f1ec7c8a21b..bb616f76848 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -780,8 +780,24 @@ static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
     /* Handle block splitting */
 }
 
-static void maybeAddLdm(ZSTD_match_t* matches, U32* nbMatches, U32* ldmStart, U32* ldmEnd) {
-
+static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* matches,
+                        U32* nbMatches, U32 ldmStart, U32 ldmEnd, U32 current, U32 startBlockIdx) {
+    ldmStart += startBlockIdx;
+    ldmEnd += startBlockIdx;
+    U32 posDifference = current - ldmStart;
+    if (posDifference > 0) {
+        return; /* don't handle extra LDMs for now */
+    }
+    assert(ldmSeqStore->pos > 0);
+    printf("Considering LDM @ current = %u, posDiff = %u - ", current, posDifference);
+    U32 candidateOffset = ldmSeqStore->seq[ldmSeqStore->pos - 1].offset + posDifference;
+    U32 candidateMatchLength = (ldmEnd - ldmStart) - posDifference;
+    printf("adjusted to (of: %u, ml %u)\n", candidateOffset, candidateMatchLength);
+    if (candidateMatchLength >= matches[*nbMatches-1].len) {
+        matches[*nbMatches].len = candidateMatchLength;
+        matches[*nbMatches].off = candidateOffset + ZSTD_REP_MOVE;
+        (*nbMatches)++;
+    }
 }
 
 /*-*******************************
@@ -866,8 +882,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
 
             U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
             
-            if (current >= currLdmStart + startBlockIdx) {
-                maybeAddLdm(matches, &nbMatches, &currLdmStart, &currLdmEnd);
+            if (ms->ldmSeqStore.size > 0 && current >= currLdmStart + startBlockIdx) {
+                maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
             }
 
             if (!nbMatches) { ip++; continue; }
@@ -991,8 +1007,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
 
                 U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
 
-                if (current >= currLdmStart) {
-                    maybeAddLdm(matches, &nbMatches, &currLdmStart, &currLdmEnd);
+                if (ms->ldmSeqStore.size > 0 && current >= currLdmStart + startBlockIdx) {
+                    maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
                 }
 
                 U32 matchNb;

From 0fd3dfc515bb6bbb9141b337f7dd8dbd7f0e6e23 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Sun, 20 Sep 2020 12:33:49 -0400
Subject: [PATCH 10/25] Add logic to consider extra LDM candidates too

---
 lib/compress/zstd_opt.c | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index bb616f76848..0ebd4bda128 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -785,13 +785,19 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
     ldmStart += startBlockIdx;
     ldmEnd += startBlockIdx;
     U32 posDifference = current - ldmStart;
-    if (posDifference > 0) {
-        return; /* don't handle extra LDMs for now */
+    U32 originalMatchLength = ldmEnd - ldmStart;
+    if (posDifference >= originalMatchLength) {
+        return;
     }
+
     assert(ldmSeqStore->pos > 0);
     printf("Considering LDM @ current = %u, posDiff = %u - ", current, posDifference);
     U32 candidateOffset = ldmSeqStore->seq[ldmSeqStore->pos - 1].offset + posDifference;
-    U32 candidateMatchLength = (ldmEnd - ldmStart) - posDifference;
+    U32 candidateMatchLength = originalMatchLength - posDifference;
+    if (candidateMatchLength < ZSTD_LDM_MINMATCH_MIN) {
+        printf("too small\n");
+        return;
+    }
     printf("adjusted to (of: %u, ml %u)\n", candidateOffset, candidateMatchLength);
     if (candidateMatchLength >= matches[*nbMatches-1].len) {
         matches[*nbMatches].len = candidateMatchLength;
@@ -882,7 +888,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
 
             U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
             
-            if (ms->ldmSeqStore.size > 0 && current >= currLdmStart + startBlockIdx) {
+            if (ms->ldmSeqStore.size > 0 && current >= currLdmStart + startBlockIdx && current < currLdmEnd + startBlockIdx) {
                 maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
             }
 

From 8dce3f8f213d8915363cbb5c30da7391d7442056 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Sun, 20 Sep 2020 12:36:45 -0400
Subject: [PATCH 11/25] Add maybeAddLdm() logic to non-first matches too

---
 lib/compress/zstd_opt.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 0ebd4bda128..d69d48cfa94 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -1013,9 +1013,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
 
                 U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
 
-                if (ms->ldmSeqStore.size > 0 && current >= currLdmStart + startBlockIdx) {
-                    maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
-                }
+                if (ms->ldmSeqStore.size > 0 && current >= currLdmStart + startBlockIdx && current < currLdmEnd + startBlockIdx) {
+                maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
+            }
 
                 U32 matchNb;
                 if (!nbMatches) {

From 878086548fef18a3b37213efffdb2dec08f7fff3 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Sun, 20 Sep 2020 18:04:45 -0400
Subject: [PATCH 12/25] Add new field to rawSeqStore_t to flag modification

---
 lib/compress/zstd_compress.c          |  4 +++-
 lib/compress/zstd_compress_internal.h |  7 +++++++
 lib/compress/zstd_ldm.c               | 13 ++++++++-----
 lib/compress/zstd_opt.c               | 19 +++++++++++++++----
 lib/compress/zstdmt_compress.c        |  4 ++--
 5 files changed, 35 insertions(+), 12 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 8c8c33a859d..31bd2405c81 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2328,6 +2328,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
             for (i = 0; i < ZSTD_REP_NUM; ++i)
                 zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i];
         }
+        printf("Deciding....\n");
         if (zc->externSeqStore.pos < zc->externSeqStore.size) {
             assert(!zc->appliedParams.ldmParams.enableLdm);
             /* Updates ldmSeqStore.pos */
@@ -2338,7 +2339,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
                                        src, srcSize);
             assert(zc->externSeqStore.pos <= zc->externSeqStore.size);
         } else if (zc->appliedParams.ldmParams.enableLdm) {
-            rawSeqStore_t ldmSeqStore = {NULL, 0, 0, 0};
+            rawSeqStore_t ldmSeqStore = {NULL, 0, 0, 0, 0};
 
             ldmSeqStore.seq = zc->ldmSequences;
             ldmSeqStore.capacity = zc->maxNbLdmSequences;
@@ -2360,6 +2361,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
         {   const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize;
             ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize);
     }   }
+    printf("Done build ldm seq store\n");
     return ZSTDbss_compress;
 }
 
diff --git a/lib/compress/zstd_compress_internal.h b/lib/compress/zstd_compress_internal.h
index 28aedd6818b..a982dbb8492 100644
--- a/lib/compress/zstd_compress_internal.h
+++ b/lib/compress/zstd_compress_internal.h
@@ -142,6 +142,13 @@ typedef struct {
   size_t pos;      /* The position where reading stopped. <= size. */
   size_t size;     /* The number of sequences. <= capacity. */
   size_t capacity; /* The capacity starting from `seq` pointer */
+
+  U32 rangeFlag;   /* If == 1, then members of this rawSeqStore represent different things:
+                    * seq.matchLength == start of a match
+                    * seq.litLength == end of a match
+                    * capacity == reference start index of this ldm seq store
+                    */
+                   
 } rawSeqStore_t;
 
 typedef struct ZSTD_matchState_t ZSTD_matchState_t;
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 451fc684596..81aac7c1a99 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -587,6 +587,7 @@ static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore) {
     size_t i;
     size_t currPos = 0;
     printf("Conversion...\n");
+    rawSeqStore->rangeFlag = 1;
     for(i = 0 ; i < rawSeqStore->size; ++i) {
         size_t matchStart;
         size_t matchEnd;
@@ -619,12 +620,14 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
      * blocks in between.
      */
     if (cParams->strategy >= ZSTD_btopt) {
-        printf("start of this seqstore: %u\n", (U32)(istart - ms->window.base));
+        printf("ldmSeqStore start idx: %u\n", (U32)(istart - ms->window.base));
         size_t cLen;
-        printSeqStore(rawSeqStore);
-        convertSeqStoreToRanges(rawSeqStore);
-        rawSeqStore->capacity = (U32)(istart - ms->window.base);
-        //rawSeqStore->pos = ??? /* represents the startOfLdmSeqStore */
+        if (!(*rawSeqStore).rangeFlag) {
+            /* only convert the rawSeqStore once, in case it spans multiple blocks */
+            printSeqStore(rawSeqStore);
+            convertSeqStoreToRanges(rawSeqStore);   /* sets rangeFlag to true */
+        }
+        (*rawSeqStore).capacity = (U32)(istart - ms->window.base);
         ms->ldmSeqStore = *rawSeqStore;
         cLen = blockCompressor(ms, seqStore, rep, src, srcSize);
         return cLen;
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index d69d48cfa94..262050a71b3 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -856,8 +856,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     U32 nextToUpdate3 = ms->nextToUpdate;
     U32 const startBlockIdx = (U32)(istart - base);
     
-    U32 currLdmStart = 0;
-    U32 currLdmEnd = 0;
+    U32 currLdmStart;
+    U32 currLdmEnd;
 
     ZSTD_optimal_t* const opt = optStatePtr->priceTable;
     ZSTD_match_t* const matches = optStatePtr->matchTable;
@@ -871,6 +871,14 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
     ip += (ip==prefixStart);
 
+    /* Set current LDM candidate to whatever might have been considered in prev block */
+    if (ms->ldmSeqStore.pos != 0) {
+        currLdmStart = ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].matchLength;
+        currLdmEnd = ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].litLength;
+    } else {
+        currLdmStart = currLdmEnd = 0;
+    }
+
     /* Match Loop */
     while (ip < ilimit) {
         U32 cur, last_pos = 0;
@@ -882,6 +890,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
 
             /* Fetch next LDM if necessary */
             if (ms->ldmSeqStore.size > 0 && current >= currLdmEnd + startBlockIdx) {
+                printf("Getting next ldm: current: %u and currLdmEnd + sbi: %u\n", current, currLdmEnd + startBlockIdx);
                 getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore,
                            ms->ldmSeqStore.capacity, startBlockIdx);
             }
@@ -889,6 +898,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
             U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
             
             if (ms->ldmSeqStore.size > 0 && current >= currLdmStart + startBlockIdx && current < currLdmEnd + startBlockIdx) {
+                printf("Adding LDM\n");
                 maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
             }
 
@@ -1014,8 +1024,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                 U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
 
                 if (ms->ldmSeqStore.size > 0 && current >= currLdmStart + startBlockIdx && current < currLdmEnd + startBlockIdx) {
-                maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
-            }
+                    maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
+                }
 
                 U32 matchNb;
                 if (!nbMatches) {
@@ -1132,6 +1142,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
         }
     }   /* while (ip < ilimit) */
 
+    printf("Finished opt\n");
     /* Return the last literals size */
     return (size_t)(iend - anchor);
 }
diff --git a/lib/compress/zstdmt_compress.c b/lib/compress/zstdmt_compress.c
index baf6ef4ca6d..f789ae5bfdd 100644
--- a/lib/compress/zstdmt_compress.c
+++ b/lib/compress/zstdmt_compress.c
@@ -266,7 +266,7 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf)
 
 /* =====   Seq Pool Wrapper   ====== */
 
-static rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0};
+static rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0};
 
 typedef ZSTDMT_bufferPool ZSTDMT_seqPool;
 
@@ -277,7 +277,7 @@ static size_t ZSTDMT_sizeof_seqPool(ZSTDMT_seqPool* seqPool)
 
 static rawSeqStore_t bufferToSeq(buffer_t buffer)
 {
-    rawSeqStore_t seq = {NULL, 0, 0, 0};
+    rawSeqStore_t seq = {NULL, 0, 0, 0, 0};
     seq.seq = (rawSeq*)buffer.start;
     seq.capacity = buffer.capacity / sizeof(rawSeq);
     return seq;

From 1d28e4356eb85f2326cb4aefd07a8f76a32bb483 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Mon, 21 Sep 2020 17:42:03 -0400
Subject: [PATCH 13/25] Almost working absolute seq store extension

---
 lib/compress/zstd_ldm.c | 28 ++++++++++++--
 lib/compress/zstd_opt.c | 86 +++++++++++++++++++++++++++--------------
 2 files changed, 81 insertions(+), 33 deletions(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 81aac7c1a99..f95a69bd32f 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -583,9 +583,9 @@ static void printSeqStore(rawSeqStore_t* rawSeqStore) {
         printf("(of:%u ml:%u ll: %u)\n", rawSeqStore->seq[i].offset, rawSeqStore->seq[i].matchLength, rawSeqStore->seq[i].litLength);
     }
 }
-static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore) {
+static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore, size_t srcSize) {
     size_t i;
-    size_t currPos = 0;
+    size_t currPos = 1;
     printf("Conversion...\n");
     rawSeqStore->rangeFlag = 1;
     for(i = 0 ; i < rawSeqStore->size; ++i) {
@@ -599,6 +599,19 @@ static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore) {
         rawSeqStore->seq[i].litLength = matchEnd;
         printf("(%u, %u)\n", matchStart, matchEnd);
     }
+    if (rawSeqStore->seq[rawSeqStore->size - 1].litLength >= srcSize) {
+        printf("SETTING RANGEFLAG TO 2\n");
+        rawSeqStore->rangeFlag = 2; /* Signifies that this is a seqstore that spans
+                                           multiple blocks. */
+    }
+}
+
+static void adjustLdmSeqStore(rawSeqStore_t* rawSeqStore, int baseDiff) {
+    size_t i = 0;
+    for (i; i < rawSeqStore->size; ++i) {
+        rawSeqStore->seq[i].matchLength += (size_t)baseDiff;
+        rawSeqStore->seq[i].litLength += (size_t)baseDiff;
+    }
 }
 
 size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
@@ -622,14 +635,21 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
     if (cParams->strategy >= ZSTD_btopt) {
         printf("ldmSeqStore start idx: %u\n", (U32)(istart - ms->window.base));
         size_t cLen;
-        if (!(*rawSeqStore).rangeFlag) {
+        if ((*rawSeqStore).rangeFlag == 0) {
             /* only convert the rawSeqStore once, in case it spans multiple blocks */
             printSeqStore(rawSeqStore);
-            convertSeqStoreToRanges(rawSeqStore);   /* sets rangeFlag to true */
+            convertSeqStoreToRanges(rawSeqStore, srcSize);   /* sets rangeFlag to true */
         }
         (*rawSeqStore).capacity = (U32)(istart - ms->window.base);
+        const BYTE* const prevBase = (BYTE const*)ms->window.base;
         ms->ldmSeqStore = *rawSeqStore;
         cLen = blockCompressor(ms, seqStore, rep, src, srcSize);
+        if (prevBase != ms->window.base) {
+            int baseDiff = (int)(prevBase - ms->window.base);
+            printf("Bases were different, adjusting, diff = %d\n", baseDiff);
+            adjustLdmSeqStore(rawSeqStore, baseDiff);
+            printSeqStore(rawSeqStore);
+        }
         return cLen;
     }
 
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 262050a71b3..6d3442fd290 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -769,28 +769,69 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
 
 /* TODO: Increment by SBI */
 static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
-                       const U32 ldmSeqStoreStartPos, U32 currPosInBlock) {
-    if (ldmSeqStore->pos >= ldmSeqStore->size)
+                       const U32 ldmSeqStoreStartPos, U32 currPosInBlock, U32 current, U32 startBlockIdx) {
+    if (ldmSeqStore->pos >= ldmSeqStore->size ||
+        ldmSeqStore->size == 0 || ldmSeqStore->rangeFlag == 0) 
         return;
-    printf("getNextLdm(): start of seqStore: %u, startBlockIdx: %u ", ldmSeqStoreStartPos, currPosInBlock);
-    *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
-    *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
+
+    if (ldmSeqStore->rangeFlag == 2) {
+        if (current >= *ldmEnd) {
+            printf("Getting next ldm: current: %u and currLdmEnd + sbi - 1: %u\n", current, *ldmEnd + startBlockIdx - 1);
+            *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
+            *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
+            printf("Final range after adjusting for absolute ldm seq store: (%u, %u)\n", *ldmStart, *ldmEnd);
+        } else {
+            return;
+        }
+    } else {
+        if (current >= *ldmEnd + startBlockIdx - 1) {
+            printf("Getting next ldm: current: %u and currLdmEnd + sbi - 1: %u\n", current, *ldmEnd + startBlockIdx - 1);
+            *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
+            *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
+            printf("Final range: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd, ldmSeqStore->pos);
+        } else {
+            return;
+        }
+    }
+
+   
+    printf("getNextLdm(): start of seqStore: %u, currPosInBlock: %u ", ldmSeqStoreStartPos, currPosInBlock);
     printf("-- range: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd, ldmSeqStore->pos);
     ldmSeqStore->pos++;
+
     /* Handle block splitting */
 }
 
 static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* matches,
                         U32* nbMatches, U32 ldmStart, U32 ldmEnd, U32 current, U32 startBlockIdx) {
-    ldmStart += startBlockIdx;
-    ldmEnd += startBlockIdx;
+    if (ldmSeqStore->size == 0)
+        return;
+    if (ldmSeqStore->rangeFlag == 1) {
+        if (!(current >= ldmStart + startBlockIdx - 1) || !(current < ldmEnd + startBlockIdx - 1))
+            return;
+    } else if (ldmSeqStore->rangeFlag == 2) {
+        if (!(current >= ldmStart) || !(current < ldmEnd))
+            return;
+    }
+    printf("About to maybe add an ldm...\n");
+
+    if (ldmSeqStore->rangeFlag == 1) {
+        ldmStart += startBlockIdx - 1;
+        ldmEnd += startBlockIdx - 1;
+    }
+
     U32 posDifference = current - ldmStart;
     U32 originalMatchLength = ldmEnd - ldmStart;
     if (posDifference >= originalMatchLength) {
+        printf("posdiff greater than matchlen!\n");
         return;
     }
 
     assert(ldmSeqStore->pos > 0);
+    if (posDifference > 0) {
+        return;
+    }
+
     printf("Considering LDM @ current = %u, posDiff = %u - ", current, posDifference);
     U32 candidateOffset = ldmSeqStore->seq[ldmSeqStore->pos - 1].offset + posDifference;
     U32 candidateMatchLength = originalMatchLength - posDifference;
@@ -800,6 +841,7 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
     }
     printf("adjusted to (of: %u, ml %u)\n", candidateOffset, candidateMatchLength);
     if (candidateMatchLength >= matches[*nbMatches-1].len) {
+        printf("large enough, adding\n");
         matches[*nbMatches].len = candidateMatchLength;
         matches[*nbMatches].off = candidateOffset + ZSTD_REP_MOVE;
         (*nbMatches)++;
@@ -870,6 +912,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     assert(optLevel <= 2);
     ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
     ip += (ip==prefixStart);
+    int ldmAdjusted = 0;
 
     /* Set current LDM candidate to whatever might have been considered in prev block */
     if (ms->ldmSeqStore.pos != 0) {
@@ -888,19 +931,9 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
             U32 const ll0 = !litlen;
             U32 const current = (U32)(ip - base);
 
-            /* Fetch next LDM if necessary */
-            if (ms->ldmSeqStore.size > 0 && current >= currLdmEnd + startBlockIdx) {
-                printf("Getting next ldm: current: %u and currLdmEnd + sbi: %u\n", current, currLdmEnd + startBlockIdx);
-                getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore,
-                           ms->ldmSeqStore.capacity, startBlockIdx);
-            }
-
+            getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore, ms->ldmSeqStore.capacity, (U32)(ip-istart), current, startBlockIdx);
             U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
-            
-            if (ms->ldmSeqStore.size > 0 && current >= currLdmStart + startBlockIdx && current < currLdmEnd + startBlockIdx) {
-                printf("Adding LDM\n");
-                maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
-            }
+            maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
 
             if (!nbMatches) { ip++; continue; }
 
@@ -1015,17 +1048,12 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                 U32 const litlen = (opt[cur].mlen == 0) ? opt[cur].litlen : 0;
                 U32 const previousPrice = opt[cur].price;
                 U32 const basePrice = previousPrice + ZSTD_litLengthPrice(0, optStatePtr, optLevel);
-
-                if (ms->ldmSeqStore.size > 0 && current >= currLdmEnd + startBlockIdx) {
-                    getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore,
-                               ms->ldmSeqStore.capacity, (U32)(ip - istart));
-                }
-
+                
+                /* Fetch next LDM if necessary */
+                
+                getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore, ms->ldmSeqStore.capacity, (U32)(ip-istart), current, startBlockIdx);
                 U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
-
-                if (ms->ldmSeqStore.size > 0 && current >= currLdmStart + startBlockIdx && current < currLdmEnd + startBlockIdx) {
-                    maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
-                }
+                maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
 
                 U32 matchNb;
                 if (!nbMatches) {

From 4222fb34a653f36f555cb9e210394ef1be74ffa7 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Tue, 22 Sep 2020 10:52:49 -0400
Subject: [PATCH 14/25] Generic progress

---
 lib/compress/zstd_ldm.c |  2 +-
 lib/compress/zstd_opt.c | 59 ++++++++++++++++++++++++-----------------
 2 files changed, 36 insertions(+), 25 deletions(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index f95a69bd32f..da7493a1824 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -599,7 +599,7 @@ static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore, size_t srcSize)
         rawSeqStore->seq[i].litLength = matchEnd;
         printf("(%u, %u)\n", matchStart, matchEnd);
     }
-    if (rawSeqStore->seq[rawSeqStore->size - 1].litLength >= srcSize) {
+    if (rawSeqStore->seq[rawSeqStore->size - 1].litLength > srcSize + 1) {
         printf("SETTING RANGEFLAG TO 2\n");
         rawSeqStore->rangeFlag = 2; /* Signifies that this is a seqstore that spans
                                            multiple blocks. */
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 6d3442fd290..4ec046844b1 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -769,37 +769,47 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
 
 /* TODO: Increment by SBI */
 static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
-                       const U32 ldmSeqStoreStartPos, U32 currPosInBlock, U32 current, U32 startBlockIdx) {
+                       const U32 ldmSeqStoreStartPos, U32 currPosInBlock, U32 current, U32 startBlockIdx, U32 endBlockIdx) {
     if (ldmSeqStore->pos >= ldmSeqStore->size ||
         ldmSeqStore->size == 0 || ldmSeqStore->rangeFlag == 0) 
         return;
 
     if (ldmSeqStore->rangeFlag == 2) {
-        if (current >= *ldmEnd) {
-            printf("Getting next ldm: current: %u and currLdmEnd + sbi - 1: %u\n", current, *ldmEnd + startBlockIdx - 1);
+        if (current > *ldmEnd) {
+            /* If our current pos is greater than current match end, we need to fetch a new match */
+            printf("Getting next ldm: current: %u and currLdmEnd: %u\n", current, *ldmEnd);
+            printf("Range before update: (%u, %u)\n", *ldmStart, *ldmEnd);
+            ldmSeqStore->pos++;
             *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
             *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
-            printf("Final range after adjusting for absolute ldm seq store: (%u, %u)\n", *ldmStart, *ldmEnd);
-        } else {
-            return;
+            printf("Newly fetched ldm: (%u, %u)\n", *ldmStart, *ldmEnd);
+        }
+        /* Handle match splitting, which only applies for multi-threaded cases
+        * If an LDM starts before the block ends, and ends after the block ends, we split the match into two.
+        * Don't increment pos so we stay on this match until it ends.
+        */
+       if (*ldmStart < endBlockIdx  && *ldmEnd > endBlockIdx) {
+            printf("Splitting match @ end: ldmEnd: %u, endBlockIdx: %u\n", *ldmEnd, endBlockIdx);
+            *ldmEnd = endBlockIdx;
+            printf("Range after split: (%u, %u)\n", *ldmStart, *ldmEnd);
+        }
+
+        if (*ldmStart < startBlockIdx && *ldmEnd > startBlockIdx) {
+            printf("Splitting match @ start: ldmStart: %u, startBlockIdx: %u\n", *ldmStart, startBlockIdx);
+            *ldmStart = startBlockIdx;
+            printf("Range after split: (%u, %u)\n", *ldmStart, *ldmEnd);
         }
+
     } else {
         if (current >= *ldmEnd + startBlockIdx - 1) {
+            printf("Range before update: (%u, %u)\n", *ldmStart, *ldmEnd);
+            ldmSeqStore->pos++;
             printf("Getting next ldm: current: %u and currLdmEnd + sbi - 1: %u\n", current, *ldmEnd + startBlockIdx - 1);
             *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
             *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
             printf("Final range: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd, ldmSeqStore->pos);
-        } else {
-            return;
         }
     }
-
-   
-    printf("getNextLdm(): start of seqStore: %u, currPosInBlock: %u ", ldmSeqStoreStartPos, currPosInBlock);
-    printf("-- range: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd, ldmSeqStore->pos);
-    ldmSeqStore->pos++;
-
-    /* Handle block splitting */
 }
 
 static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* matches,
@@ -813,7 +823,6 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
         if (!(current >= ldmStart) || !(current < ldmEnd))
             return;
     }
-    printf("About to maybe add an ldm...\n");
 
     if (ldmSeqStore->rangeFlag == 1) {
         ldmStart += startBlockIdx - 1;
@@ -832,15 +841,15 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
         return;
     }
 
-    printf("Considering LDM @ current = %u, posDiff = %u - ", current, posDifference);
-    U32 candidateOffset = ldmSeqStore->seq[ldmSeqStore->pos - 1].offset + posDifference;
+    printf("Considering LDM range (%u, %u) @ current = %u, posDiff = %u - ", ldmStart, ldmEnd, current, posDifference);
+    U32 candidateOffset = ldmSeqStore->seq[ldmSeqStore->pos].offset + posDifference;
     U32 candidateMatchLength = originalMatchLength - posDifference;
     if (candidateMatchLength < ZSTD_LDM_MINMATCH_MIN) {
         printf("too small\n");
         return;
     }
     printf("adjusted to (of: %u, ml %u)\n", candidateOffset, candidateMatchLength);
-    if (candidateMatchLength >= matches[*nbMatches-1].len) {
+    if (candidateMatchLength >= matches[*nbMatches-1].len /*&& candidateOffset + ZSTD_REP_MOVE <= matches[*nbMatches].off*/) {
         printf("large enough, adding\n");
         matches[*nbMatches].len = candidateMatchLength;
         matches[*nbMatches].off = candidateOffset + ZSTD_REP_MOVE;
@@ -915,13 +924,15 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     int ldmAdjusted = 0;
 
     /* Set current LDM candidate to whatever might have been considered in prev block */
-    if (ms->ldmSeqStore.pos != 0) {
-        currLdmStart = ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].matchLength;
-        currLdmEnd = ms->ldmSeqStore.seq[ms->ldmSeqStore.pos].litLength;
+    if (ms->ldmSeqStore.size != 0) {
+        size_t readIdx = ms->ldmSeqStore.pos == 0 ? 0 : ms->ldmSeqStore.pos - 1;
+        currLdmStart = ms->ldmSeqStore.seq[readIdx].matchLength;
+        currLdmEnd = ms->ldmSeqStore.seq[readIdx].litLength;
     } else {
         currLdmStart = currLdmEnd = 0;
     }
 
+
     /* Match Loop */
     while (ip < ilimit) {
         U32 cur, last_pos = 0;
@@ -931,7 +942,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
             U32 const ll0 = !litlen;
             U32 const current = (U32)(ip - base);
 
-            getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore, ms->ldmSeqStore.capacity, (U32)(ip-istart), current, startBlockIdx);
+            getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore, ms->ldmSeqStore.capacity, (U32)(ip-istart), current, startBlockIdx, startBlockIdx + srcSize);
             U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, ip, iend, dictMode, rep, ll0, minMatch);
             maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
 
@@ -1051,7 +1062,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
                 
                 /* Fetch next LDM if necessary */
                 
-                getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore, ms->ldmSeqStore.capacity, (U32)(ip-istart), current, startBlockIdx);
+                getNextLdm(&currLdmStart, &currLdmEnd, &ms->ldmSeqStore, ms->ldmSeqStore.capacity, (U32)(inr-istart), current, startBlockIdx, startBlockIdx + srcSize);
                 U32 nbMatches = ZSTD_BtGetAllMatches(matches, ms, &nextToUpdate3, inr, iend, dictMode, opt[cur].rep, ll0, minMatch);
                 maybeAddLdm(&ms->ldmSeqStore, matches, &nbMatches, currLdmStart, currLdmEnd, current, startBlockIdx);
 

From 6f9a94b8aa616a156cdb9a24390668fcb145fd15 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Tue, 22 Sep 2020 11:44:37 -0400
Subject: [PATCH 15/25] Working single-threaded

---
 lib/compress/zstd_compress.c |  4 ++--
 lib/compress/zstd_ldm.c      |  1 +
 lib/compress/zstd_opt.c      | 27 ++++++++++++++-------------
 3 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/lib/compress/zstd_compress.c b/lib/compress/zstd_compress.c
index 31bd2405c81..b4e455f5a19 100644
--- a/lib/compress/zstd_compress.c
+++ b/lib/compress/zstd_compress.c
@@ -2328,7 +2328,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
             for (i = 0; i < ZSTD_REP_NUM; ++i)
                 zc->blockState.nextCBlock->rep[i] = zc->blockState.prevCBlock->rep[i];
         }
-        printf("Deciding....\n");
+        printf("-----NEW BLOCK-----\n");
         if (zc->externSeqStore.pos < zc->externSeqStore.size) {
             assert(!zc->appliedParams.ldmParams.enableLdm);
             /* Updates ldmSeqStore.pos */
@@ -2361,7 +2361,7 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize)
         {   const BYTE* const lastLiterals = (const BYTE*)src + srcSize - lastLLSize;
             ZSTD_storeLastLiterals(&zc->seqStore, lastLiterals, lastLLSize);
     }   }
-    printf("Done build ldm seq store\n");
+    printf("Finished BuildSeqStore()\n");
     return ZSTDbss_compress;
 }
 
diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index da7493a1824..ac20eaf0b1d 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -599,6 +599,7 @@ static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore, size_t srcSize)
         rawSeqStore->seq[i].litLength = matchEnd;
         printf("(%u, %u)\n", matchStart, matchEnd);
     }
+    printf("size:%u\n", rawSeqStore->size);
     if (rawSeqStore->seq[rawSeqStore->size - 1].litLength > srcSize + 1) {
         printf("SETTING RANGEFLAG TO 2\n");
         rawSeqStore->rangeFlag = 2; /* Signifies that this is a seqstore that spans
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 4ec046844b1..bc5c2c4adbb 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -770,7 +770,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_BtGetAllMatches (
 /* TODO: Increment by SBI */
 static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
                        const U32 ldmSeqStoreStartPos, U32 currPosInBlock, U32 current, U32 startBlockIdx, U32 endBlockIdx) {
-    if (ldmSeqStore->pos >= ldmSeqStore->size ||
+    if (ldmSeqStore->pos >= ldmSeqStore->size - 1 || /* pos == size-1 means currLdm is the last one, should never fetch another */
         ldmSeqStore->size == 0 || ldmSeqStore->rangeFlag == 0) 
         return;
 
@@ -802,12 +802,12 @@ static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
 
     } else {
         if (current >= *ldmEnd + startBlockIdx - 1) {
-            printf("Range before update: (%u, %u)\n", *ldmStart, *ldmEnd);
+            printf("Getting next raw ldm range at: current: %u with seqStore.pos: %u .size: %u\n", current, ldmSeqStore->pos, ldmSeqStore->size);
+            printf("Current raw ldm range: (%u, %u) -> abs: (%u, %u)\n", *ldmStart, *ldmEnd, *ldmStart + startBlockIdx - 1, *ldmEnd + startBlockIdx - 1);
             ldmSeqStore->pos++;
-            printf("Getting next ldm: current: %u and currLdmEnd + sbi - 1: %u\n", current, *ldmEnd + startBlockIdx - 1);
             *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
             *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
-            printf("Final range: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd, ldmSeqStore->pos);
+            printf("New raw ldm range: (%u, %u) -> abs: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd,  *ldmStart + startBlockIdx - 1, *ldmEnd + startBlockIdx - 1, ldmSeqStore->pos);
         }
     }
 }
@@ -824,35 +824,36 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
             return;
     }
 
+    printf("Considering LDM range (%u, %u) -> abs: (%u, %u) @ current = %u", ldmStart, ldmEnd, ldmStart + startBlockIdx - 1, ldmEnd + startBlockIdx - 1, current);
+
     if (ldmSeqStore->rangeFlag == 1) {
         ldmStart += startBlockIdx - 1;
         ldmEnd += startBlockIdx - 1;
     }
-
     U32 posDifference = current - ldmStart;
+    printf(" posDiff = %u\n", posDifference);
     U32 originalMatchLength = ldmEnd - ldmStart;
     if (posDifference >= originalMatchLength) {
         printf("posdiff greater than matchlen!\n");
         return;
     }
-
-    assert(ldmSeqStore->pos > 0);
+    //assert(ldmSeqStore->pos > 0);
     if (posDifference > 0) {
         return;
     }
 
-    printf("Considering LDM range (%u, %u) @ current = %u, posDiff = %u - ", ldmStart, ldmEnd, current, posDifference);
-    U32 candidateOffset = ldmSeqStore->seq[ldmSeqStore->pos].offset + posDifference;
+    U32 candidateOffCode = ldmSeqStore->seq[ldmSeqStore->pos].offset + posDifference + ZSTD_REP_MOVE;
     U32 candidateMatchLength = originalMatchLength - posDifference;
     if (candidateMatchLength < ZSTD_LDM_MINMATCH_MIN) {
         printf("too small\n");
         return;
     }
-    printf("adjusted to (of: %u, ml %u)\n", candidateOffset, candidateMatchLength);
-    if (candidateMatchLength >= matches[*nbMatches-1].len /*&& candidateOffset + ZSTD_REP_MOVE <= matches[*nbMatches].off*/) {
+    printf("adjusted to (of(code): %u, ml %u)\n", candidateOffCode, candidateMatchLength);
+    if (candidateMatchLength >= matches[*nbMatches-1].len) {
         printf("large enough, adding\n");
+        /* Add sifting */
         matches[*nbMatches].len = candidateMatchLength;
-        matches[*nbMatches].off = candidateOffset + ZSTD_REP_MOVE;
+        matches[*nbMatches].off = candidateOffCode;
         (*nbMatches)++;
     }
 }
@@ -928,11 +929,11 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
         size_t readIdx = ms->ldmSeqStore.pos == 0 ? 0 : ms->ldmSeqStore.pos - 1;
         currLdmStart = ms->ldmSeqStore.seq[readIdx].matchLength;
         currLdmEnd = ms->ldmSeqStore.seq[readIdx].litLength;
+        printf("Starting opt with ldm : (%u, %u)\n", currLdmStart, currLdmEnd);
     } else {
         currLdmStart = currLdmEnd = 0;
     }
 
-
     /* Match Loop */
     while (ip < ilimit) {
         U32 cur, last_pos = 0;

From b24ddf5df3cfb90d0173a67d582232bd6aadbe66 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Tue, 22 Sep 2020 12:56:28 -0400
Subject: [PATCH 16/25] EnblockIdx

---
 lib/compress/zstd_opt.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index bc5c2c4adbb..f33e5848954 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -907,6 +907,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     U32 const minMatch = (cParams->minMatch == 3) ? 3 : 4;
     U32 nextToUpdate3 = ms->nextToUpdate;
     U32 const startBlockIdx = (U32)(istart - base);
+    U32 const endBlockIdx = startBlockIdx + srcSize;
     
     U32 currLdmStart;
     U32 currLdmEnd;
@@ -918,7 +919,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     /* init */
     DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
                 (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
-    printf("ZSTD_compressBlock_opt_generic: current=%u, sbi=%u\n", (U32)(ip - base), startBlockIdx);
+    printf("ZSTD_compressBlock_opt_generic: current=%u, sbi=%u, ebi=%u\n", (U32)(ip - base), startBlockIdx, endBlockIdx);
     assert(optLevel <= 2);
     ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
     ip += (ip==prefixStart);

From 7cd4e6efaf40a33a862b9b7a5a33c0bf20c3065e Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Tue, 22 Sep 2020 19:56:02 -0400
Subject: [PATCH 17/25] Various changes sept 22

---
 lib/compress/zstd_opt.c | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index f33e5848954..97d6e612793 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -824,23 +824,27 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
             return;
     }
 
-    printf("Considering LDM range (%u, %u) -> abs: (%u, %u) @ current = %u", ldmStart, ldmEnd, ldmStart + startBlockIdx - 1, ldmEnd + startBlockIdx - 1, current);
-
     if (ldmSeqStore->rangeFlag == 1) {
         ldmStart += startBlockIdx - 1;
         ldmEnd += startBlockIdx - 1;
     }
-    U32 posDifference = current - ldmStart;
-    printf(" posDiff = %u\n", posDifference);
+    
     U32 originalMatchLength = ldmEnd - ldmStart;
-    if (posDifference >= originalMatchLength) {
-        printf("posdiff greater than matchlen!\n");
-        return;
-    }
+    
+
+    U32 posDifference = current - ldmStart;
+    if (posDifference == 0)
+        printf("Considering LDM range (%u, %u) -> abs: (%u, %u) @ current = %u", ldmStart, ldmEnd, ldmStart + startBlockIdx - 1, ldmEnd + startBlockIdx - 1, current);
+
     //assert(ldmSeqStore->pos > 0);
     if (posDifference > 0) {
         return;
     }
+    
+    if (posDifference >= originalMatchLength) {
+        printf("posdiff greater than matchlen!\n");
+        return;
+    }
 
     U32 candidateOffCode = ldmSeqStore->seq[ldmSeqStore->pos].offset + posDifference + ZSTD_REP_MOVE;
     U32 candidateMatchLength = originalMatchLength - posDifference;
@@ -849,12 +853,13 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
         return;
     }
     printf("adjusted to (of(code): %u, ml %u)\n", candidateOffCode, candidateMatchLength);
-    if (candidateMatchLength >= matches[*nbMatches-1].len) {
+    if (candidateMatchLength > matches[*nbMatches-1].len) {
         printf("large enough, adding\n");
         /* Add sifting */
         matches[*nbMatches].len = candidateMatchLength;
         matches[*nbMatches].off = candidateOffCode;
         (*nbMatches)++;
+        
     }
 }
 

From 748c7d4926cb7e75a431c65d62e715f98e8ca77f Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Sep 2020 11:27:37 -0400
Subject: [PATCH 18/25] Working single-threaded nci

---
 lib/compress/zstd_ldm.c |  4 ++--
 lib/compress/zstd_opt.c | 48 +++++++++++++++++++----------------------
 2 files changed, 24 insertions(+), 28 deletions(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index ac20eaf0b1d..3b7df71bba5 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -585,7 +585,7 @@ static void printSeqStore(rawSeqStore_t* rawSeqStore) {
 }
 static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore, size_t srcSize) {
     size_t i;
-    size_t currPos = 1;
+    size_t currPos = 0;
     printf("Conversion...\n");
     rawSeqStore->rangeFlag = 1;
     for(i = 0 ; i < rawSeqStore->size; ++i) {
@@ -600,7 +600,7 @@ static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore, size_t srcSize)
         printf("(%u, %u)\n", matchStart, matchEnd);
     }
     printf("size:%u\n", rawSeqStore->size);
-    if (rawSeqStore->seq[rawSeqStore->size - 1].litLength > srcSize + 1) {
+    if (rawSeqStore->seq[rawSeqStore->size - 1].litLength > srcSize) {
         printf("SETTING RANGEFLAG TO 2\n");
         rawSeqStore->rangeFlag = 2; /* Signifies that this is a seqstore that spans
                                            multiple blocks. */
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 97d6e612793..f9e629b8099 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -801,13 +801,16 @@ static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
         }
 
     } else {
-        if (current >= *ldmEnd + startBlockIdx - 1) {
+        /* In this case, all of the LDMs are within this one block */
+        U32 ldmStartAdjusted = *ldmStart + startBlockIdx;
+        U32 ldmEndAdjusted = *ldmEnd + startBlockIdx;
+        if (current >= ldmEndAdjusted) {
             printf("Getting next raw ldm range at: current: %u with seqStore.pos: %u .size: %u\n", current, ldmSeqStore->pos, ldmSeqStore->size);
-            printf("Current raw ldm range: (%u, %u) -> abs: (%u, %u)\n", *ldmStart, *ldmEnd, *ldmStart + startBlockIdx - 1, *ldmEnd + startBlockIdx - 1);
+            printf("Current raw ldm range: (%u, %u) -> abs: (%u, %u)\n", *ldmStart, *ldmEnd, ldmStartAdjusted, ldmEndAdjusted);
             ldmSeqStore->pos++;
             *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
             *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
-            printf("New raw ldm range: (%u, %u) -> abs: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd,  *ldmStart + startBlockIdx - 1, *ldmEnd + startBlockIdx - 1, ldmSeqStore->pos);
+            printf("New raw ldm range: (%u, %u) -> abs: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd,  *ldmStart + startBlockIdx, *ldmEnd + startBlockIdx, ldmSeqStore->pos);
         }
     }
 }
@@ -816,33 +819,26 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
                         U32* nbMatches, U32 ldmStart, U32 ldmEnd, U32 current, U32 startBlockIdx) {
     if (ldmSeqStore->size == 0)
         return;
+    assert(ldmSeqStore->rangeFlag != 0);
+    /* Adjusted ldms for when the ldm seq store was calculated for this block only */
+    U32 ldmStartAdjusted = ldmSeqStore->rangeFlag == 1 ? ldmStart + startBlockIdx : ldmStart;
+    U32 ldmEndAdjusted = ldmSeqStore->rangeFlag == 1 ? ldmEnd + startBlockIdx : ldmEnd;
+
+    /* Current must be within the adjusted ldm */
     if (ldmSeqStore->rangeFlag == 1) {
-        if (!(current >= ldmStart + startBlockIdx - 1) || !(current < ldmEnd + startBlockIdx - 1))
+        if (current < ldmStartAdjusted || current >= ldmEndAdjusted)
             return;
     } else if (ldmSeqStore->rangeFlag == 2) {
-        if (!(current >= ldmStart) || !(current < ldmEnd))
+        if (!(current >= ldmStartAdjusted) || !(current < ldmEndAdjusted))
             return;
     }
-
-    if (ldmSeqStore->rangeFlag == 1) {
-        ldmStart += startBlockIdx - 1;
-        ldmEnd += startBlockIdx - 1;
-    }
-    
-    U32 originalMatchLength = ldmEnd - ldmStart;
     
-
-    U32 posDifference = current - ldmStart;
+    U32 originalMatchLength = ldmEndAdjusted - ldmStartAdjusted;
+    U32 posDifference = current - ldmStartAdjusted;
     if (posDifference == 0)
-        printf("Considering LDM range (%u, %u) -> abs: (%u, %u) @ current = %u", ldmStart, ldmEnd, ldmStart + startBlockIdx - 1, ldmEnd + startBlockIdx - 1, current);
+        printf("Considering LDM range (%u, %u) -> abs: (%u, %u) @ current = %u\n", ldmStart, ldmEnd, ldmStartAdjusted, ldmEndAdjusted, current);
 
-    //assert(ldmSeqStore->pos > 0);
-    if (posDifference > 0) {
-        return;
-    }
-    
-    if (posDifference >= originalMatchLength) {
-        printf("posdiff greater than matchlen!\n");
+    if (posDifference > 0 || posDifference >= originalMatchLength) {
         return;
     }
 
@@ -853,12 +849,12 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
         return;
     }
     printf("adjusted to (of(code): %u, ml %u)\n", candidateOffCode, candidateMatchLength);
-    if (candidateMatchLength > matches[*nbMatches-1].len) {
+    if ((*nbMatches == 0 || candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) {
         printf("large enough, adding\n");
         /* Add sifting */
-        matches[*nbMatches].len = candidateMatchLength;
-        matches[*nbMatches].off = candidateOffCode;
-        (*nbMatches)++;
+            matches[*nbMatches].len = candidateMatchLength;
+            matches[*nbMatches].off = candidateOffCode;
+            (*nbMatches)++;
         
     }
 }

From e8f381c403cba72a1b529fc87e6de90c679a478f Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Sep 2020 11:28:02 -0400
Subject: [PATCH 19/25] Sifting is not segfaulting?

---
 lib/compress/zstd_opt.c | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index f9e629b8099..33227e5fd33 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -852,10 +852,27 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
     if ((*nbMatches == 0 || candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) {
         printf("large enough, adding\n");
         /* Add sifting */
+        if (*nbMatches == 0) {
             matches[*nbMatches].len = candidateMatchLength;
             matches[*nbMatches].off = candidateOffCode;
             (*nbMatches)++;
-        
+        } else {
+            printf("Sifting...\n");
+            if (candidateMatchLength == matches[*nbMatches-1].len) {
+                U32 candidateMatchIdx = *nbMatches;
+                matches[*nbMatches].len = candidateMatchLength;
+                matches[*nbMatches].off = candidateOffCode;
+                while (candidateMatchIdx > 0 &&
+                       candidateOffCode > matches[candidateMatchIdx - 1].off &&
+                       candidateMatchLength == matches[candidateMatchIdx - 1].len) {
+                    ZSTD_match_t tmp = matches[candidateMatchIdx - 1];
+                    matches[candidateMatchIdx - 1].off = candidateOffCode;
+                    matches[candidateMatchIdx - 1].len = candidateMatchLength;
+                    matches[candidateMatchIdx] = tmp;
+                }
+                (*nbMatches)++;
+            }
+        }
     }
 }
 

From 2d7539e4a326bed01daff9fb1db5c1043e617d38 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Sep 2020 11:28:32 -0400
Subject: [PATCH 20/25] Sifting commented out

---
 lib/compress/zstd_opt.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 33227e5fd33..32ece6410c5 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -851,7 +851,11 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
     printf("adjusted to (of(code): %u, ml %u)\n", candidateOffCode, candidateMatchLength);
     if ((*nbMatches == 0 || candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) {
         printf("large enough, adding\n");
+        matches[*nbMatches].len = candidateMatchLength;
+        matches[*nbMatches].off = candidateOffCode;
+        (*nbMatches)++;
         /* Add sifting */
+        /*
         if (*nbMatches == 0) {
             matches[*nbMatches].len = candidateMatchLength;
             matches[*nbMatches].off = candidateOffCode;
@@ -872,7 +876,7 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
                 }
                 (*nbMatches)++;
             }
-        }
+        }*/
     }
 }
 

From 71097a18368d830d762235110b3131e5b66b4e9a Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Sep 2020 12:35:02 -0400
Subject: [PATCH 21/25] Seems to roundtrip nci

---
 lib/compress/zstd_opt.c | 64 +++++++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 28 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 32ece6410c5..3d1f4a468ce 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -777,27 +777,27 @@ static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
     if (ldmSeqStore->rangeFlag == 2) {
         if (current > *ldmEnd) {
             /* If our current pos is greater than current match end, we need to fetch a new match */
-            printf("Getting next ldm: current: %u and currLdmEnd: %u\n", current, *ldmEnd);
-            printf("Range before update: (%u, %u)\n", *ldmStart, *ldmEnd);
+            //printf("Getting next ldm: current: %u and currLdmEnd: %u\n", current, *ldmEnd);
+            //printf("Range before update: (%u, %u)\n", *ldmStart, *ldmEnd);
             ldmSeqStore->pos++;
             *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
             *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
-            printf("Newly fetched ldm: (%u, %u)\n", *ldmStart, *ldmEnd);
+            //printf("Newly fetched ldm: (%u, %u)\n", *ldmStart, *ldmEnd);
         }
         /* Handle match splitting, which only applies for multi-threaded cases
         * If an LDM starts before the block ends, and ends after the block ends, we split the match into two.
         * Don't increment pos so we stay on this match until it ends.
         */
        if (*ldmStart < endBlockIdx  && *ldmEnd > endBlockIdx) {
-            printf("Splitting match @ end: ldmEnd: %u, endBlockIdx: %u\n", *ldmEnd, endBlockIdx);
+            //printf("Splitting match @ end: ldmEnd: %u, endBlockIdx: %u\n", *ldmEnd, endBlockIdx);
             *ldmEnd = endBlockIdx;
-            printf("Range after split: (%u, %u)\n", *ldmStart, *ldmEnd);
+            //printf("Range after split: (%u, %u)\n", *ldmStart, *ldmEnd);
         }
 
         if (*ldmStart < startBlockIdx && *ldmEnd > startBlockIdx) {
-            printf("Splitting match @ start: ldmStart: %u, startBlockIdx: %u\n", *ldmStart, startBlockIdx);
+            //printf("Splitting match @ start: ldmStart: %u, startBlockIdx: %u\n", *ldmStart, startBlockIdx);
             *ldmStart = startBlockIdx;
-            printf("Range after split: (%u, %u)\n", *ldmStart, *ldmEnd);
+            //printf("Range after split: (%u, %u)\n", *ldmStart, *ldmEnd);
         }
 
     } else {
@@ -805,12 +805,12 @@ static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
         U32 ldmStartAdjusted = *ldmStart + startBlockIdx;
         U32 ldmEndAdjusted = *ldmEnd + startBlockIdx;
         if (current >= ldmEndAdjusted) {
-            printf("Getting next raw ldm range at: current: %u with seqStore.pos: %u .size: %u\n", current, ldmSeqStore->pos, ldmSeqStore->size);
-            printf("Current raw ldm range: (%u, %u) -> abs: (%u, %u)\n", *ldmStart, *ldmEnd, ldmStartAdjusted, ldmEndAdjusted);
+            //printf("Getting next raw ldm range at: current: %u with seqStore.pos: %u .size: %u\n", current, ldmSeqStore->pos, ldmSeqStore->size);
+            //printf("Current raw ldm range: (%u, %u) -> abs: (%u, %u)\n", *ldmStart, *ldmEnd, ldmStartAdjusted, ldmEndAdjusted);
             ldmSeqStore->pos++;
             *ldmStart = ldmSeqStore->seq[ldmSeqStore->pos].matchLength;
             *ldmEnd = ldmSeqStore->seq[ldmSeqStore->pos].litLength;
-            printf("New raw ldm range: (%u, %u) -> abs: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd,  *ldmStart + startBlockIdx, *ldmEnd + startBlockIdx, ldmSeqStore->pos);
+            //printf("New raw ldm range: (%u, %u) -> abs: (%u, %u) at pos: %u\n", *ldmStart, *ldmEnd,  *ldmStart + startBlockIdx, *ldmEnd + startBlockIdx, ldmSeqStore->pos);
         }
     }
 }
@@ -834,49 +834,57 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
     }
     
     U32 originalMatchLength = ldmEndAdjusted - ldmStartAdjusted;
+    printf("Original matchlen: %u with ldmStartAdjusted: %u and ldmEndAdjusted: %u\n", originalMatchLength, ldmStartAdjusted, ldmEndAdjusted);
     U32 posDifference = current - ldmStartAdjusted;
-    if (posDifference == 0)
-        printf("Considering LDM range (%u, %u) -> abs: (%u, %u) @ current = %u\n", ldmStart, ldmEnd, ldmStartAdjusted, ldmEndAdjusted, current);
-
     if (posDifference > 0 || posDifference >= originalMatchLength) {
         return;
     }
+    
+    printf("Considering LDM range (%u, %u) -> abs: (%u, %u) @ current = %u\n", ldmStart, ldmEnd, ldmStartAdjusted, ldmEndAdjusted, current);
+
 
     U32 candidateOffCode = ldmSeqStore->seq[ldmSeqStore->pos].offset + posDifference + ZSTD_REP_MOVE;
     U32 candidateMatchLength = originalMatchLength - posDifference;
     if (candidateMatchLength < ZSTD_LDM_MINMATCH_MIN) {
-        printf("too small\n");
+        //printf("too small\n");
         return;
     }
-    printf("adjusted to (of(code): %u, ml %u)\n", candidateOffCode, candidateMatchLength);
+    //printf("adjusted to (of(code): %u, ml %u)\n", candidateOffCode, candidateMatchLength);
     if ((*nbMatches == 0 || candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) {
-        printf("large enough, adding\n");
-        matches[*nbMatches].len = candidateMatchLength;
+        //printf("large enough, adding\n");
+        /*matches[*nbMatches].len = candidateMatchLength;
         matches[*nbMatches].off = candidateOffCode;
-        (*nbMatches)++;
+        (*nbMatches)++;*/
         /* Add sifting */
-        /*
+        
         if (*nbMatches == 0) {
             matches[*nbMatches].len = candidateMatchLength;
             matches[*nbMatches].off = candidateOffCode;
             (*nbMatches)++;
         } else {
-            printf("Sifting...\n");
             if (candidateMatchLength == matches[*nbMatches-1].len) {
                 U32 candidateMatchIdx = *nbMatches;
                 matches[*nbMatches].len = candidateMatchLength;
                 matches[*nbMatches].off = candidateOffCode;
+                //printf("Sifting...: idx: %u, len: %u, off: %u\n", candidateMatchIdx, candidateMatchLength, candidateOffCode);
+                //printf("Current best is...: idx: %u, len: %u, off: %u\n", *nbMatches-1,  matches[*nbMatches-1].len, matches[*nbMatches-1].off);
+                if (candidateOffCode != matches[*nbMatches].off)
+                    printf("DIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off);
+                //printf("Current best is...: idx: %u, len: %u, off: %u\n", *nbMatches-1,  matches[*nbMatches-1].len, matches[*nbMatches-1].off);
                 while (candidateMatchIdx > 0 &&
-                       candidateOffCode > matches[candidateMatchIdx - 1].off &&
-                       candidateMatchLength == matches[candidateMatchIdx - 1].len) {
+                       matches[candidateMatchIdx].off > matches[candidateMatchIdx - 1].off &&
+                       matches[candidateMatchIdx].len == matches[candidateMatchIdx - 1].len) {
+                    //printf("Compared to: idx: %u, len: %u, off: %u", candidateMatchIdx - 1, matches[candidateMatchIdx - 1].len, matches[candidateMatchIdx - 1].off);
                     ZSTD_match_t tmp = matches[candidateMatchIdx - 1];
-                    matches[candidateMatchIdx - 1].off = candidateOffCode;
-                    matches[candidateMatchIdx - 1].len = candidateMatchLength;
+                    matches[candidateMatchIdx - 1] = matches[candidateMatchIdx];
                     matches[candidateMatchIdx] = tmp;
+                    --candidateMatchIdx;
                 }
                 (*nbMatches)++;
+            } else {
+                printf("MATCHDIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off);
             }
-        }*/
+        }
     }
 }
 
@@ -941,7 +949,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
     /* init */
     DEBUGLOG(5, "ZSTD_compressBlock_opt_generic: current=%u, prefix=%u, nextToUpdate=%u",
                 (U32)(ip - base), ms->window.dictLimit, ms->nextToUpdate);
-    printf("ZSTD_compressBlock_opt_generic: current=%u, sbi=%u, ebi=%u\n", (U32)(ip - base), startBlockIdx, endBlockIdx);
+    //printf("ZSTD_compressBlock_opt_generic: current=%u, sbi=%u, ebi=%u\n", (U32)(ip - base), startBlockIdx, endBlockIdx);
     assert(optLevel <= 2);
     ZSTD_rescaleFreqs(optStatePtr, (const BYTE*)src, srcSize, optLevel);
     ip += (ip==prefixStart);
@@ -952,7 +960,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
         size_t readIdx = ms->ldmSeqStore.pos == 0 ? 0 : ms->ldmSeqStore.pos - 1;
         currLdmStart = ms->ldmSeqStore.seq[readIdx].matchLength;
         currLdmEnd = ms->ldmSeqStore.seq[readIdx].litLength;
-        printf("Starting opt with ldm : (%u, %u)\n", currLdmStart, currLdmEnd);
+        //printf("Starting opt with ldm : (%u, %u)\n", currLdmStart, currLdmEnd);
     } else {
         currLdmStart = currLdmEnd = 0;
     }
@@ -1205,7 +1213,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms,
         }
     }   /* while (ip < ilimit) */
 
-    printf("Finished opt\n");
+    //printf("Finished opt\n");
     /* Return the last literals size */
     return (size_t)(iend - anchor);
 }

From 7924834391bc4edc27c8822b0be4c2f748326da8 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Sep 2020 12:38:15 -0400
Subject: [PATCH 22/25] Can now roundtrip silesia.tar

---
 lib/compress/zstd_opt.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 3d1f4a468ce..c68ff9eb618 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -834,12 +834,11 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
     }
     
     U32 originalMatchLength = ldmEndAdjusted - ldmStartAdjusted;
-    printf("Original matchlen: %u with ldmStartAdjusted: %u and ldmEndAdjusted: %u\n", originalMatchLength, ldmStartAdjusted, ldmEndAdjusted);
     U32 posDifference = current - ldmStartAdjusted;
     if (posDifference > 0 || posDifference >= originalMatchLength) {
         return;
     }
-    
+    printf("Original matchlen: %u with ldmStartAdjusted: %u and ldmEndAdjusted: %u\n", originalMatchLength, ldmStartAdjusted, ldmEndAdjusted);
     printf("Considering LDM range (%u, %u) -> abs: (%u, %u) @ current = %u\n", ldmStart, ldmEnd, ldmStartAdjusted, ldmEndAdjusted, current);
 
 

From 74082623d00b05ab13ba95c7db62d5fd13376f9f Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Sep 2020 14:22:40 -0400
Subject: [PATCH 23/25] Don't convert 0 size seqstores

---
 lib/compress/zstd_ldm.c | 2 ++
 lib/compress/zstd_opt.c | 5 ++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 3b7df71bba5..48293a9ac8a 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -584,6 +584,8 @@ static void printSeqStore(rawSeqStore_t* rawSeqStore) {
     }
 }
 static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore, size_t srcSize) {
+    if (rawSeqStore->size == 0)
+        return;
     size_t i;
     size_t currPos = 0;
     printf("Conversion...\n");
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index c68ff9eb618..234e14d31f2 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -838,7 +838,7 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
     if (posDifference > 0 || posDifference >= originalMatchLength) {
         return;
     }
-    printf("Original matchlen: %u with ldmStartAdjusted: %u and ldmEndAdjusted: %u\n", originalMatchLength, ldmStartAdjusted, ldmEndAdjusted);
+    printf("Original matchlen: %u - ", originalMatchLength);
     printf("Considering LDM range (%u, %u) -> abs: (%u, %u) @ current = %u\n", ldmStart, ldmEnd, ldmStartAdjusted, ldmEndAdjusted, current);
 
 
@@ -882,6 +882,9 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
                 (*nbMatches)++;
             } else {
                 printf("MATCHDIFF: ldm: (len: %u, off: %u), best: (len: %u, off: %u)\n", candidateMatchLength, candidateOffCode, matches[*nbMatches-1].len, matches[*nbMatches-1].off);
+                matches[*nbMatches].len = candidateMatchLength;
+                matches[*nbMatches].off = candidateOffCode;
+                (*nbMatches)++;
             }
         }
     }

From 5d0bd45606674836ce5ff6cce7c821206dbce421 Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Sep 2020 15:14:19 -0400
Subject: [PATCH 24/25] Working multithreading-ish?

---
 lib/compress/zstd_ldm.c | 23 +++++++++++++----------
 lib/compress/zstd_opt.c |  1 +
 2 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/lib/compress/zstd_ldm.c b/lib/compress/zstd_ldm.c
index 48293a9ac8a..f242529a245 100644
--- a/lib/compress/zstd_ldm.c
+++ b/lib/compress/zstd_ldm.c
@@ -583,7 +583,16 @@ static void printSeqStore(rawSeqStore_t* rawSeqStore) {
         printf("(of:%u ml:%u ll: %u)\n", rawSeqStore->seq[i].offset, rawSeqStore->seq[i].matchLength, rawSeqStore->seq[i].litLength);
     }
 }
-static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore, size_t srcSize) {
+
+static void adjustLdmSeqStore(rawSeqStore_t* rawSeqStore, int baseDiff) {
+    size_t i = 0;
+    for (i; i < rawSeqStore->size; ++i) {
+        rawSeqStore->seq[i].matchLength += (size_t)baseDiff;
+        rawSeqStore->seq[i].litLength += (size_t)baseDiff;
+    }
+}
+
+static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore, size_t srcSize, U32 seqStoreStartPos) {
     if (rawSeqStore->size == 0)
         return;
     size_t i;
@@ -602,18 +611,12 @@ static void convertSeqStoreToRanges(rawSeqStore_t* rawSeqStore, size_t srcSize)
         printf("(%u, %u)\n", matchStart, matchEnd);
     }
     printf("size:%u\n", rawSeqStore->size);
+    /* this is maybe a lil bit janky of a way to check for a multi-block seqstore */
     if (rawSeqStore->seq[rawSeqStore->size - 1].litLength > srcSize) {
         printf("SETTING RANGEFLAG TO 2\n");
         rawSeqStore->rangeFlag = 2; /* Signifies that this is a seqstore that spans
                                            multiple blocks. */
-    }
-}
-
-static void adjustLdmSeqStore(rawSeqStore_t* rawSeqStore, int baseDiff) {
-    size_t i = 0;
-    for (i; i < rawSeqStore->size; ++i) {
-        rawSeqStore->seq[i].matchLength += (size_t)baseDiff;
-        rawSeqStore->seq[i].litLength += (size_t)baseDiff;
+        adjustLdmSeqStore(rawSeqStore, seqStoreStartPos);
     }
 }
 
@@ -641,7 +644,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore,
         if ((*rawSeqStore).rangeFlag == 0) {
             /* only convert the rawSeqStore once, in case it spans multiple blocks */
             printSeqStore(rawSeqStore);
-            convertSeqStoreToRanges(rawSeqStore, srcSize);   /* sets rangeFlag to true */
+            convertSeqStoreToRanges(rawSeqStore, srcSize, (U32)(istart - ms->window.base));   /* sets rangeFlag to true */
         }
         (*rawSeqStore).capacity = (U32)(istart - ms->window.base);
         const BYTE* const prevBase = (BYTE const*)ms->window.base;
diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 234e14d31f2..9721d76a664 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -775,6 +775,7 @@ static void getNextLdm(U32* ldmStart, U32* ldmEnd, rawSeqStore_t* ldmSeqStore,
         return;
 
     if (ldmSeqStore->rangeFlag == 2) {
+        // No need to adjust if we have an absolute seq store
         if (current > *ldmEnd) {
             /* If our current pos is greater than current match end, we need to fetch a new match */
             //printf("Getting next ldm: current: %u and currLdmEnd: %u\n", current, *ldmEnd);

From db0662291ddfac4d61ee449ab25eccdac2778ddf Mon Sep 17 00:00:00 2001
From: senhuang42 <senhuang96@fb.com>
Date: Wed, 23 Sep 2020 15:27:58 -0400
Subject: [PATCH 25/25] Add some reminders

---
 lib/compress/zstd_opt.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/compress/zstd_opt.c b/lib/compress/zstd_opt.c
index 9721d76a664..05bb750740c 100644
--- a/lib/compress/zstd_opt.c
+++ b/lib/compress/zstd_opt.c
@@ -836,7 +836,7 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
     
     U32 originalMatchLength = ldmEndAdjusted - ldmStartAdjusted;
     U32 posDifference = current - ldmStartAdjusted;
-    if (posDifference > 0 || posDifference >= originalMatchLength) {
+    if (posDifference > 0 /* TODO: change */ || posDifference >= originalMatchLength /*- MINMATCH*/ /* underflow here if we do this? */) {
         return;
     }
     printf("Original matchlen: %u - ", originalMatchLength);
@@ -851,7 +851,7 @@ static void maybeAddLdm(const rawSeqStore_t* const ldmSeqStore, ZSTD_match_t* ma
     }
     //printf("adjusted to (of(code): %u, ml %u)\n", candidateOffCode, candidateMatchLength);
     if ((*nbMatches == 0 || candidateMatchLength >= matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM) {
-        //printf("large enough, adding\n");
+        printf("large enough, adding\n");
         /*matches[*nbMatches].len = candidateMatchLength;
         matches[*nbMatches].off = candidateOffCode;
         (*nbMatches)++;*/