diff --git a/.dockerignore b/.dockerignore index 6427da1d..94a2cc8d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,5 @@ docs metrics nimcache tests +vendor/nimble/nat_traversal/vendor/miniupnp/miniupnpc/build +vendor/nimble/nat_traversal/vendor/libnatpmp-upstream/libnatpmp.a diff --git a/.github/workflows/docker-dist-tests.yml b/.github/workflows/docker-dist-tests.yml index 3299d91e..de3f723d 100644 --- a/.github/workflows/docker-dist-tests.yml +++ b/.github/workflows/docker-dist-tests.yml @@ -28,6 +28,11 @@ on: required: false type: boolean default: false + release_tests_namespace: + description: Kubernetes namespace for release tests + required: false + type: string + default: 'devnet' jobs: get-contracts-hash: @@ -57,6 +62,7 @@ jobs: tag_stable: ${{ startsWith(github.ref, 'refs/tags/') }} contract_image: "durabilitylabs/archivist-contracts:sha-${{ needs.get-contracts-hash.outputs.hash }}-dist-tests" run_release_tests: ${{ inputs.run_release_tests }} + release_tests_namespace: ${{ inputs.release_tests_namespace || 'devnet' }} secrets: inherit deploy-testnet: diff --git a/.github/workflows/docker-reusable.yml b/.github/workflows/docker-reusable.yml index 51b4376d..74f9e845 100644 --- a/.github/workflows/docker-reusable.yml +++ b/.github/workflows/docker-reusable.yml @@ -49,6 +49,11 @@ on: required: false type: string default: false + release_tests_namespace: + description: Kubernetes namespace for release tests + required: false + type: string + default: 'devnet' contract_image: description: Specifies compatible smart contract image required: false @@ -291,5 +296,6 @@ jobs: branch: ${{ needs.compute-tests-inputs.outputs.branch }} archivistdockerimage: ${{ needs.compute-tests-inputs.outputs.archivistdockerimage }} workflow_source: ${{ needs.compute-tests-inputs.outputs.workflow_source }} + namespace: ${{ inputs.release_tests_namespace || 'devnet' }} secrets: KUBE_CONFIG: ${{ secrets.DEVNET_KUBE_CONFIG }} diff --git a/archivist/archivist.nim b/archivist/archivist.nim index 7f168e31..a45ebeff 100644 --- a/archivist/archivist.nim +++ b/archivist/archivist.nim @@ -216,11 +216,8 @@ proc new*( overlayTtl = config.overlayTtl.seconds, ) - maintenance = BlockMaintainer.new( - repoStore, - interval = config.overlayMaintenanceInterval, - numberOfBlocksPerInterval = config.overlayMaintenanceNumberOfBlocks, - ) + maintenance = + BlockMaintainer.new(repoStore, interval = config.overlayMaintenanceInterval) peerStore = PeerCtxStore.new() pendingBlocks = PendingBlocksManager.new() diff --git a/archivist/archivisttypes.nim b/archivist/archivisttypes.nim index b5f18022..72baaa39 100644 --- a/archivist/archivisttypes.nim +++ b/archivist/archivisttypes.nim @@ -9,6 +9,7 @@ {.push raises: [].} +import std/math import std/tables import std/sugar @@ -25,13 +26,13 @@ export tables const # Size of blocks for storage / network exchange, - DefaultBlockSize* = NBytes 1024 * 64 - DefaultCellSize* = NBytes 2048 + DefaultBlockSize* = 512.KiBs + DefaultCellSize* = 2.KiBs # Proving defaults DefaultMaxSlotDepth* = 32 DefaultMaxDatasetDepth* = 8 - DefaultBlockDepth* = 5 + DefaultBlockDepth* = int(log2(float(DefaultBlockSize div DefaultCellSize))) DefaultCellElms* = 67 DefaultSamplesNum* = 5 diff --git a/archivist/blockexchange/engine/engine.nim b/archivist/blockexchange/engine/engine.nim index e4890bd8..d907dafa 100644 --- a/archivist/blockexchange/engine/engine.nim +++ b/archivist/blockexchange/engine/engine.nim @@ -451,10 +451,18 @@ proc blocksDeliveryHandler*( validatedBlocksDelivery.keepItIf(it.address.cid != bd.address.cid) continue - # TODO: The putBlock here should be replace by something like - - # storeManifestBlock(...) - if err =? (await self.localStore.putBlock(bd.blk)).errorOption: - error "Unable to store block", err = err.msg + if not isManifest: + error "Skipping, blocks should either be part of a tree or a manifest", + cid = bd.blk.cid + continue + + without manifest =? Manifest.decode(bd.blk), err: + error "Unable to decode manifest block", err = err.msg + validatedBlocksDelivery.keepItIf(it.address.cid != bd.address.cid) + continue + + if err =? (await self.localStore.storeManifest(manifest)).errorOption: + error "Unable to store manifest", err = err.msg validatedBlocksDelivery.keepItIf(it.address.cid != bd.address.cid) archivist_block_exchange_blocks_received.inc(validatedBlocksDelivery.len.int64) diff --git a/archivist/clock.nim b/archivist/clock.nim index 3ab40b86..45371dd9 100644 --- a/archivist/clock.nim +++ b/archivist/clock.nim @@ -30,6 +30,19 @@ proc withTimeout*( await future.cancelAndWait() raise newException(Timeout, "Timed out") +proc withTimeout*[T]( + future: Future[T], clock: Clock, expiry: SecondsSince1970 +): Future[T] {.async.} = + let timeout = clock.waitUntil(expiry) + try: + await future or timeout + finally: + await timeout.cancelAndWait() + if not future.completed: + await future.cancelAndWait() + raise newException(Timeout, "Timed out") + return future.read() + proc toBytes*(i: SecondsSince1970): seq[byte] = let asUint = cast[uint64](i) @(asUint.toBytes) diff --git a/archivist/erasure/erasure.nim b/archivist/erasure/erasure.nim index 26139d7a..876c0628 100644 --- a/archivist/erasure/erasure.nim +++ b/archivist/erasure/erasure.nim @@ -634,6 +634,12 @@ proc decode*( ## be recovered ## + logScope: + treeCid = encoded.treeCid + originalTreeCid = encoded.originalTreeCid + + trace "Preparing to decode dataset" + let emptyCid = ?emptyCid(encoded.version, encoded.hcodec, encoded.codec) params = EncodingParams.initFromEncoded(encoded, emptyCid) @@ -663,6 +669,7 @@ proc decode*( if err =? (await self.repoStore.putSomeProofs(tree, idxIter)).errorOption: return failure(err) + trace "Successfully decoded original dataset" success(), ) @@ -689,6 +696,12 @@ proc repair*( ## to avoid leaving garbage in the store ## + logScope: + treeCid = encoded.treeCid + originalTreeCid = encoded.originalTreeCid + + trace "Preparing to repair dataset" + let emptyCid = ?emptyCid(encoded.version, encoded.hcodec, encoded.codec) params = EncodingParams.initFromEncoded(encoded, emptyCid) @@ -708,8 +721,9 @@ proc repair*( "Original tree root differs from the tree root computed out of recovered data" ) - await self.repoStore.putAllProofs(tree) - , + ?await self.repoStore.putAllProofs(tree) + trace "Successfully repaired original dataset" + success(), ) # TODO: We don't get valid parity data from leopard, @@ -719,6 +733,7 @@ proc repair*( let repaired = ?(await self.encode(encoded, encoded.ecK, encoded.ecM, encoded.protectedStrategy)) + trace "Successfully re-encoded original dataset" if repaired.treeCid != encoded.treeCid: return failure( "Original tree root differs from the repaired tree root encoded out of recovered data" diff --git a/archivist/marketplace/sales/states/filled.nim b/archivist/marketplace/sales/states/filled.nim index 56195634..8b403c84 100644 --- a/archivist/marketplace/sales/states/filled.nim +++ b/archivist/marketplace/sales/states/filled.nim @@ -55,10 +55,18 @@ method run*( if onFilled =? agent.onFilled: onFilled(request, data.slotIndex) + # Add buffer past contract end so overlay data survives through the + # last proof window. Buffer covers period (last proof window) + + # proofTimeout (challenge window). Without this, maintenance can drop + # the overlay while the proving loop is still running its last proof. let expiry = await marketplace.getRequestEnd(data.requestId) + let periodicity = marketplace.periodicity() + let buffer = periodicity.seconds + marketplace.proofTimeout() + let expiryWithBuffer = expiry + buffer let cid = request.content.cid let slotIndex = data.slotIndex - if err =? (await storage.updateSlotExpiry(cid, slotIndex, expiry)).errorOption: + if err =? + (await storage.updateSlotExpiry(cid, slotIndex, expiryWithBuffer)).errorOption: return some State(SaleErrored(error: err)) when defined(archivist_system_testing_options): diff --git a/archivist/marketplace/sales/states/initialproving.nim b/archivist/marketplace/sales/states/initialproving.nim index e8305f9b..acb27e99 100644 --- a/archivist/marketplace/sales/states/initialproving.nim +++ b/archivist/marketplace/sales/states/initialproving.nim @@ -70,10 +70,14 @@ method run*( let cid = request.content.cid let slotIndex = data.slotIndex let challenge = await context.marketplace.getChallenge(slot.id) - without proof =? await storage.proveSlot(cid, slotIndex, challenge), err: + # Deadline is periodEnd - proof must be generated before the period closes. + let periodEnd = periodicity.periodEnd(provingPeriod).toSecondsSince1970 + + without proof =? + (await storage.proveSlot(cid, slotIndex, challenge).withTimeout(clock, periodEnd)), + err: error "Failed to generate initial proof", error = err.msg return some State(SaleErrored(error: err)) - let periodAtFinish = periodicity.periodOf(StorageTimestamp.init(clock.now())) if periodAtFinish != provingPeriod: warn "Failed to generate initial proof in time", diff --git a/archivist/marketplace/sales/states/proving.nim b/archivist/marketplace/sales/states/proving.nim index b75a2d2d..46a5dfcf 100644 --- a/archivist/marketplace/sales/states/proving.nim +++ b/archivist/marketplace/sales/states/proving.nim @@ -1,5 +1,6 @@ import std/options import pkg/questionable/results +import pkg/chronos import ../../../clock import ../../../logutils import ../../../utils/exceptions @@ -85,7 +86,18 @@ proc proveLoop( (await marketplace.willProofBeRequired(slotId)): let challenge = await marketplace.getChallenge(slotId) info "Generating required proof", challenge = challenge - await state.prove(slot, challenge, context, provingPeriod) + # Deadline is the end of the current proving period. Proof must be + # generated and submitted before periodEnd. + let periodicity = marketplace.periodicity() + let periodEnd = periodicity.periodEnd(provingPeriod).toSecondsSince1970 + try: + await state.prove(slot, challenge, context, provingPeriod).withTimeout( + clock, periodEnd + ) + except Timeout: + warn "Proof generation timed out", provingPeriod = provingPeriod + await waitUntilPeriod(provingPeriod + 1'u8) + continue let periodAtFinish = getCurrentPeriod() if periodAtFinish != provingPeriod: warn "Failed to generate proof in time", periodAtFinish = periodAtFinish @@ -123,7 +135,6 @@ method run*( ): Future[?State] {.async: (raises: []).} = let data = SalesAgent(machine).data let context = SalesAgent(machine).context - let clock = context.clock without request =? data.request: raiseAssert "no sale request" diff --git a/archivist/marketplacestorage.nim b/archivist/marketplacestorage.nim index 9abcf4f0..5621a096 100644 --- a/archivist/marketplacestorage.nim +++ b/archivist/marketplacestorage.nim @@ -35,8 +35,7 @@ method proveSlot*( method updateSlotExpiry*( storage: MarketplaceStorage, cid: Cid, slotIndex: uint64, expiry: StorageTimestamp ): Future[?!void] {.async: (raises: [CancelledError]).} = - # TODO: update only the slot expiry, not the expiry of the entiry dataset - await storage.node.updateExpiry(cid, expiry.toSecondsSince1970) + await storage.node.updateSlotExpiry(cid, slotIndex, expiry.toSecondsSince1970) method deleteSlot*( storage: MarketplaceStorage, cid: Cid, slotIndex: uint64 diff --git a/archivist/node.nim b/archivist/node.nim index fc0c502b..9d7df739 100644 --- a/archivist/node.nim +++ b/archivist/node.nim @@ -155,6 +155,33 @@ proc updateExpiry*( await self.updateExpiry(manifest, expiry) +proc updateSlotExpiry*( + self: ArchivistNodeRef, + manifestCid: Cid, + slotIndex: uint64, + expiry: SecondsSince1970, +): Future[?!void] {.async: (raises: [CancelledError]).} = + ## Update expiry for both the tree overlay and the slot overlay + ## + + without manifest =? await self.fetchManifest(manifestCid), error: + trace "Unable to fetch manifest for cid", manifestCid + return failure(error) + + # Validate before any mutations + if not manifest.verifiable or slotIndex.int > manifest.slotRoots.high: + error "Slot not found in manifest", manifestCid, slotIndex + return failure(newException(ArchivistError, "Slot not found in manifest")) + + # Update tree overlay expiry + ?await self.repoStore.putOverlay(manifest.treeCid, expiry = expiry) + + # Update slot overlay expiry + let slotRoot = manifest.slotRoots[slotIndex.int] + ?await self.repoStore.putOverlay(slotRoot, expiry = expiry) + + return success() + proc fetchBatched*( self: ArchivistNodeRef, cid: Cid, @@ -355,8 +382,11 @@ proc retrieve*( return await self.streamSingleBlock(cid) - # track manifest CID in overlay for cleanup - ?await self.repoStore.putOverlay(manifest.treeCid, manifestCid = cid.some) + let blk = ?await self.repoStore.storeManifest(manifest) + + if blk.cid != cid: + error "Retrieved manifest cid dont match!", original = cid, retrieved = blk.cid + return failure(newException(ArchivistError, "Retrieved manifest cid dont match!")) await self.streamEntireDataset(manifest, cid) @@ -376,19 +406,13 @@ proc deleteEntireDataset( # Deletion is a strictly local operation var store = self.networkStore.localStore - without manifestBlock =? await store.getBlock(cid), err: - return failure(err) - - without manifest =? Manifest.decode(manifestBlock), err: + without manifest =? (await self.repoStore.fetchManifest(cid)), err: return failure(err) if err =? (await self.repoStore.dropOverlay(manifest.treeCid)).errorOption: error "Error dropping manifest overlay", cid, err = err.msg return failure(err) - if err =? (await store.delBlock(cid)).errorOption: - warn "Manifest block already removed", cid, err = err.msg - success() proc delete*( @@ -523,8 +547,11 @@ proc store*( blockBatch.setLen(0) await allFutures(inFlight) - # return the first failed fut - discard inFlight.mapIt(?catch(it.read)) + for fut in inFlight: + if err =? catchAsync(?fut.read).errorOption: + error "Unable to store uploaded data", err = err.msg + return failure(err) + inFlight.setLen(0) let @@ -567,9 +594,6 @@ proc store*( # store the manifest let manifestBlk = ?await self.repoStore.storeManifest(manifest) - # track manifest CID in overlay for cleanup - ?await self.repoStore.putOverlay(treeCid, manifestCid = manifestBlk.cid.some) - info "Stored data", manifestCid = manifestBlk.cid, treeCid = treeCid, @@ -621,7 +645,6 @@ proc ensureProtectedManifest( leoDecoderProvider, self.taskpool, ) encodedManifest = ?await erasure.encode(manifest, ecK, ecM) - manifestBlk = ?await self.repoStore.storeManifest(encodedManifest) success encodedManifest @@ -737,6 +760,37 @@ proc requestStorage*( return failure err let purchase = ?await purchasing.purchase(request) + + # Clean up verifiable and slot overlays when purchase completes (success or failure). + # The original dataset overlay is not touched - it has its own TTL. + let + self = self + verifiableCid = request.content.cid + + proc cleanupPurchaseOverlays() {.async: (raises: []).} = + try: + await purchase.future + except CatchableError as exc: + trace "Purchase future failed, cleaning up overlays", error = exc.msg + + try: + without manifest =? await self.fetchManifest(verifiableCid), err: + warn "Unable to fetch manifest for purchase cleanup", + cid = verifiableCid, error = err.msg + return + + for slotRoot in manifest.slotRoots: + if err =? (await self.repoStore.dropOverlay(slotRoot)).errorOption: + warn "Error dropping slot overlay", slotRoot, error = err.msg + + if err =? (await self.repoStore.dropOverlay(manifest.treeCid)).errorOption: + warn "Error dropping verifiable overlay", + treeCid = manifest.treeCid, error = err.msg + except CancelledError: + trace "Purchase overlay cleanup cancelled" + + self.trackedFutures.track(cleanupPurchaseOverlays()) + success purchase.id proc validateVerifiableManifest(manifest: Manifest, slotSize: uint64): ?!void = @@ -770,9 +824,6 @@ proc storeSlot*( error "Validation of verifiable manifest failed", err = err.msg return failure(err) - # track manifest CID in overlay for cleanup - ?await self.repoStore.putOverlay(manifest.treeCid, manifestCid = cid.some) - if err =? (await self.updateExpiry(manifest, expiry)).errorOption: error "Unable to update manifest expiry", cid, err = err.msg return failure(err) @@ -797,7 +848,7 @@ proc storeSlot*( return failure(err) if repair: - trace "start repairing slot", slotIdx + trace "Start repairing slot", slotIdx let erasure = Erasure.new( self.networkStore, self.repoStore, leoEncoderProvider, leoDecoderProvider, self.taskpool, @@ -826,6 +877,12 @@ proc storeSlot*( manifest = manifest.slotRoots[slotIndex.int], recovered = slotRoot.toSlotCid() return failure(newException(ArchivistError, "Slot root mismatch")) + # Track verifiable manifest CID on the slot overlay for cleanup + discard + ?await self.repoStore.storeVerifiableManifest( + manifest, slotIdx = slotIndex.Natural.some, expiry = expiry + ) + trace "Slot successfully retrieved and reconstructed" return success() @@ -847,7 +904,7 @@ proc proveSlot*( trace "Prover enabled" let - manifest = ?await self.fetchManifest(cid) + manifest = ?await self.repoStore.fetchManifest(cid) builder = ?Poseidon2Builder.new( self.networkStore, self.repoStore, manifest, manifest.verifiableStrategy @@ -890,20 +947,12 @@ proc deleteSlot*( let slotCid = manifest.slotRoots[slotIdx] - # Mark slot overlay as Failure so maintenance will drop it and cleanup manifest - if err =? ( - await self.repoStore.putOverlay( - slotCid, status = OverlayStatus.Failure.some, manifestCid = cid.some - ) - ).errorOption: + # Delete slot overlay + if err =? (await self.repoStore.dropOverlay(slotCid)).errorOption: warn "Error marking slot overlay failed", err = err.msg - # Mark tree overlay as Failure so maintenance will drop it and cleanup manifest - if err =? ( - await self.repoStore.putOverlay( - manifest.treeCid, status = OverlayStatus.Failure.some, manifestCid = cid.some - ) - ).errorOption: + # Delete mainfest overlay + if err =? (await self.repoStore.dropOverlay(manifest.treeCid)).errorOption: warn "Error marking tree overlay failed", err = err.msg trace "Slot marked as failed" diff --git a/archivist/rest/api.nim b/archivist/rest/api.nim index d3364453..9acfb45e 100644 --- a/archivist/rest/api.nim +++ b/archivist/rest/api.nim @@ -240,6 +240,8 @@ proc initDataApi(node: ArchivistNodeRef, repoStore: RepoStore, router: var RestR mimetype = mimetype, ) ), error: + if error of QuotaNotEnoughError: + return RestApiResponse.error(Http413, error.msg) error "Error uploading file", exc = error.msg return RestApiResponse.error(Http500, error.msg) diff --git a/archivist/stores/maintenance.nim b/archivist/stores/maintenance.nim index 39607de6..e21cc0f1 100644 --- a/archivist/stores/maintenance.nim +++ b/archivist/stores/maintenance.nim @@ -18,7 +18,6 @@ import pkg/questionable/results import ./repostore import ../utils/timer -import ../utils/safeasynciter import ../clock import ../logutils import ../systemclock @@ -40,7 +39,6 @@ proc new*( T: type BlockMaintainer, repoStore: RepoStore, interval: Duration, - numberOfBlocksPerInterval = 100, timer = Timer.new("maintenance"), clock: Clock = SystemClock.new(), ): BlockMaintainer = @@ -53,25 +51,15 @@ proc new*( proc dropExpiredOverlays( self: BlockMaintainer ): Future[void] {.async: (raises: [CancelledError]).} = - without iter =? (await self.repoStore.listOverlays()), err: + without overlays =? ( + await self.repoStore.listOverlaysByExpiry(limit = -1, offset = 0) + ), err: warn "Unable to list overlays", err = err.msg return - defer: - if err =? (await iter.dispose()).errorOption: - warn "Error disposing overlay iterator", err = err.msg - let now = self.clock.now - for cidFut in iter: - without treeCid =? (await cidFut), err: - warn "Unable to get overlay CID from iterator", err = err.msg - continue - - without meta =? (await self.repoStore.getOverlay(treeCid)), err: - warn "Unable to get overlay metadata", treeCid, err = err.msg - continue - + for (treeCid, meta) in overlays: # Deleting - finish cleanup, if delete in progress, dropOverlay will # no-op # Failure - always drop @@ -85,10 +73,6 @@ proc dropExpiredOverlays( if err =? (await self.repoStore.dropOverlay(treeCid)).errorOption: error "Error dropping overlay", treeCid, status = meta.status, err = err.msg - if manifestCid =? meta.manifestCid: - if err =? (await self.repoStore.delBlock(manifestCid)).errorOption: - warn "Error dropping manifest", err = err.msg - await sleepAsync(1.millis) # cooperative scheduling proc start*(self: BlockMaintainer) = diff --git a/archivist/stores/repostore/operations.nim b/archivist/stores/repostore/operations.nim index 8635ec89..a916fa40 100644 --- a/archivist/stores/repostore/operations.nim +++ b/archivist/stores/repostore/operations.nim @@ -2,6 +2,7 @@ import std/sets import std/tables +import std/options import pkg/chronos import pkg/kvstore @@ -20,6 +21,7 @@ import ../../blocktype import ../../clock import ../../logutils import ../../merkletree +import ../../manifest import ../../utils logScope: @@ -169,7 +171,7 @@ proc delFromBlocksStore*( # bypases CAS for cituations like this) toDelete = ?await self.repoDs.get(keys) - # delete on disk blocks first - best effort if crash + # delete on disk blocks first - best effort if crash # occurs we might miss some blocks, but we can recover # the delete, since the metadata is still present skipped = (?await self.repoDs.delete(toDelete.toKeyRecord)).toHashSet @@ -250,7 +252,9 @@ proc tryDeleteBlocks*( self.tryDeleteBlocks(@[cid]) type BlockLeafTuple = - tuple[index: Natural, blkCid: Cid, cellCid: ?Cid, proof: ArchivistProof] + tuple[ + index: Natural, blkCid: Cid, cellCid: ?Cid, proof: ArchivistProof, data: seq[byte] + ] proc putLeafBlockMetaImpl( self: RepoStore, treeCid: Cid, blocks: seq[BlockLeafTuple] @@ -287,12 +291,16 @@ proc putLeafBlockMetaImpl( if overlayMeta.status == Deleting: return failure(newException(OverlayDeletingError, "Overlay is being deleted")) + self.deletingLock.enter(treeCid) + defer: + self.deletingLock.leave(treeCid) + var blkToLeafMap: Table[Key, (RawKVRecord, HashSet[RawKVRecord])] leafsMap: Table[Key, RawKVRecord] blocksBits = BitSeq.init(blocks.mapIt(it[0]).max() + 1) - for (index, blkCid, cellCid, proof) in blocks: + for (index, blkCid, cellCid, proof, data) in blocks: let blkKey = ?blockMetaKey(blkCid) leafKey = ?blockLeafKey(treeCidStr, index) @@ -361,7 +369,6 @@ proc putLeafBlockMetaImpl( for rec in refreshed: var record = rec - trace "Processing record", key = record.key if BlockLeafKey.ancestor(record.key): let incomingLeafRec = ?toRecord[LeafMetadata](?catch(leafsMap[record.key])) var currentLeafRec = ?toRecord[LeafMetadata](record) @@ -423,6 +430,32 @@ proc putLeafBlockMetaImpl( # cache the final overlay self.overlayCache[?overlayKey(treeCid)] = overlayMeta + # Write block data to disk and update counters (inside semaphore) + var + diskRecords: seq[RawKVRecord] + diskKeySizes: seq[(Key, int)] + seen: HashSet[Cid] + + for (index, blkCid, cellCid, proof, data) in blocks: + if data.len > 0 and not blkCid.isEmpty and blkCid notin seen: + seen.incl(blkCid) + let key = ?makePrefixKey(self.postFixLen, blkCid) + diskKeySizes.add((key, data.len)) + diskRecords.add(RawKVRecord.init(key, data)) + + if diskRecords.len > 0: + trace "Writing blocks to disk", count = diskRecords.len + let skipped = (?await self.repoDs.put(move(diskRecords))).toHashSet + + var newBlocks, newBytes = 0 + for (key, size) in diskKeySizes: + if key notin skipped: + newBytes += size + newBlocks += 1 + + if newBlocks > 0: + ?await self.updateCounters(quotaDelta = newBytes, blocksDelta = newBlocks) + success() proc putLeafBlockMeta*( @@ -430,7 +463,20 @@ proc putLeafBlockMeta*( ): Future[?!void] {.async: (raises: [CancelledError], raw: true).} = ## Put or update leaf and block metadata (plain blocks). ## - self.putLeafBlockMetaImpl(treeCid, blocks.mapIt((it[0], it[1], Cid.none, it[2]))) + self.putLeafBlockMetaImpl( + treeCid, blocks.mapIt((it[0], it[1], Cid.none, it[2], newSeq[byte]())) + ) + +proc putLeafBlockMeta*( + self: RepoStore, + treeCid: Cid, + blocks: seq[(Natural, Cid, ArchivistProof, seq[byte])], +): Future[?!void] {.async: (raises: [CancelledError], raw: true).} = + ## Put or update leaf and block metadata with block data. + ## + self.putLeafBlockMetaImpl( + treeCid, blocks.mapIt((it[0], it[1], Cid.none, it[2], it[3])) + ) proc putLeafBlockMeta*( self: RepoStore, treeCid: Cid, index: Natural, blkCid: Cid, proof: ArchivistProof @@ -452,7 +498,15 @@ proc putCellLeafBlockMeta*( self.putLeafBlockMetaImpl( treeCid, - blocks.mapIt((index: it[0], blkCid: it[2], cellCid: it[1].some, proof: it[3])), + blocks.mapIt( + ( + index: it[0], + blkCid: it[2], + cellCid: it[1].some, + proof: it[3], + data: newSeq[byte](), + ) + ), ) proc delLeafBlockMetadata*( @@ -481,16 +535,37 @@ proc delLeafBlockMetadata*( trace "Deleting leaf and block metadata" + # Mark overlay as Deleting BEFORE draining the barrier. + # This prevents new writers from entering the barrier after drain completes + # but before the atomic metadata update. + let + ovKey = ?overlayKey(treeCid) + preDeleteOverlay = ?await self.metaDs.get(ovKey, OverlayMetadata) + + var preDeleteMeta = preDeleteOverlay.val + preDeleteMeta.status = Deleting + ?await self.metaDs.tryPut(preDeleteOverlay.fromRecord(preDeleteMeta), maxRetries = 3) + # Update cache so concurrent readers see Deleting immediately + self.overlayCache[ovKey] = preDeleteMeta + + await self.deletingLock.drain(treeCid) + + # Re-fetch overlay after drain to get latest state let - existingOverlayMeta = ?await self.metaDs.get(?overlayKey(treeCid), OverlayMetadata) + existingOverlayMeta = ?await self.metaDs.get(ovKey, OverlayMetadata) uniqueIdxs = index.deduplicate() var overlayMeta = existingOverlayMeta.val if not uniqueIdxs.anyIt(it < overlayMeta.blocks.len and overlayMeta.blocks[it]): trace "No bits set in BitSeq for indices to delete, fast-path return" + # Restore previous status since nothing is actually being deleted + overlayMeta.status = preDeleteOverlay.val.status + ?await self.metaDs.tryPut( + existingOverlayMeta.fromRecord(overlayMeta), maxRetries = 3 + ) + self.overlayCache[ovKey] = overlayMeta return success() - # Continue with existing logic let treeCidStr = $treeCid leafKeys = uniqueIdxs.mapIt(?blockLeafKey(treeCidStr, it)) @@ -562,7 +637,6 @@ proc delLeafBlockMetadata*( trace "Got refreshed metadata", count = refreshed.len for rec in refreshed: var record = rec - trace "Processing record", key = record.key if BlockLeafKey.ancestor(record.key): var leaf = ?toRecord[LeafMetadata](record) leaf.val.deleted = true # mark for delete @@ -612,7 +686,7 @@ proc delLeafBlockMetadata*( ) # cache the final overlay after delete - self.overlayCache[?overlayKey(treeCid)] = overlayMeta + self.overlayCache[ovKey] = overlayMeta let toDeleteLeafMeta = @@ -666,3 +740,234 @@ proc getLeafMetadata*( return failure(err) success(leafMd.val) + +proc storeManifestBlock*( + self: RepoStore, rootCids: seq[Cid], manifest: Manifest, expiry = ZeroSeconds +): Future[?!Block] {.async: (raises: [CancelledError]), gcsafe.} = + ## Low-level manifest storage: creates/updates overlays for each rootCid + ## with manifestCid set, atomically manages BlockMetadata.refCount, and + ## writes the manifest block to disk. + ## + ## This proc is rootCid-agnostic -- callers decide whether rootCids are + ## treeCids or slotRoots. + ## + + let manifestBlk = ?manifest.toBlock + + let overlayExpiry = + if expiry != ZeroSeconds: + expiry + else: + self.clock.now() + self.overlayTtl + + var + # initial key -> record mapping + overlaysMap = rootCids.mapIt( + ( + ?overlayKey(it), + KVRecord[OverlayMetadata].init( + ?overlayKey(it), + OverlayMetadata( + manifestCid: Cid.none, + status: Pending, + expiry: overlayExpiry, + blocks: BitSeq.init(0), + ), + ), + ) + ).toTable + + # Get existing overlays from KVStore to check their manifestCid state + let overlayRecsMap = ( + ?await self.metaDs.get(toSeq(overlaysMap.keys), OverlayMetadata) + ).mapIt((it.key, it)).toTable + + # Track which overlays require a new manifest attachment for refCount updates. + var newAttachmentKeys: seq[Key] + + for (key, val) in toSeq(overlaysMap.pairs): + var rec = overlayRecsMap.getOrDefault(key, val) + # get rec from kvstore recs, or the new rec + if rec.val.manifestCid.isNone: + newAttachmentKeys.add(key) + # If explicit expiry provided, refresh expiry even when attaching manifest + if expiry != ZeroSeconds: + rec.val.expiry = overlayExpiry + rec.val.manifestCid = manifestBlk.cid.some + overlaysMap[key] = rec + else: + if rec.val.manifestCid.get != manifestBlk.cid: + trace "Existing overlay manifestCid, doesn't match provided", + existing = rec.val.manifestCid.get, provided = manifestBlk.cid + return failure( + newException( + ArchivistError, "Existing overlay manifestCid, doesn't match provided" + ) + ) + else: + # If explicit expiry provided, we need to refresh expiry on already-attached overlay + # Keep it in overlaysMap so it gets written through CAS + if expiry != ZeroSeconds: + rec.val.expiry = overlayExpiry + overlaysMap[key] = rec + else: + # manifest already set, we don't need to update it or update manifest refCount + overlaysMap.del(key) + continue + + if overlaysMap.len == 0: + trace "No overlays to attach manifest", manifestCid = manifestBlk.cid + return success manifestBlk + + let + blkMetaKey = ?blockMetaKey(manifestBlk.cid) + # init block meta with correct refCount - only count new attachments + blkMetaRec = KVRecord[BlockMetadata].init( + blkMetaKey, + BlockMetadata(cid: manifestBlk.cid, refCount: newAttachmentKeys.len.Natural), + ) + + ?await self.metaDs.tryPutAtomic( + @[blkMetaRec.toRaw] & toSeq(overlaysMap.values).mapIt(it.toRaw), + maxRetries = 10, + proc( + records: seq[RawKVRecord], conflicts: seq[Key] + ): Future[?!seq[RawKVRecord]] {.async: (raises: [CancelledError]), gcsafe.} = + var records = records.mapIt((it.key, it)).toTable + let refreshed = (?await self.metaDs.get(conflicts)).mapIt((it.key, it)).toTable + + var newAttachmentsCount = newAttachmentKeys.len + for raw in refreshed.values: + if ArchivistOverlaysKey.ancestor(raw.key): + var record = ?toRecord[OverlayMetadata](raw) + let wasNewAttachment = raw.key in newAttachmentKeys + if not record.val.manifestCid.isSome: + if not wasNewAttachment: + newAttachmentsCount.inc + # New attachment - refresh expiry if explicit + if expiry != ZeroSeconds: + record.val.expiry = overlayExpiry + record.val.manifestCid = manifestBlk.cid.some + records[record.key] = record.toRaw + elif record.val.manifestCid.get == manifestBlk.cid: + if wasNewAttachment: + newAttachmentsCount.dec + # Already attached to same manifest - refresh expiry if explicit + # Don't increment refCount (overlay already had manifestCid) + if expiry != ZeroSeconds: + record.val.expiry = overlayExpiry + # already attached, keep refreshed record in batch (no-op write) + records[record.key] = record.toRaw + else: + return failure( + newException( + ArchivistError, "Existing overlay manifestCid, doesn't match provided" + ) + ) + + records[blkMetaKey] = + if blkMetaKey in refreshed: + var rec = ?toRecord[BlockMetadata](?catch(refreshed[blkMetaKey])) + rec.val.refCount += newAttachmentsCount.Natural + rec.toRaw + else: + var rec = ?toRecord[BlockMetadata](?catch(records[blkMetaKey])) + rec.val.refCount = newAttachmentsCount.Natural + rec.toRaw + + success toSeq(records.values) + , + ) + + for rootCid in rootCids: + let key = ?overlayKey(rootCid) + self.overlayCache.del(key) + + if err =? ( + await self.repoDs.put( + RawKVRecord.init( + ?makePrefixKey(self.postFixLen, manifestBlk.cid), manifestBlk.data + ) + ) + ).errorOption: + if err of KVConflictError: + trace "Manifest already on disk, skipping write", cid = manifestBlk.cid + else: + trace "Error storing manifest", cid = manifestBlk.cid + return failure(err) + else: + ?await self.updateCounters(quotaDelta = manifestBlk.data.len, blocksDelta = 1) + if onBlock =? self.onBlockStored: + await onBlock(manifestBlk.cid) + + trace "Stored manifest block", cid = manifestBlk.cid + + success manifestBlk + +proc dropManifest*( + self: RepoStore, treeCid: Cid +): Future[?!void] {.async: (raises: [CancelledError]), gcsafe.} = + ## Detach a manifest from an overlay, atomically decrementing the + ## manifest's BlockMetadata.refCount. If refCount reaches 0, + ## tryDeleteBlocks will delete the manifest block from disk. + ## + + logScope: + treeCid = treeCid + + let key = ?overlayKey(treeCid) + without var overlayRec =? (await self.metaDs.get(key, OverlayMetadata)), err: + if err of KVStoreKeyNotFound: + trace "Overlay not found, nothing to detach" + return success() + return failure(err) + + without manifestCid =? overlayRec.val.manifestCid: + trace "Overlay has no manifest attached" + return success() + + trace "Detaching manifest from overlay", manifestCid + + overlayRec.val.manifestCid = Cid.none + + let blkMetaKey = ?blockMetaKey(manifestCid) + var blkMetaRec = ?await self.metaDs.get(blkMetaKey, BlockMetadata) + if blkMetaRec.val.refCount > 0: + blkMetaRec.val.refCount.dec + + ?await self.metaDs.tryPutAtomic( + @[overlayRec.toRaw, blkMetaRec.toRaw], + maxRetries = 10, + proc( + records: seq[RawKVRecord], conflicts: seq[Key] + ): Future[?!seq[RawKVRecord]] {.async: (raises: [CancelledError]), gcsafe.} = + var records = records.mapIt((it.key, it)).toTable + let refreshed = (?await self.metaDs.get(conflicts)).mapIt((it.key, it)).toTable + + var shouldDecRef = true + if key in refreshed: + var rec = ?toRecord[OverlayMetadata](?catch(refreshed[key])) + if rec.val.manifestCid == manifestCid.some: + rec.val.manifestCid = Cid.none + records[key] = rec.toRaw + else: + # manifest already detached or replaced by concurrent writer, skip + # keep refreshed record in batch (no-op write) + shouldDecRef = false + records[key] = rec.toRaw + + if blkMetaKey in refreshed: + var rec = ?toRecord[BlockMetadata](?catch(refreshed[blkMetaKey])) + if shouldDecRef and rec.val.refCount > 0: + rec.val.refCount.dec + records[blkMetaKey] = rec.toRaw + + success toSeq(records.values) + , + ) + + self.overlayCache.del(key) + discard ?await self.tryDeleteBlocks(manifestCid) + trace "Manifest detached from overlay", manifestCid + + success() diff --git a/archivist/stores/repostore/overlays/overlays.nim b/archivist/stores/repostore/overlays/overlays.nim index 4552504a..e6175b95 100644 --- a/archivist/stores/repostore/overlays/overlays.nim +++ b/archivist/stores/repostore/overlays/overlays.nim @@ -3,7 +3,6 @@ {.push raises: [].} import std/algorithm -import std/strutils import std/tables import pkg/chronos @@ -271,60 +270,38 @@ proc dropOverlay*( logScope: treeCid = treeCid - if treeCid in self.deletingLock: - trace "Overlay deletion already in progress, skipping" - return success() - - self.deletingLock.incl(treeCid) - defer: - self.deletingLock.excl(treeCid) - trace "Dropping overlay and cleaning up blocks" if err =? (await self.putOverlay(treeCid, status = Deleting.some)).errorOption: error "Unable to mark overlay as deleting", exc = err.msg return failure(err) - # Read overlay metadata to get manifestCid before deletion - var manifestCid: ?Cid - if meta =? (await self.getOverlay(treeCid)): - manifestCid = meta.manifestCid + while true: + without overlay =? (await self.getOverlay(treeCid)), err: + if err of KVStoreKeyNotFound: + trace "Overlay already deleted", treeCid + break + warn "Overlay missing", treeCid, err = err.msg + return failure(err) - # Query all leaf records for this tree - let - queryKey = ?blockLeafQueryKey(treeCid) - iter = ?(await query(self.metaDs, Query.init(queryKey), LeafMetadata)) - - # Extract indices from the leaf keys - var indices: seq[Natural] - for recordFut in iter: - if record =? ?catchAsync(?(await recordFut)): - # Key format: /meta/leafs/{treeCid}/{index} - # Extract index from last namespace segment - let indexStr = record.key.value - without idx =? parseInt(indexStr).catch, err: - return failure( - newException( - ValueError, "Invalid index in key: " & indexStr & " - " & err.msg - ) - ) - indices.add(idx.Natural) + var indices: seq[Natural] + for i in 0 ..< overlay.blocks.len: + if overlay.blocks[i.Natural]: + indices.add(i.Natural) + + # Delete leaf metadata and decrement refcounts (two-phase atomic) + if indices.len == 0: + break - # Delete leaf metadata and decrement refcounts (two-phase atomic) - if indices.len > 0: - ?await delLeafBlockMetadata(self, treeCid, indices) + ?await self.delLeafBlockMetadata(treeCid, indices) trace "Deleted leaf metadata and updated refcounts", count = indices.len + # Detach manifest and try to delete if refcount reaches 0 + ?await self.dropManifest(treeCid) + # Delete overlay metadata ?await self.deleteOverlay(treeCid) - if cid =? manifestCid: - let - key = ?makePrefixKey(self.postFixLen, cid) - record = ?await self.repoDs.get(key) - - ?await self.repoDs.tryDelete(record) - trace "Overlay dropped successfully" success() diff --git a/archivist/stores/repostore/store.nim b/archivist/stores/repostore/store.nim index cf4d5c4d..05feb1c1 100644 --- a/archivist/stores/repostore/store.nim +++ b/archivist/stores/repostore/store.nim @@ -29,6 +29,7 @@ import ../../logutils import ../../merkletree import ../../utils import ../../manifest +import ../../clock export blocktype, cid, overlays @@ -186,48 +187,26 @@ method putBlocks*( var totalSize = 0 - uniqueBlks: HashSet[Block] # filter out duplicate leafs for different tree branches + uniqueBlks: HashSet[Cid] let blocks = collect(newSeq): for (blk, idx, proof) in items.deduplicate(): if not blk.cid.isEmpty: totalSize += blk.data.len - uniqueBlks.incl(blk) - (idx, blk.cid, proof) + uniqueBlks.incl(blk.cid) + (idx, blk.cid, proof, blk.data) trace "Putting blocks", actualBlocks = blocks.len, totalSize if not self.available(totalSize.NBytes): return failure(newException(QuotaNotEnoughError, "Blocks would exceed quota!")) - # Atomic metadata update (leaf + block refcount) - if err =? (await self.putLeafBlockMeta(treeCid, blocks)).errorOption: - trace "Unable to store Leaf and Block Metadata", err = err.msg - return failure(err) - - trace "Writting blocks to disc", actualBlocks = blocks.len - # Write blocks to FS (best effort, idempotent) - # Build records and capture sizes before moving into put - var - records = uniqueBlks.mapIt( - RawKVRecord.init(?makePrefixKey(self.postFixLen, it.cid), it.data) - ) - keySizes = records.mapIt((it.key, it.val.len)) - - let skipped = (?await self.repoDs.put(move(records))).toHashSet - - # Count only unique blocks that were successfully written - var newBlocks, newBytes = 0 - for (key, size) in keySizes: - if key notin skipped: - newBytes += size - newBlocks += 1 - - if newBlocks > 0: - ?await self.updateCounters(quotaDelta = newBytes, blocksDelta = newBlocks) + # Atomic metadata update + ?await self.putLeafBlockMeta(treeCid, blocks) if onBlock =? self.onBlockStored: - await allFutures(uniqueBlks.mapIt(onBlock(it.cid))) + for cid in uniqueBlks: + await onBlock(cid) return success() @@ -605,21 +584,33 @@ proc release*( method storeManifest*( self: RepoStore, manifest: Manifest ): Future[?!Block] {.async: (raises: [CancelledError]), gcsafe.} = - let manifestBlk = ?manifest.toBlock + ## Store a manifest block with an overlay for the manifest's treeCid. + ## + + await self.storeManifestBlock(@[manifest.treeCid], manifest) + +proc storeVerifiableManifest*( + self: RepoStore, manifest: Manifest, slotIdx = Natural.none, expiry = ZeroSeconds +): Future[?!Block] {.async: (raises: [CancelledError]), gcsafe.} = + ## Store a verifiable manifest with overlays for slot roots. + ## If slotIdx is provided, only that slot's overlay is created. + ## Otherwise, overlays for all slot roots are created. + ## - if err =? ( - await self.putOverlay( - treeCid = manifest.treeCid, manifestCid = manifestBlk.cid.some + if not manifest.verifiable: + return failure( + newException( + ArchivistError, "storeVerifiableManifest requires a verifiable manifest" + ) ) - ).errorOption: - trace "Unable to set manifestCid for overlay metadata", - treeCid = manifest.treeCid, manifestCid = manifestBlk.cid - return failure(err) - ?await self.putBlockInternal(manifestBlk) - trace "Stored manifest block", cid = manifestBlk.cid + let rootCids = + if idx =? slotIdx: + @[manifest.slotRoots[idx]] + else: + manifest.slotRoots - success manifestBlk + await self.storeManifestBlock(rootCids, manifest, expiry) method fetchManifest*( self: RepoStore, cid: Cid diff --git a/archivist/stores/repostore/types.nim b/archivist/stores/repostore/types.nim index e561ed37..1123d72f 100644 --- a/archivist/stores/repostore/types.nim +++ b/archivist/stores/repostore/types.nim @@ -22,6 +22,7 @@ import ../../errors import ../../merkletree import ../../systemclock import ../../units +import ../../utils/asyncbarrier const DefaultOverlayTtl* = SecondsSince1970 30.days.seconds # ttl in seconds @@ -42,7 +43,7 @@ type totalBlocks*: Natural overlayTtl*: SecondsSince1970 started*: bool - deletingLock*: HashSet[Cid] + deletingLock*: KeyedBarrier[Cid] overlayCache*: Table[Key, OverlayMetadata] QuotaUsage* {.serialize.} = object @@ -146,4 +147,5 @@ func new*( quotaMaxBytes: quotaMaxBytes, overlayTtl: overlayTtl, onBlockStored: CidCallback.none, + deletingLock: KeyedBarrier[Cid].new(), ) diff --git a/archivist/utils.nim b/archivist/utils.nim index 78c8db2d..91ca3745 100644 --- a/archivist/utils.nim +++ b/archivist/utils.nim @@ -20,8 +20,10 @@ import ./utils/asyncheapqueue import ./utils/fileutils import ./utils/asynciter import ./utils/safeasynciter +import ./utils/asyncbarrier -export asyncheapqueue, fileutils, asynciter, safeasynciter, chronos, bitseqs +export + asyncheapqueue, fileutils, asynciter, safeasynciter, chronos, bitseqs, asyncbarrier when defined(posix): import os, posix diff --git a/archivist/utils/asyncbarrier.nim b/archivist/utils/asyncbarrier.nim new file mode 100644 index 00000000..4c1cbe39 --- /dev/null +++ b/archivist/utils/asyncbarrier.nim @@ -0,0 +1,67 @@ +{.push raises: [].} + +import std/tables + +import pkg/chronos + +type + AsyncBarrier* = ref object + count: int + event: AsyncEvent + + KeyedBarrier*[K] = ref object + barriers: Table[K, AsyncBarrier] + +proc new*(T: type AsyncBarrier): AsyncBarrier = + AsyncBarrier(event: newAsyncEvent()) + +proc count*(self: AsyncBarrier): int = + self.count + +proc enter*(self: AsyncBarrier) = + self.count += 1 + self.event.clear() + +proc leave*(self: AsyncBarrier) = + doAssert self.count > 0, "leave() called without matching enter()" + self.count -= 1 + if self.count == 0: + self.event.fire() + +proc drain*(self: AsyncBarrier): Future[void] {.async: (raises: [CancelledError]).} = + if self.count <= 0: + return + await self.event.wait() + +proc new*[K](T: type KeyedBarrier[K]): KeyedBarrier[K] = + KeyedBarrier[K]() + +proc contains*[K](self: KeyedBarrier[K], key: K): bool = + key in self.barriers + +proc count*[K](self: KeyedBarrier[K], key: K): int = + self.barriers.withValue(key, barrier): + return barrier[].count + return 0 + +proc enter*[K](self: KeyedBarrier[K], key: K) = + self.barriers.withValue(key, barrier): + barrier[].enter() + return + + let barrier = AsyncBarrier.new() + barrier.enter() + self.barriers[key] = barrier + +proc leave*[K](self: KeyedBarrier[K], key: K) = + self.barriers.withValue(key, barrier): + barrier[].leave() + + if barrier[].count <= 0: + self.barriers.del(key) + +proc drain*[K]( + self: KeyedBarrier[K], key: K +): Future[void] {.async: (raises: [CancelledError]).} = + self.barriers.withValue(key, barrier): + await barrier[].drain() diff --git a/openapi.yaml b/openapi.yaml index 3d29082e..41ef0f5b 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -553,6 +553,8 @@ paths: text/plain: schema: type: string + "413": + description: The upload would exceed the node's storage quota "422": description: The mimetype of the filename is invalid "500": diff --git a/tests/archivist/blockexchange/engine/testblockexc.nim b/tests/archivist/blockexchange/engine/testblockexc.nim index 8ec6cac5..e4a5db3c 100644 --- a/tests/archivist/blockexchange/engine/testblockexc.nim +++ b/tests/archivist/blockexchange/engine/testblockexc.nim @@ -10,6 +10,7 @@ import pkg/archivist/blockexchange import pkg/archivist/chunker import pkg/archivist/discovery import pkg/archivist/blocktype as bt +import pkg/archivist/manifest import ../../../asynctest import ../../examples @@ -23,13 +24,26 @@ asyncchecksuite "NetworkStore engine - 2 nodes": peerCtx1, peerCtx2: BlockExcPeerCtx pricing1, pricing2: Pricing blocks1, blocks2: seq[bt.Block] + manifest1, manifest2: Manifest pendingBlocks1, pendingBlocks2: seq[BlockHandle] setup: blocks1 = (await makeRandomBlocks(datasetSize = 2048, blockSize = 256'nb)).tryGet blocks2 = (await makeRandomBlocks(datasetSize = 2048, blockSize = 256'nb)).tryGet - nodeCmps1 = generateNodes(1, blocks1).components[0] - nodeCmps2 = generateNodes(1, blocks2).components[0] + nodeCmps1 = generateNodes(1).components[0] + nodeCmps2 = generateNodes(1).components[0] + + # Store blocks as trees on each node + manifest1 = (await storeDataGetManifest(nodeCmps1.localStore, blocks1)).tryGet() + manifest2 = (await storeDataGetManifest(nodeCmps2.localStore, blocks2)).tryGet() + + # Attach manifests to their local overlays + discard (await nodeCmps1.localStore.storeManifest(manifest1)).tryGet() + discard (await nodeCmps2.localStore.storeManifest(manifest2)).tryGet() + + # Pre-create overlays on the opposite nodes so they can receive leaf blocks + discard (await nodeCmps1.localStore.storeManifest(manifest2)).tryGet() + discard (await nodeCmps2.localStore.storeManifest(manifest1)).tryGet() await allFuturesThrowing( nodeCmps1.switch.start(), @@ -40,12 +54,18 @@ asyncchecksuite "NetworkStore engine - 2 nodes": nodeCmps2.engine.start(), ) - # initialize our want lists - pendingBlocks1 = - blocks2[0 .. 3].mapIt(nodeCmps1.pendingBlocks.getWantHandle(it.cid)) + # Use leaf addresses for want handles + pendingBlocks1 = blocks2[0 .. 3].mapIt( + nodeCmps1.pendingBlocks.getWantHandle( + BlockAddress.init(manifest2.treeCid, blocks2.find(it).Natural) + ) + ) - pendingBlocks2 = - blocks1[0 .. 3].mapIt(nodeCmps2.pendingBlocks.getWantHandle(it.cid)) + pendingBlocks2 = blocks1[0 .. 3].mapIt( + nodeCmps2.pendingBlocks.getWantHandle( + BlockAddress.init(manifest1.treeCid, blocks1.find(it).Natural) + ) + ) pricing1 = Pricing.example() pricing2 = Pricing.example() @@ -81,12 +101,20 @@ asyncchecksuite "NetworkStore engine - 2 nodes": await allFuturesThrowing(allFinished(pendingBlocks2)).wait(10.seconds) check: - (await allFinished(blocks1[0 .. 3].mapIt(nodeCmps2.localStore.getBlock(it.cid)))) + ( + await allFinished( + (0 .. 3).mapIt(nodeCmps2.localStore.getBlock(manifest1.treeCid, it.Natural)) + ) + ) .filterIt(it.completed and it.read.isOk) .mapIt($it.read.get.cid) .sorted(cmp[string]) == blocks1[0 .. 3].mapIt($it.cid).sorted(cmp[string]) - (await allFinished(blocks2[0 .. 3].mapIt(nodeCmps1.localStore.getBlock(it.cid)))) + ( + await allFinished( + (0 .. 3).mapIt(nodeCmps1.localStore.getBlock(manifest2.treeCid, it.Natural)) + ) + ) .filterIt(it.completed and it.read.isOk) .mapIt($it.read.get.cid) .sorted(cmp[string]) == blocks2[0 .. 3].mapIt($it.cid).sorted(cmp[string]) @@ -96,12 +124,22 @@ asyncchecksuite "NetworkStore engine - 2 nodes": check peerCtx2.account .? address == pricing2.address.some test "Should send want-have for block": - let blk = bt.Block.new("Block 1".toBytes).tryGet() - let blkFut = nodeCmps1.pendingBlocks.getWantHandle(blk.cid) - (await nodeCmps2.localStore.putBlock(blk)).tryGet() + let + blk = bt.Block.new("Block 1".toBytes).tryGet() + senderManifest = + (await storeDataGetManifest(nodeCmps2.localStore, @[blk])).tryGet() + + # Attach manifest to sender's overlay + discard (await nodeCmps2.localStore.storeManifest(senderManifest)).tryGet() + # Pre-create overlay on receiver + discard (await nodeCmps1.localStore.storeManifest(senderManifest)).tryGet() + + let + leafAddr = BlockAddress.init(senderManifest.treeCid, 0.Natural) + blkFut = nodeCmps1.pendingBlocks.getWantHandle(leafAddr) let entry = WantListEntry( - address: blk.address, + address: leafAddr, priority: 1, cancel: false, wantType: WantType.WantBlock, @@ -111,12 +149,15 @@ asyncchecksuite "NetworkStore engine - 2 nodes": peerCtx1.peerWants.add(entry) check nodeCmps2.engine.taskQueue.pushOrUpdateNoWait(peerCtx1).isOk - check eventually (await nodeCmps1.localStore.hasBlock(blk.cid)).tryGet() + check eventually ( + await nodeCmps1.localStore.hasBlock(senderManifest.treeCid, 0.Natural) + ).tryGet() check eventually (await blkFut) == blk test "Should get blocks from remote": - let blocks = - await allFinished(blocks2[4 .. 7].mapIt(nodeCmps1.networkStore.getBlock(it.cid))) + let blocks = await allFinished( + (4 .. 7).mapIt(nodeCmps1.networkStore.getBlock(manifest2.treeCid, it.Natural)) + ) check blocks.mapIt(it.read().tryGet()) == blocks2[4 .. 7] @@ -126,20 +167,25 @@ asyncchecksuite "NetworkStore engine - 2 nodes": # should fail retrieving block from remote check not await blk.cid in nodeCmps1.networkStore - # second trigger blockexc to resolve any pending requests - # for the block - (await nodeCmps2.networkStore.putBlock(blk)).tryGet() + # Store as tree on sender + let senderManifest = + (await storeDataGetManifest(nodeCmps2.localStore, @[blk])).tryGet() + + # Attach manifest to sender's overlay + discard (await nodeCmps2.localStore.storeManifest(senderManifest)).tryGet() + # Pre-create overlay on receiver + discard (await nodeCmps1.localStore.storeManifest(senderManifest)).tryGet() # should succeed retrieving block from remote - check await nodeCmps1.networkStore.getBlock(blk.cid).withTimeout(100.millis) - # should succeed + check await nodeCmps1.networkStore + .getBlock(senderManifest.treeCid, 0.Natural) + .withTimeout(100.millis) test "Should receive payments for blocks that were sent": - discard - await allFinished(blocks2[4 .. 7].mapIt(nodeCmps2.networkStore.putBlock(it))) - - discard - await allFinished(blocks2[4 .. 7].mapIt(nodeCmps1.networkStore.getBlock(it.cid))) + # blocks2 already stored as tree on nodeCmps2, overlay on nodeCmps1 + discard await allFinished( + (4 .. 7).mapIt(nodeCmps1.networkStore.getBlock(manifest2.treeCid, it.Natural)) + ) let channel = !peerCtx1.paymentChannel @@ -151,10 +197,32 @@ asyncchecksuite "NetworkStore - multiple nodes": var nodes: seq[NodesComponents] blocks: seq[bt.Block] + manifest0, manifest1, manifest2, manifest3: Manifest setup: blocks = (await makeRandomBlocks(datasetSize = 4096, blockSize = 256'nb)).tryGet nodes = generateNodes(5) + + # Store blocks as trees on each source node (4 blocks per node) + manifest0 = + (await storeDataGetManifest(nodes[0].localStore, blocks[0 .. 3])).tryGet() + manifest1 = + (await storeDataGetManifest(nodes[1].localStore, blocks[4 .. 7])).tryGet() + manifest2 = + (await storeDataGetManifest(nodes[2].localStore, blocks[8 .. 11])).tryGet() + manifest3 = + (await storeDataGetManifest(nodes[3].localStore, blocks[12 .. 15])).tryGet() + + # Attach manifests to their local overlays + discard (await nodes[0].localStore.storeManifest(manifest0)).tryGet() + discard (await nodes[1].localStore.storeManifest(manifest1)).tryGet() + discard (await nodes[2].localStore.storeManifest(manifest2)).tryGet() + discard (await nodes[3].localStore.storeManifest(manifest3)).tryGet() + + # Pre-create overlays on downloader (node 4) for the shards it will request + discard (await nodes[4].localStore.storeManifest(manifest0)).tryGet() + discard (await nodes[4].localStore.storeManifest(manifest3)).tryGet() + for e in nodes: await e.engine.start() @@ -170,14 +238,18 @@ asyncchecksuite "NetworkStore - multiple nodes": downloader = nodes[4].networkStore engine = downloader.engine - # Add blocks from 1st peer to want list - let - downloadCids = blocks[0 .. 3].mapIt(it.cid) & blocks[12 .. 15].mapIt(it.cid) - - pendingBlocks = downloadCids.mapIt(engine.pendingBlocks.getWantHandle(it)) - - for i in 0 .. 15: - (await nodes[i div 4].networkStore.engine.localStore.putBlock(blocks[i])).tryGet() + # Add blocks from 1st and 4th peers to want list using leaf addresses + let pendingBlocks = + (0 .. 3).mapIt( + engine.pendingBlocks.getWantHandle( + BlockAddress.init(manifest0.treeCid, it.Natural) + ) + ) & + (0 .. 3).mapIt( + engine.pendingBlocks.getWantHandle( + BlockAddress.init(manifest3.treeCid, it.Natural) + ) + ) await connectNodes(nodes) await sleepAsync(100.millis) @@ -185,24 +257,36 @@ asyncchecksuite "NetworkStore - multiple nodes": await allFuturesThrowing(allFinished(pendingBlocks)) check: - (await allFinished(downloadCids.mapIt(downloader.localStore.getBlock(it)))) + ( + await allFinished( + (0 .. 3).mapIt(downloader.localStore.getBlock(manifest0.treeCid, it.Natural)) & + (0 .. 3).mapIt( + downloader.localStore.getBlock(manifest3.treeCid, it.Natural) + ) + ) + ) .filterIt(it.completed and it.read.isOk) .mapIt($it.read.get.cid) - .sorted(cmp[string]) == downloadCids.mapIt($it).sorted(cmp[string]) + .sorted(cmp[string]) == + (blocks[0 .. 3] & blocks[12 .. 15]).mapIt($it.cid).sorted(cmp[string]) test "Should exchange blocks with multiple nodes": let downloader = nodes[4].networkStore engine = downloader.engine - # Add blocks from 1st peer to want list + # Add blocks from 1st and 4th peers to want list let - pendingBlocks1 = blocks[0 .. 3].mapIt(engine.pendingBlocks.getWantHandle(it.cid)) - pendingBlocks2 = - blocks[12 .. 15].mapIt(engine.pendingBlocks.getWantHandle(it.cid)) - - for i in 0 .. 15: - (await nodes[i div 4].networkStore.engine.localStore.putBlock(blocks[i])).tryGet() + pendingBlocks1 = (0 .. 3).mapIt( + engine.pendingBlocks.getWantHandle( + BlockAddress.init(manifest0.treeCid, it.Natural) + ) + ) + pendingBlocks2 = (0 .. 3).mapIt( + engine.pendingBlocks.getWantHandle( + BlockAddress.init(manifest3.treeCid, it.Natural) + ) + ) await connectNodes(nodes) await sleepAsync(100.millis) diff --git a/tests/archivist/blockexchange/engine/testengine.nim b/tests/archivist/blockexchange/engine/testengine.nim index 0d2378ba..66bfbced 100644 --- a/tests/archivist/blockexchange/engine/testengine.nim +++ b/tests/archivist/blockexchange/engine/testengine.nim @@ -336,10 +336,10 @@ asyncchecksuite "NetworkStore engine handlers": let resolved = await allFinished(pending) check resolved.mapIt(it.read) == blocks - # Non-leaf deliveries are NOT persisted - hasBlock should fail + # Non-leaf non-manifest deliveries are NOT persisted for b in blocks: let present = await engine.localStore.hasBlock(b.cid) - check present.tryGet() + check not present.tryGet() test "Should send payments for received blocks": let diff --git a/tests/archivist/helpers/nodeutils.nim b/tests/archivist/helpers/nodeutils.nim index 4772cb38..153a1d30 100644 --- a/tests/archivist/helpers/nodeutils.nim +++ b/tests/archivist/helpers/nodeutils.nim @@ -163,8 +163,6 @@ proc generateNodes*( announceAddrs = @[listenAddr], store = discoveryDs, ) - for blk in blocks: - (waitFor store.putBlock(blk)).tryGet() (store, discovery) let diff --git a/tests/archivist/marketplace/sales/mockstorage.nim b/tests/archivist/marketplace/sales/mockstorage.nim index 4981fa2d..294b4b7f 100644 --- a/tests/archivist/marketplace/sales/mockstorage.nim +++ b/tests/archivist/marketplace/sales/mockstorage.nim @@ -11,6 +11,7 @@ type MockStorage* = ref object of StorageInterface proveSlotResult: ?!Groth16Proof updateSlotExpiryResult: ?!void slotFailedResult: ?!void + proveSlotShouldHang*: bool storeSlotCalls: seq[(Cid, uint64, uint64, StorageTimestamp, bool)] proveSlotCalls: seq[(Cid, uint64, ProofChallenge)] updateSlotExpiryCalls: seq[(Cid, uint64, StorageTimestamp)] @@ -71,6 +72,8 @@ method proveSlot*( mock: MockStorage, cid: Cid, slotIndex: uint64, challenge: ProofChallenge ): Future[?!Groth16Proof] {.async: (raises: [CancelledError]).} = mock.proveSlotCalls.add((cid, slotIndex, challenge)) + if mock.proveSlotShouldHang: + await sleepAsync(1.hours) mock.proveSlotResult method updateSlotExpiry*( diff --git a/tests/archivist/marketplace/sales/states/testfilled.nim b/tests/archivist/marketplace/sales/states/testfilled.nim index 34c26bb3..25098e17 100644 --- a/tests/archivist/marketplace/sales/states/testfilled.nim +++ b/tests/archivist/marketplace/sales/states/testfilled.nim @@ -49,8 +49,11 @@ suite "sales state 'filled'": slot.host = await marketplace.getSigner() marketplace.filled = @[slot] - let expectedExpiry = 123'StorageTimestamp - marketplace.requestEnds[request.id] = expectedExpiry + let requestEnd = 123'StorageTimestamp + marketplace.requestEnds[request.id] = requestEnd + # Expiry includes a buffer of period + proofTimeout past contract end + let periodicity = marketplace.periodicity() + let expectedExpiry = requestEnd + periodicity.seconds + marketplace.proofTimeout() let next = await state.run(agent) check !next of SaleProving check storage.updateSlotExpiryCalls.len > 0 diff --git a/tests/archivist/marketplace/sales/states/testinitialproving.nim b/tests/archivist/marketplace/sales/states/testinitialproving.nim index 37aab94c..63b38ee0 100644 --- a/tests/archivist/marketplace/sales/states/testinitialproving.nim +++ b/tests/archivist/marketplace/sales/states/testinitialproving.nim @@ -96,3 +96,21 @@ asyncchecksuite "sales state 'initialproving'": let next = await future check !next of SaleErrored + + test "switches to errored state when proof generation times out": + storage.proveSlotShouldHang = true + + let future = state.run(agent) + await allowProofToStart() + + # Yield to let state machine enter proveSlot and register the timeout + check eventually storage.proveSlotCalls.len == 1 + + # Advance clock past periodEnd to trigger clock-based timeout + let periodicity = marketplace.periodicity() + let provingPeriod = periodicity.periodOf(clock.now()) + let periodEnd = periodicity.periodEnd(provingPeriod) + clock.set(periodEnd.toSecondsSince1970 + 1) + + let next = await future + check !next of SaleErrored diff --git a/tests/archivist/marketplace/sales/states/testproving.nim b/tests/archivist/marketplace/sales/states/testproving.nim index b0f69f4b..67ca1f31 100644 --- a/tests/archivist/marketplace/sales/states/testproving.nim +++ b/tests/archivist/marketplace/sales/states/testproving.nim @@ -101,3 +101,30 @@ asyncchecksuite "sales state 'proving'": check challenge == marketplace.proofChallenge await future.cancelAndWait() + + test "continues proving loop after proof generation timeout": + marketplace.slotState[slot.id] = SlotState.Filled + marketplace.setProofRequired(slot.id, true) + storage.proveSlotShouldHang = true + + let future = state.run(agent) + await marketplace.advanceToNextPeriod() + + # Wait for the proving loop to enter proveSlot and register the timeout + check eventually storage.proveSlotCalls.len == 1 + + # Advance clock past periodEnd to trigger clock-based timeout + let periodicity = marketplace.periodicity() + let provingPeriod = periodicity.periodOf(clock.now()) + let periodEnd = periodicity.periodEnd(provingPeriod) + clock.set(periodEnd.toSecondsSince1970 + 1) + + # Loop should survive the timeout and wait for next period. + # Finish the slot so the loop exits cleanly. + storage.proveSlotShouldHang = false + marketplace.setProofRequired(slot.id, false) + marketplace.slotState[slot.id] = SlotState.Finished + await marketplace.advanceToNextPeriod() + + check eventually future.finished + check !(future.read()) of SalePayout diff --git a/tests/archivist/node/tempnode.nim b/tests/archivist/node/tempnode.nim index aa7ae683..02865042 100644 --- a/tests/archivist/node/tempnode.nim +++ b/tests/archivist/node/tempnode.nim @@ -8,6 +8,8 @@ import pkg/archivist/discovery import pkg/archivist/stores import pkg/archivist/blockexchange import pkg/archivist/node +import pkg/archivist/clock +import pkg/archivist/systemclock type TemporaryNode* = ref object repoDs: KVStore @@ -24,11 +26,12 @@ type TemporaryNode* = ref object networkStore: NetworkStore node: ArchivistNodeRef -proc initializeLocalStore(temporary: TemporaryNode) = +proc initializeLocalStore(temporary: TemporaryNode, clock: Clock = SystemClock.new()) = temporary.tp = Taskpool.new(num_threads = 4) temporary.repoDs = SQLiteKVStore.new(SqliteMemory, temporary.tp).tryGet() temporary.metaDs = SQLiteKVStore.new(SqliteMemory, temporary.tp).tryGet() - temporary.localStore = RepoStore.new(temporary.repoDs, temporary.metaDs) + temporary.localStore = + RepoStore.new(temporary.repoDs, temporary.metaDs, clock = clock) proc initializeNetwork(temporary: TemporaryNode) = temporary.p2p = newStandardSwitch() @@ -71,9 +74,11 @@ proc initializeNode(temporary: TemporaryNode) = Taskpool.new(), ) -proc create*(_: type TemporaryNode): Future[TemporaryNode] {.async.} = +proc create*( + _: type TemporaryNode, clock: Clock = SystemClock.new() +): Future[TemporaryNode] {.async.} = let temporary = TemporaryNode() - temporary.initializeLocalStore() + temporary.initializeLocalStore(clock) temporary.initializeNetwork() temporary.initializePendingBlocks() temporary.initializeDiscovery() diff --git a/tests/archivist/node/testnode.nim b/tests/archivist/node/testnode.nim index 94bc5fd6..41b5a872 100644 --- a/tests/archivist/node/testnode.nim +++ b/tests/archivist/node/testnode.nim @@ -16,6 +16,12 @@ import pkg/archivistdht/discv5/protocol as discv5 import pkg/archivist/logutils import pkg/archivist/stores import pkg/archivist/marketplace/contracts +import pkg/archivist/marketplace/purchasing +import pkg/archivist/marketplace/purchasing/purchase +import pkg/archivist/marketplace/sales +import pkg/archivist/marketplace/availability/store +import pkg/archivist/marketplace/storageinterface +import pkg/archivist/marketplace/node {.all.} import pkg/archivist/blockexchange import pkg/archivist/chunker import pkg/archivist/slots @@ -33,11 +39,19 @@ import ../../asynctest import ../examples import ../helpers import ../helpers/mockmarketplace +import ../helpers/mockclock +import ../helpers/mocktimer import ../slots/helpers +import archivist/stores/maintenance + import ./helpers import ./tempnode +import std/importutils +privateAccess(MarketplaceNode) # enable access to private fields +privateAccess(ArchivistNode) # enable access to private fields + proc overlayCount( repo: RepoStore ): Future[?!int] {.async: (raises: [CancelledError]).} = @@ -382,3 +396,294 @@ suite "Test Node - Purchase request": "Attempt to proceed with protected manifest with parameters " & "3/2 but required: 4/2" check (await overlayCount(localStore)).tryGet() == overlaysBefore + +suite "Test Node - Purchase overlay cleanup": + var temporary: TemporaryNode + var node: ArchivistNodeRef + var localStore: RepoStore + var networkStore: NetworkStore + var marketplace: MockMarketplace + var clock: MockClock + + setup: + temporary = await TemporaryNode.create() + node = temporary.node + localStore = temporary.localStore + networkStore = temporary.networkStore + clock = MockClock.new() + marketplace = MockMarketplace.new(clock) + + # Wire up marketplace so node.requestStorage works + let + availDs = SQLiteKVStore.new(SqliteMemory, Taskpool.new()).tryGet() + availability = AvailabilityStore.new(availDs) + node.marketplace = + MarketplaceNode( + clock: clock, + purchasing: Purchasing.new(marketplace, clock), + sales: Sales.new(marketplace, clock, availability, StorageInterface()), + ).some + + teardown: + node.marketplace = MarketplaceNode.none + await temporary.destroy() + + test "Should cleanup verifiable and slot overlays when purchase finishes": + let + referenceBlocks = ( + await makeRandomBlocks( + datasetSize = 4 * DefaultBlockSize.int, blockSize = DefaultBlockSize + ) + ).tryGet() + referenceManifest = + (await storeDataGetManifest(localStore, referenceBlocks)).tryGet() + manifestBlk = (await localStore.storeManifest(referenceManifest)).tryGet() + + # 1 overlay for the original dataset + check (await overlayCount(localStore)).tryGet() == 1 + + let + quotaBefore = localStore.quotaUsedBytes + blocksBefore = localStore.totalBlocks + + let purchaseId = ( + await node.requestStorage( + cid = manifestBlk.cid, + nodes = 5, + tolerance = 2, + duration = 100'StorageDuration, + pricePerBytePerSecond = 1'TokensPerSecond, + proofProbability = 3.u256, + expiry = 200'StorageDuration, + collateralPerByte = 1'Tokens, + ) + ).tryGet() + + # Overlays: original + protected/verifiable + 5 slot overlays = 7 + check (await overlayCount(localStore)).tryGet() == 7 + # Erasure coding added blocks, so quota and block count grew + check localStore.quotaUsedBytes > quotaBefore + check localStore.totalBlocks > blocksBefore + + # Fulfill the purchase - triggers PurchaseSubmitted -> PurchaseStarted + check eventually marketplace.requested.len > 0 + let request = marketplace.requested[0] + let requestEnd = StorageTimestamp.init(clock.now() + 42) + marketplace.requestEnds[request.id] = requestEnd + marketplace.emitRequestFulfilled(request.id) + + # Advance clock past request end - triggers PurchaseStarted -> PurchaseFinished + clock.set(requestEnd.toSecondsSince1970 + 1) + + # Wait for purchase to complete + let purchasing = node.marketplace .? purchasing + check eventually purchasing .? getPurchase(purchaseId) .? finished == true.some + + # Wait for cleanup callback to run + await sleepAsync(500.milliseconds) + + # Original overlay remains, protected/verifiable + 5 slot overlays dropped + check (await overlayCount(localStore)).tryGet() == 1 + # Quota and block count back to just original data + check localStore.quotaUsedBytes == quotaBefore + check localStore.totalBlocks == blocksBefore + + test "Should cleanup verifiable and slot overlays when purchase fails": + let + referenceBlocks = ( + await makeRandomBlocks( + datasetSize = 4 * DefaultBlockSize.int, blockSize = DefaultBlockSize + ) + ).tryGet() + referenceManifest = + (await storeDataGetManifest(localStore, referenceBlocks)).tryGet() + manifestBlk = (await localStore.storeManifest(referenceManifest)).tryGet() + + # 1 overlay for the original dataset + check (await overlayCount(localStore)).tryGet() == 1 + + let + quotaBefore = localStore.quotaUsedBytes + blocksBefore = localStore.totalBlocks + + let purchaseId = ( + await node.requestStorage( + cid = manifestBlk.cid, + nodes = 5, + tolerance = 2, + duration = 100'StorageDuration, + pricePerBytePerSecond = 1'TokensPerSecond, + proofProbability = 3.u256, + expiry = 200'StorageDuration, + collateralPerByte = 1'Tokens, + ) + ).tryGet() + + # Overlays: original + protected/verifiable + 5 slot overlays = 7 + check (await overlayCount(localStore)).tryGet() == 7 + # Erasure coding added blocks, so quota and block count grew + check localStore.quotaUsedBytes > quotaBefore + check localStore.totalBlocks > blocksBefore + + # Fulfill the purchase, then fail it + check eventually marketplace.requested.len > 0 + let request = marketplace.requested[0] + let requestEnd = StorageTimestamp.init(clock.now() + 42) + marketplace.requestEnds[request.id] = requestEnd + marketplace.emitRequestFulfilled(request.id) + + # Wait for PurchaseStarted, then emit failure + await sleepAsync(100.milliseconds) + marketplace.emitRequestFailed(request.id) + + # Wait for purchase to complete (with error) + let purchasing = node.marketplace .? purchasing + check eventually purchasing .? getPurchase(purchaseId) .? finished == true.some + + # Wait for cleanup callback to run + await sleepAsync(100.milliseconds) + + # Original overlay remains, protected/verifiable + 5 slot overlays dropped + check (await overlayCount(localStore)).tryGet() == 1 + # Quota and block count back to just original data + check localStore.quotaUsedBytes == quotaBefore + check localStore.totalBlocks == blocksBefore + + test "Should cleanup all manifests and leave repo empty after full lifecycle": + let + referenceBlocks = ( + await makeRandomBlocks( + datasetSize = 4 * DefaultBlockSize.int, blockSize = DefaultBlockSize + ) + ).tryGet() + referenceManifest = + (await storeDataGetManifest(localStore, referenceBlocks)).tryGet() + manifestBlk = (await localStore.storeManifest(referenceManifest)).tryGet() + originalTreeCid = referenceManifest.treeCid + + # 1 overlay for the original dataset + check (await overlayCount(localStore)).tryGet() == 1 + + let purchaseId = ( + await node.requestStorage( + cid = manifestBlk.cid, + nodes = 5, + tolerance = 2, + duration = 100'StorageDuration, + pricePerBytePerSecond = 1'TokensPerSecond, + proofProbability = 3.u256, + expiry = 200'StorageDuration, + collateralPerByte = 1'Tokens, + ) + ).tryGet() + + # Overlays: original + protected/verifiable + 5 slot overlays = 7 + check (await overlayCount(localStore)).tryGet() == 7 + + # Fulfill the purchase - triggers PurchaseSubmitted -> PurchaseStarted + check eventually marketplace.requested.len > 0 + let request = marketplace.requested[0] + let requestEnd = StorageTimestamp.init(clock.now() + 42) + marketplace.requestEnds[request.id] = requestEnd + marketplace.emitRequestFulfilled(request.id) + + # Advance clock past request end - triggers PurchaseStarted -> PurchaseFinished + clock.set(requestEnd.toSecondsSince1970 + 1) + + # Wait for purchase to complete + let purchasing = node.marketplace .? purchasing + check eventually purchasing .? getPurchase(purchaseId) .? finished == true.some + + # Wait for cleanup callback to run + await sleepAsync(100.milliseconds) + + # Only original overlay remains after purchase completes + check (await overlayCount(localStore)).tryGet() == 1 + + # Simulate data expiry: drop the original overlay too + (await localStore.dropOverlay(originalTreeCid)).tryGet() + + # Everything has been cleaned up - repo is empty + check (await overlayCount(localStore)).tryGet() == 0 + check localStore.quotaUsedBytes == 0.NBytes + check localStore.totalBlocks == 0.Natural + +suite "Test Node - Maintenance expiry with requestStorage": + var temporary: TemporaryNode + var node: ArchivistNodeRef + var localStore: RepoStore + var networkStore: NetworkStore + var marketplace: MockMarketplace + var clock: MockClock + var mockTimer: MockTimer + var maintainer: BlockMaintainer + + setup: + clock = MockClock.new() + clock.set(1000) + temporary = await TemporaryNode.create(clock) + node = temporary.node + localStore = temporary.localStore + networkStore = temporary.networkStore + marketplace = MockMarketplace.new(clock) + mockTimer = MockTimer.new() + maintainer = + BlockMaintainer.new(localStore, 1.days, timer = mockTimer, clock = clock) + + let + availDs = SQLiteKVStore.new(SqliteMemory, Taskpool.new()).tryGet() + availability = AvailabilityStore.new(availDs) + node.marketplace = + MarketplaceNode( + clock: clock, + purchasing: Purchasing.new(marketplace, clock), + sales: Sales.new(marketplace, clock, availability, StorageInterface()), + ).some + + teardown: + await maintainer.stop() + node.marketplace = MarketplaceNode.none + await temporary.destroy() + + test "Should cleanup manifests via maintenance while purchase is in-flight": + let + referenceBlocks = ( + await makeRandomBlocks( + datasetSize = 4 * DefaultBlockSize.int, blockSize = DefaultBlockSize + ) + ).tryGet() + referenceManifest = + (await storeDataGetManifest(localStore, referenceBlocks)).tryGet() + manifestBlk = (await localStore.storeManifest(referenceManifest)).tryGet() + originalTreeCid = referenceManifest.treeCid + + let + quotaOriginal = localStore.quotaUsedBytes + blocksOriginal = localStore.totalBlocks + + check (await overlayCount(localStore)).tryGet() == 1 + + let purchaseId = ( + await node.requestStorage( + cid = manifestBlk.cid, + nodes = 5, + tolerance = 2, + duration = 100'StorageDuration, + pricePerBytePerSecond = 1'TokensPerSecond, + proofProbability = 3.u256, + expiry = 200'StorageDuration, + collateralPerByte = 1'Tokens, + ) + ).tryGet() + + check (await overlayCount(localStore)).tryGet() == 7 + check localStore.quotaUsedBytes > quotaOriginal + + clock.set(1000 + 30 * 24 * 3600 + 100) # well past 30-day TTL + + maintainer.start() + await mockTimer.invokeCallback() + + check (await overlayCount(localStore)).tryGet() == 0 + check localStore.quotaUsedBytes == 0.NBytes + check localStore.totalBlocks == 0.Natural diff --git a/tests/archivist/slots/backends/helpers.nim b/tests/archivist/slots/backends/helpers.nim index ce323251..721add77 100644 --- a/tests/archivist/slots/backends/helpers.nim +++ b/tests/archivist/slots/backends/helpers.nim @@ -80,8 +80,8 @@ func jsonToProofInput*( big: BigInt[256] hash: Poseidon2Hash data: array[32, byte] - assert bool(big.fromDecimal(it.str)) - assert data.marshal(big, littleEndian) + doAssert bool(big.fromDecimal(it.str)) + doAssert data.marshal(big, littleEndian) Poseidon2Hash.fromBytes(data).get ).concat # flatten out elements @@ -93,7 +93,7 @@ func jsonToProofInput*( var big: BigInt[254] hash: Poseidon2Hash - assert bool(big.fromDecimal(it.getStr)) + doAssert bool(big.fromDecimal(it.getStr)) hash.fromBig(big) hash ) @@ -104,7 +104,7 @@ func jsonToProofInput*( var big: BigInt[254] hash: Poseidon2Hash - assert bool(big.fromDecimal(it.str)) + doAssert bool(big.fromDecimal(it.str)) hash.fromBig(big) hash ) @@ -113,7 +113,7 @@ func jsonToProofInput*( var big: BigInt[254] hash: Poseidon2Hash - assert bool(big.fromDecimal(inputJson["dataSetRoot"].str)) + doAssert bool(big.fromDecimal(inputJson["dataSetRoot"].str)) hash.fromBig(big) hash @@ -121,7 +121,7 @@ func jsonToProofInput*( var big: BigInt[254] hash: Poseidon2Hash - assert bool(big.fromDecimal(inputJson["slotRoot"].str)) + doAssert bool(big.fromDecimal(inputJson["slotRoot"].str)) hash.fromBig(big) hash @@ -129,7 +129,7 @@ func jsonToProofInput*( var big: BigInt[254] hash: Poseidon2Hash - assert bool(big.fromDecimal(inputJson["entropy"].str)) + doAssert bool(big.fromDecimal(inputJson["entropy"].str)) hash.fromBig(big) hash diff --git a/tests/archivist/slots/sampler/testsampler.nim b/tests/archivist/slots/sampler/testsampler.nim index a34d5232..20b8edfe 100644 --- a/tests/archivist/slots/sampler/testsampler.nim +++ b/tests/archivist/slots/sampler/testsampler.nim @@ -23,6 +23,10 @@ import ../backends/helpers import ../helpers import ../../helpers +let + blockCells = DefaultBlockSize div DefaultCellSize + blockDepth = DefaultBlockDepth + suite "Test Sampler - control samples": var inputData: string @@ -36,7 +40,8 @@ suite "Test Sampler - control samples": test "Should verify control samples": let - blockCells = 32 + numSlotBlocks = proofInput.nCellsPerSlot div blockCells + slotDepth = ceilingLog2(numSlotBlocks) cellIdxs = proofInput.entropy.cellIndices(proofInput.slotRoot, proofInput.nCellsPerSlot, 5) @@ -48,13 +53,13 @@ suite "Test Sampler - control samples": cellProof = Poseidon2Proof.init( cellIdx.toCellInBlk(blockCells), proofInput.nCellsPerSlot, - sample.merklePaths[0 ..< 5], + sample.merklePaths[0 ..< blockDepth], ).tryGet slotProof = Poseidon2Proof.init( cellIdx.toBlkInSlot(blockCells), proofInput.nCellsPerSlot, - sample.merklePaths[5 ..< 9], + sample.merklePaths[blockDepth ..< blockDepth + slotDepth], ).tryGet cellData = sample.cellData @@ -64,9 +69,13 @@ suite "Test Sampler - control samples": check slotProof.verify(slotLeaf, proofInput.slotRoot).tryGet test "Should verify control dataset root": - let datasetProof = Poseidon2Proof.init( - proofInput.slotIndex, proofInput.nSlotsPerDataSet, proofInput.slotProof[0 ..< 4] - ).tryGet + let + datasetDepth = ceilingLog2(proofInput.nSlotsPerDataSet) + datasetProof = Poseidon2Proof.init( + proofInput.slotIndex, + proofInput.nSlotsPerDataSet, + proofInput.slotProof[0 ..< datasetDepth], + ).tryGet check datasetProof.verify(proofInput.slotRoot, proofInput.datasetRoot).tryGet @@ -140,13 +149,15 @@ suite "Test Sampler": sample = (await sampler.getSample(cellIdx, slotTreeCid, slotRoot)).tryGet cellProof = Poseidon2Proof.init( - cellIdx.toCellInBlk(nBlockCells), nSlotCells, sample.merklePaths[0 ..< 5] + cellIdx.toCellInBlk(nBlockCells), + nSlotCells, + sample.merklePaths[0 ..< blockDepth], ).tryGet slotProof = Poseidon2Proof.init( cellIdx.toBlkInSlot(nBlockCells), nSlotCells, - sample.merklePaths[5 ..< sample.merklePaths.len], + sample.merklePaths[blockDepth ..< sample.merklePaths.len], ).tryGet cellData = sample.cellData diff --git a/tests/archivist/slots/sampler/testutils.nim b/tests/archivist/slots/sampler/testutils.nim index a6b13dca..ca9fb828 100644 --- a/tests/archivist/slots/sampler/testutils.nim +++ b/tests/archivist/slots/sampler/testutils.nim @@ -23,7 +23,7 @@ import ../../helpers import ../../examples import ../../merkletree/helpers -asyncchecksuite "Test proof sampler utils": +suite "Test proof sampler utils": let cellsPerBlock = DefaultBlockSize div DefaultCellSize var @@ -89,7 +89,7 @@ asyncchecksuite "Test proof sampler utils": check: slotCellIndices(3) == getExpectedIndices(3) - for (input, expected) in [(10, 0), (31, 0), (32, 1), (63, 1), (64, 2)]: + for (input, expected) in [(10, 0), (255, 0), (256, 1), (511, 1), (512, 2)]: test "Can get slotBlockIndex from slotCellIndex (" & $input & " -> " & $expected & ")": let slotBlockIndex = toBlkInSlot(input, numCells = cellsPerBlock) @@ -97,7 +97,7 @@ asyncchecksuite "Test proof sampler utils": check: slotBlockIndex == expected - for (input, expected) in [(10, 10), (31, 31), (32, 0), (63, 31), (64, 0)]: + for (input, expected) in [(10, 10), (255, 255), (256, 0), (511, 255), (512, 0)]: test "Can get blockCellIndex from slotCellIndex (" & $input & " -> " & $expected & ")": let blockCellIndex = toCellInBlk(input, numCells = cellsPerBlock) diff --git a/tests/archivist/stores/repostore/overlays/testcells.nim b/tests/archivist/stores/repostore/overlays/testcells.nim index a9156751..ff8db0b3 100644 --- a/tests/archivist/stores/repostore/overlays/testcells.nim +++ b/tests/archivist/stores/repostore/overlays/testcells.nim @@ -257,6 +257,7 @@ proc testCells*( check (await innerRepo.blockRefCount(blk.cid)).tryGet() == 3.Natural check innerRepo.quotaUsedBytes == 256.NBytes + check innerRepo.totalBlocks == 1.Natural ( await innerRepo.delLeafBlockMetadata( @@ -266,6 +267,7 @@ proc testCells*( check not (await blk.cid in innerRepo) check innerRepo.quotaUsedBytes == 0.NBytes + check innerRepo.totalBlocks == 0.Natural proc runFsSqliteTests() = let repoDir = createTempDir("archivist-", "-repostore") diff --git a/tests/archivist/stores/repostore/overlays/testconcurrent.nim b/tests/archivist/stores/repostore/overlays/testconcurrent.nim index 987d1069..f8e506a1 100644 --- a/tests/archivist/stores/repostore/overlays/testconcurrent.nim +++ b/tests/archivist/stores/repostore/overlays/testconcurrent.nim @@ -91,6 +91,9 @@ proc testConcurrent*( let inconsistencies = (await repo.verifyBlockBitState(treeCid)).tryGet() check inconsistencies.len == 0 + check repo.quotaUsedBytes == NBytes(800 + 801 + 802) + check repo.totalBlocks == 3.Natural + test "Sequential CAS updates handle retries correctly": let blk1 = createTestBlock(840) @@ -201,6 +204,9 @@ proc testConcurrent*( check (await repo.getBlock(blk3.cid)).isErr check (await repo.getBlock(blk4.cid)).isErr + check repo.quotaUsedBytes == NBytes(900 + 901) + check repo.totalBlocks == 2.Natural + test "putBlocks aborts when delete started first": let blk1 = createTestBlock(910) @@ -241,26 +247,30 @@ proc testConcurrent*( ) ).tryGet() - (await repo.putBlocks(treeCid, @[(blk1, 0.Natural, proof1)])).tryGet() - - let - putFuture = repo.putBlocks( - treeCid, @[(blk2, 1.Natural, proof2), (blk3, 2.Natural, proof3)] - ) - dropFuture = repo.dropOverlay(treeCid) - - await allFutures(@[putFuture, dropFuture]) + # TODO: the correct way to test this is with a mock of the kvstore, + # but that breaks the structure of the tests and it's a significant + # amount of effort - but this is one of the few legitimate usages + # of mocking ;) + let putFuture = repo.putBlocks( + treeCid, + @[ + (blk1, 0.Natural, proof1), + (blk2, 1.Natural, proof2), + (blk3, 2.Natural, proof3), + ], + ) + (await repo.dropOverlay(treeCid)).tryGet let putResult = await putFuture - let dropResult = await dropFuture - - check dropResult.isOk if putResult.isErr: check putResult.error() of OverlayDeletingError check (await repo.getOverlay(treeCid)).isErr + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural + proc runFsSqliteTests() = let repoDir = createTempDir("archivist-", "-repostore") diff --git a/tests/archivist/stores/repostore/overlays/testlifecycle.nim b/tests/archivist/stores/repostore/overlays/testlifecycle.nim index 56946e12..f8f64acf 100644 --- a/tests/archivist/stores/repostore/overlays/testlifecycle.nim +++ b/tests/archivist/stores/repostore/overlays/testlifecycle.nim @@ -58,14 +58,33 @@ proc testLifecycle*( await after() test "Should drop overlay and remove leafs, blocks, and metadata": - let blk = createTestBlock(120) - let (treeCid, index) = (await putBlockWithOverlay(repo, blk)).tryGet() + let + blk = createTestBlock(120) + (manifest, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var blocks = BitSeq.init(1) + blocks.setBit(0) + + ( + await repo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = blocks + ) + ).tryGet() + (await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)])).tryGet() + let manifestBlk = (await repo.storeManifest(manifest)).tryGet() + + check repo.quotaUsedBytes > 0.NBytes + check repo.totalBlocks > 0.Natural (await repo.dropOverlay(treeCid)).tryGet() - let leafResult = await repo.getLeafMetadata(treeCid, index) - let blockResult = await repo.getBlock(blk.cid) - let overlayResult = await repo.getOverlay(treeCid) + let + leafResult = await repo.getLeafMetadata(treeCid, 0.Natural) + blockResult = await repo.getBlock(blk.cid) + overlayResult = await repo.getOverlay(treeCid) + manifestResult = await repo.getBlock(manifestBlk.cid) check leafResult.isErr check leafResult.error() of BlockNotFoundError @@ -73,6 +92,10 @@ proc testLifecycle*( check blockResult.error() of BlockNotFoundError check overlayResult.isErr check overlayResult.error() of KVStoreKeyNotFound + check manifestResult.isErr + check manifestResult.error() of BlockNotFoundError + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural test "Should only decrement refcount for shared blocks when one overlay is dropped": let @@ -102,12 +125,34 @@ proc testLifecycle*( (await repo.putBlocks(treeCid2, @[(shared, 1.Natural, proof2)])).tryGet() check (await repo.blockRefCount(shared.cid)).tryGet() == 2.Natural + let + manifest1 = Manifest.new( + treeCid = treeCid1, + blockSize = NBytes(shared.data.len), + datasetSize = NBytes(shared.data.len + extra1.data.len), + ) + + manifest2 = Manifest.new( + treeCid = treeCid2, + blockSize = NBytes(shared.data.len), + datasetSize = NBytes(extra2.data.len + shared.data.len), + ) + + manifestBlk1 = (await repo.storeManifest(manifest1)).tryGet() + manifestBlk2 = (await repo.storeManifest(manifest2)).tryGet() + + quotaBeforeDrop = repo.quotaUsedBytes + blocksBeforeDrop = repo.totalBlocks + (await repo.dropOverlay(treeCid1)).tryGet() check (await repo.blockRefCount(shared.cid)).tryGet() == 1.Natural check (await repo.getBlock(shared.cid)).isOk check (await repo.getOverlay(treeCid1)).isErr + check repo.quotaUsedBytes < quotaBeforeDrop + check repo.totalBlocks < blocksBeforeDrop + test "Should drop empty overlay metadata": let treeCid = Cid.example @@ -601,6 +646,9 @@ proc testLifecycle*( check tmpLeafRes.isErr check tmpLeafRes.error() of BlockNotFoundError + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural + test "Should drop tmp overlay metadata when body is cancelled": let realTreeCid = Cid.example var @@ -668,6 +716,473 @@ proc testLifecycle*( check blkRes.isErr check blkRes.error() of BlockNotFoundError + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural + + test "Should delete verifiable manifest when no slot overlays remain": + var blocks: seq[bt.Block] + for size in [100, 101, 102, 103]: + blocks.add(createTestBlock(size)) + + let + (baseManifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() + protManifest = Manifest.new( + manifest = baseManifest, + treeCid = treeCid, + datasetSize = NBytes(8 * 103), + ecK = 2, + ecM = 2, + ) + + var slotRoots: seq[Cid] + for _ in 0 ..< 4: + slotRoots.add(Cid.example) + + let verManifest = Manifest.new(protManifest, Cid.example, slotRoots).tryGet() + + var bits = BitSeq.init(blocks.len) + for i in 0 ..< blocks.len: + bits.setBit(i) + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some, blocks = bits)).tryGet() + + var blkProofs: seq[(bt.Block, Natural, ArchivistProof)] + for i, blk in blocks: + blkProofs.add((blk, i.Natural, tree.getProof(i).tryGet())) + + (await repo.putBlocks(treeCid, blkProofs)).tryGet() + + let + protManifestBlk = (await repo.storeManifest(protManifest)).tryGet() + verManifestBlk = (await repo.storeVerifiableManifest(verManifest)).tryGet() + + # Drop tree overlay: protected manifest deleted, verifiable manifest untouched + (await repo.dropOverlay(treeCid)).tryGet() + + check (await repo.getBlock(protManifestBlk.cid)).isErr + check (await repo.getBlock(verManifestBlk.cid)).isOk + + # Drop all slot overlays; last one deletes the verifiable manifest + for slotRoot in slotRoots: + (await repo.dropOverlay(slotRoot)).tryGet() + + check (await repo.getBlock(verManifestBlk.cid)).isErr + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural + + test "Should skip verifiable manifest delete when slot overlays still exist": + var blocks: seq[bt.Block] + for size in [100, 101, 102, 103]: + blocks.add(createTestBlock(size)) + + let + (baseManifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() + protManifest = Manifest.new( + manifest = baseManifest, + treeCid = treeCid, + datasetSize = NBytes(8 * 103), + ecK = 2, + ecM = 2, + ) + + var slotRoots: seq[Cid] + for _ in 0 ..< 4: + slotRoots.add(Cid.example) + + let verManifest = Manifest.new(protManifest, Cid.example, slotRoots).tryGet() + + var bits = BitSeq.init(blocks.len) + for i in 0 ..< blocks.len: + bits.setBit(i) + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some, blocks = bits)).tryGet() + + var blkProofs: seq[(bt.Block, Natural, ArchivistProof)] + for i, blk in blocks: + blkProofs.add((blk, i.Natural, tree.getProof(i).tryGet())) + + (await repo.putBlocks(treeCid, blkProofs)).tryGet() + + let + protManifestBlk = (await repo.storeManifest(protManifest)).tryGet() + verManifestBlk = (await repo.storeVerifiableManifest(verManifest)).tryGet() + verManifestBytes = verManifestBlk.data.len.NBytes + + # Drop tree overlay: data blocks + protected manifest gone + (await repo.dropOverlay(treeCid)).tryGet() + + for blk in blocks: + check (await repo.getBlock(blk.cid)).isErr + + check (await repo.getBlock(protManifestBlk.cid)).isErr + check (await repo.getBlock(verManifestBlk.cid)).isOk + check repo.quotaUsedBytes == verManifestBytes + check repo.totalBlocks == 1.Natural + + # Drop all but the last slot overlay; verifiable manifest persists + for i in 0 ..< slotRoots.len - 1: + (await repo.dropOverlay(slotRoots[i])).tryGet() + + check (await repo.getBlock(verManifestBlk.cid)).isOk + + # Drop last slot overlay; verifiable manifest is now deleted + (await repo.dropOverlay(slotRoots[^1])).tryGet() + + check (await repo.getBlock(verManifestBlk.cid)).isErr + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural + + test "Should handle protected and verifiable manifests independently": + var blocks: seq[bt.Block] + for size in [100, 101, 102, 103]: + blocks.add(createTestBlock(size)) + + let + (baseManifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() + protManifest = Manifest.new( + manifest = baseManifest, + treeCid = treeCid, + datasetSize = NBytes(8 * 103), + ecK = 2, + ecM = 2, + ) + + var slotRoots: seq[Cid] + for _ in 0 ..< 4: + slotRoots.add(Cid.example) + + let verManifest = Manifest.new(protManifest, Cid.example, slotRoots).tryGet() + + var bits = BitSeq.init(blocks.len) + for i in 0 ..< blocks.len: + bits.setBit(i) + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some, blocks = bits)).tryGet() + + var blkProofs: seq[(bt.Block, Natural, ArchivistProof)] + for i, blk in blocks: + blkProofs.add((blk, i.Natural, tree.getProof(i).tryGet())) + + (await repo.putBlocks(treeCid, blkProofs)).tryGet() + + # Step 1: store both manifests + let + protManifestBlk = (await repo.storeManifest(protManifest)).tryGet() + verManifestBlk = (await repo.storeVerifiableManifest(verManifest)).tryGet() + dataBytes = NBytes(100 + 101 + 102 + 103) + protManifestBytes = protManifestBlk.data.len.NBytes + verManifestBytes = verManifestBlk.data.len.NBytes + + # All blocks + both manifests should be on disk + for blk in blocks: + check (await repo.getBlock(blk.cid)).isOk + check (await repo.getBlock(protManifestBlk.cid)).isOk + check (await repo.getBlock(verManifestBlk.cid)).isOk + check repo.quotaUsedBytes == dataBytes + protManifestBytes + verManifestBytes + check repo.totalBlocks == (blocks.len + 2).Natural + + # Step 2: drop tree overlay - data blocks and protected manifest are cleaned up + (await repo.dropOverlay(treeCid)).tryGet() + + for blk in blocks: + check (await repo.getBlock(blk.cid)).isErr + + check (await repo.getBlock(protManifestBlk.cid)).isErr + + # Verifiable manifest survives (slot overlays still exist) + check (await repo.getBlock(verManifestBlk.cid)).isOk + check repo.quotaUsedBytes == verManifestBytes + check repo.totalBlocks == 1.Natural + + # Step 3: drop all slot overlays - verifiable manifest is cleaned up + for slotRoot in slotRoots: + (await repo.dropOverlay(slotRoot)).tryGet() + + check (await repo.getBlock(verManifestBlk.cid)).isErr + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural + + test "Should track manifest refCount across slot overlays": + var blocks: seq[bt.Block] + for size in [100, 101, 102, 103]: + blocks.add(createTestBlock(size)) + + let + (baseManifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() + protManifest = Manifest.new( + manifest = baseManifest, + treeCid = treeCid, + datasetSize = NBytes(8 * 103), + ecK = 2, + ecM = 2, + ) + + var slotRoots: seq[Cid] + for _ in 0 ..< 4: + slotRoots.add(Cid.example) + + let verManifest = Manifest.new(protManifest, Cid.example, slotRoots).tryGet() + + # Store verifiable manifest - should create 4 slot overlays with refCount=4 + let verManifestBlk = (await repo.storeVerifiableManifest(verManifest)).tryGet() + + check (await repo.blockRefCount(verManifestBlk.cid)).tryGet() == 4.Natural + + # Drop one slot overlay - refCount should be 3 + (await repo.dropOverlay(slotRoots[0])).tryGet() + check (await repo.blockRefCount(verManifestBlk.cid)).tryGet() == 3.Natural + + # Drop two more - refCount should be 1 + (await repo.dropOverlay(slotRoots[1])).tryGet() + (await repo.dropOverlay(slotRoots[2])).tryGet() + check (await repo.blockRefCount(verManifestBlk.cid)).tryGet() == 1.Natural + + # Drop last - manifest block should be deleted (refCount 0) + (await repo.dropOverlay(slotRoots[3])).tryGet() + check (await repo.getBlock(verManifestBlk.cid)).isErr + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural + + test "Should cleanup manifest exactly once when slot overlays are dropped concurrently": + var blocks: seq[bt.Block] + for size in [100, 101, 102, 103]: + blocks.add(createTestBlock(size)) + + let + (baseManifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() + protManifest = Manifest.new( + manifest = baseManifest, + treeCid = treeCid, + datasetSize = NBytes(8 * 103), + ecK = 2, + ecM = 2, + ) + + var slotRoots: seq[Cid] + for _ in 0 ..< 4: + slotRoots.add(Cid.example) + + let verManifest = Manifest.new(protManifest, Cid.example, slotRoots).tryGet() + let verManifestBlk = (await repo.storeVerifiableManifest(verManifest)).tryGet() + + check (await repo.blockRefCount(verManifestBlk.cid)).tryGet() == 4.Natural + + # Drop all slot overlays concurrently - the original race condition scenario + var dropFutures: seq[Future[?!void]] + for slotRoot in slotRoots: + dropFutures.add(repo.dropOverlay(slotRoot)) + + await allFutures(dropFutures) + + for fut in dropFutures: + check fut.value.isOk + + # Manifest should be fully cleaned up + check (await repo.getBlock(verManifestBlk.cid)).isErr + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural + + test "Should attach only one slot overlay when slotIdx is provided": + var blocks: seq[bt.Block] + for size in [100, 101, 102, 103]: + blocks.add(createTestBlock(size)) + + let + (baseManifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() + protManifest = Manifest.new( + manifest = baseManifest, + treeCid = treeCid, + datasetSize = NBytes(8 * 103), + ecK = 2, + ecM = 2, + ) + + var slotRoots: seq[Cid] + for _ in 0 ..< 4: + slotRoots.add(Cid.example) + + let verManifest = Manifest.new(protManifest, Cid.example, slotRoots).tryGet() + + # Store with slotIdx=1 - only slot 1 overlay should be created + let verManifestBlk = ( + await repo.storeVerifiableManifest(verManifest, slotIdx = 1.Natural.some) + ).tryGet() + + # Only one overlay created, so refCount should be 1 + check (await repo.blockRefCount(verManifestBlk.cid)).tryGet() == 1.Natural + + # Only slot 1 overlay should have manifestCid + let overlay1 = (await repo.getOverlay(slotRoots[1])).tryGet() + check overlay1.manifestCid == verManifestBlk.cid.some + + # Other slot overlays should not exist + for i in [0, 2, 3]: + check (await repo.getOverlay(slotRoots[i])).isErr + + # Dropping the single slot overlay should delete the manifest + (await repo.dropOverlay(slotRoots[1])).tryGet() + check (await repo.getBlock(verManifestBlk.cid)).isErr + + test "Should reject slotIdx for non-verifiable manifest": + let + blk = createTestBlock(100) + (manifest, _) = makeManifestAndTree(@[blk]).tryGet() + + let result = + await repo.storeVerifiableManifest(manifest, slotIdx = 0.Natural.some) + check result.isErr + + test "Should fire onBlockStored callback when storing manifest": + let + blk = createTestBlock(100) + (manifest, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + + var callbackCid: ?Cid + repo.onBlockStored = ( + proc(cid: Cid): Future[void] {.async: (raises: []).} = + callbackCid = cid.some + ).some + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some)).tryGet() + let manifestBlk = (await repo.storeManifest(manifest)).tryGet() + + check callbackCid.isSome + check callbackCid == manifestBlk.cid.some + + test "Should handle idempotent storeManifest without double-counting refCount": + let + blk = createTestBlock(100) + (manifest, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some)).tryGet() + + # Store manifest first time + let manifestBlk1 = (await repo.storeManifest(manifest)).tryGet() + check (await repo.blockRefCount(manifestBlk1.cid)).tryGet() == 1.Natural + + let quotaAfterFirst = repo.quotaUsedBytes + + # Store same manifest again - overlay already has manifestCid, + # so overlaysToUpdate should be empty and refCount unchanged + let manifestBlk2 = (await repo.storeManifest(manifest)).tryGet() + check manifestBlk1.cid == manifestBlk2.cid + check (await repo.blockRefCount(manifestBlk1.cid)).tryGet() == 1.Natural + + # Quota should not increase (block already on disk) + check repo.quotaUsedBytes == quotaAfterFirst + + test "Should refresh expiry on existing overlay when storing manifest with explicit expiry": + # Create a protected manifest first (use 4 blocks for ecK=2, ecM=2 = 4 slots) + var blocks: seq[bt.Block] + for size in [100, 101, 102, 103]: + blocks.add(createTestBlock(size)) + + let + (baseManifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() + protManifest = Manifest.new( + manifest = baseManifest, + treeCid = treeCid, + datasetSize = NBytes(8 * 103), + ecK = 2, + ecM = 2, + ) + + # Create 4 slot roots (numSlots = ecK + ecM = 4) + var slotRoots: seq[Cid] + for _ in 0 ..< 4: + slotRoots.add(Cid.example) + + let verManifest = !Manifest.new(protManifest, Cid.example, slotRoots) + let newExpiry = now + 2000 + + # Store verifiable manifest first (without explicit expiry) + let manifestBlk1 = (await repo.storeVerifiableManifest(verManifest)).tryGet() + + # Get the slot overlay (slot 0) - check expiry was set from first call (TTL-based) + let slotOverlay1 = (await repo.getOverlay(slotRoots[0])).tryGet() + check slotOverlay1.manifestCid == manifestBlk1.cid.some + let firstExpiry = slotOverlay1.expiry + + # Store same verifiable manifest again with explicit expiry - should refresh expiry + # without incrementing refCount (overlay already has manifestCid) + let manifestBlk2 = + (await repo.storeVerifiableManifest(verManifest, expiry = newExpiry)).tryGet() + check manifestBlk1.cid == manifestBlk2.cid + + # Expiry should be updated to explicit value, not the first TTL-based value + let slotOverlay2 = (await repo.getOverlay(slotRoots[0])).tryGet() + check slotOverlay2.expiry == newExpiry # Explicit expiry should be applied + check slotOverlay2.expiry != firstExpiry # Should be different from TTL-based + + # RefCount should remain 4 (one per slot root) since we didn't add new attachments + check (await repo.blockRefCount(manifestBlk1.cid)).tryGet() == 4.Natural + + test "Should not double-count refCount when storing same slot concurrently": + var blocks: seq[bt.Block] + for size in [100, 101, 102, 103]: + blocks.add(createTestBlock(size)) + + let + (baseManifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() + protManifest = Manifest.new( + manifest = baseManifest, + treeCid = treeCid, + datasetSize = NBytes(8 * 103), + ecK = 2, + ecM = 2, + ) + + var slotRoots: seq[Cid] + for _ in 0 ..< 4: + slotRoots.add(Cid.example) + + let + verManifest = !Manifest.new(protManifest, Cid.example, slotRoots) + newExpiry = now + 3000 + + let futures = + @[ + repo.storeVerifiableManifest( + verManifest, slotIdx = 0.Natural.some, expiry = newExpiry + ), + repo.storeVerifiableManifest( + verManifest, slotIdx = 0.Natural.some, expiry = newExpiry + ), + ] + + await allFutures(futures) + + for fut in futures: + check fut.read.isOk + + let manifestBlk = futures[0].read.tryGet() + check futures[1].read.tryGet().cid == manifestBlk.cid + check (await repo.blockRefCount(manifestBlk.cid)).tryGet() == 1.Natural + check (await repo.getOverlay(slotRoots[0])).tryGet().expiry == newExpiry + + test "Should succeed dropManifest on overlay without manifestCid": + let treeCid = Cid.example + + (await repo.putOverlay(treeCid = treeCid, status = Completed.some)).tryGet() + + # dropManifest on overlay with no manifestCid should be a no-op + (await repo.dropManifest(treeCid)).tryGet() + + # Overlay should still exist and be unchanged + let overlay = (await repo.getOverlay(treeCid)).tryGet() + check overlay.status == Completed + check overlay.manifestCid.isNone + proc runFsSqliteTests() = let repoDir = createTempDir("archivist-", "-repostore") diff --git a/tests/archivist/stores/repostore/overlays/testmetadata.nim b/tests/archivist/stores/repostore/overlays/testmetadata.nim index cc17f163..286d5fc7 100644 --- a/tests/archivist/stores/repostore/overlays/testmetadata.nim +++ b/tests/archivist/stores/repostore/overlays/testmetadata.nim @@ -153,6 +153,8 @@ proc testMetadata*( # Set with block (await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)])).tryGet() check (await repo.hasBlock(treeCid, 0.Natural)).tryGet() == true + check repo.quotaUsedBytes == 100.NBytes + check repo.totalBlocks == 1.Natural # Out of range (fast-path, no store hit) check (await repo.hasBlock(treeCid, 10.Natural)).tryGet() == false @@ -180,6 +182,8 @@ proc testMetadata*( let blkRes = await repo.getBlock(shared.cid) check blkRes.isErr + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural test "Should handle non-contiguous indices in putBlocks (BitSeq length fix)": let diff --git a/tests/archivist/stores/testmaintenance.nim b/tests/archivist/stores/testmaintenance.nim index c48f02f0..dacc08a9 100644 --- a/tests/archivist/stores/testmaintenance.nim +++ b/tests/archivist/stores/testmaintenance.nim @@ -7,23 +7,25 @@ ## This file may not be copied, modified, or distributed except according to ## those terms. -import std/sets - import pkg/chronos import pkg/kvstore import pkg/taskpools import pkg/questionable import pkg/questionable/results import pkg/stew/byteutils +import pkg/stew/bitseqs import pkg/libp2p/multicodec import pkg/archivist/blocktype as bt +import pkg/archivist/merkletree import pkg/archivist/stores +import pkg/archivist/utils/asyncbarrier import ../../asynctest import ../helpers/mocktimer import ../helpers/mockclock +import ../helpers import ../examples import archivist/stores/maintenance @@ -155,21 +157,25 @@ suite "BlockMaintainer": check (await repo.getOverlay(treeCid)).isErr - test "Should skip overlay actively being deleted (runtime lock)": + test "Should handle concurrent dropOverlay on same treeCid": let treeCid = Cid.example (await repo.putOverlay(treeCid, status = Deleting.some, expiry = 200)).tryGet() - # Simulate an active deletion by holding the lock - repo.deletingLock.incl(treeCid) + # Simulate an in-flight put by entering the barrier + repo.deletingLock.enter(treeCid) maintainer.start() + # Timer callback calls dropOverlay, which enters delLeafBlockMetadata + # and blocks on acquire (waiting for the put to finish). + # But since this overlay has no blocks, the loop breaks before + # reaching delLeafBlockMetadata, so it completes and deletes the overlay. await mockTimer.invokeCallback() - # Overlay should still exist because the lock prevented re-deletion - check (await repo.getOverlay(treeCid)).isOk + # Overlay is gone - dropOverlay found no blocks and cleaned up metadata + check (await repo.getOverlay(treeCid)).isErr - repo.deletingLock.excl(treeCid) + repo.deletingLock.leave(treeCid) test "Should drop expired Pending overlay": let treeCid = Cid.example @@ -190,3 +196,112 @@ suite "BlockMaintainer": await mockTimer.invokeCallback() check (await repo.getOverlay(treeCid)).isOk + + test "Should return quota to zero when dropping expired overlay with manifest": + let + blk = bt.Block.new("test-data-for-maintenance".toBytes).tryGet() + (manifest, tree) = makeManifestAndTree(@[blk]).tryGet() + treeCid = tree.rootCid.tryGet() + proof = tree.getProof(0).tryGet() + + var blocks = BitSeq.init(1) + blocks.setBit(0) + + ( + await repo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = blocks, expiry = 50 + ) + ).tryGet() + + (await repo.putBlocks(treeCid, @[(blk, 0.Natural, proof)])).tryGet() + discard (await repo.storeManifest(manifest)).tryGet() + + check repo.quotaUsedBytes > 0.NBytes + check repo.totalBlocks > 0.Natural + + maintainer.start() + await mockTimer.invokeCallback() + + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural + + test "Should cleanup protected and verifiable manifests via maintenance expiry": + var blocks: seq[bt.Block] + for i in 0 ..< 4: + blocks.add(bt.Block.new(("block-data-" & $i & "-for-maint").toBytes).tryGet()) + + let + (baseManifest, tree) = makeManifestAndTree(blocks).tryGet() + treeCid = tree.rootCid.tryGet() + maxDataLen = 22 # length of each "block-data-X-for-maint" string + protManifest = Manifest.new( + manifest = baseManifest, + treeCid = treeCid, + datasetSize = NBytes(8 * maxDataLen), + ecK = 2, + ecM = 2, + ) + + var slotRoots: seq[Cid] + for _ in 0 ..< 4: + slotRoots.add(Cid.example) + + let verManifest = Manifest.new(protManifest, Cid.example, slotRoots).tryGet() + + # Create tree overlay with expiry in the past (clock is at 100) + var bits = BitSeq.init(blocks.len) + for i in 0 ..< blocks.len: + bits.setBit(i) + + ( + await repo.putOverlay( + treeCid = treeCid, status = Completed.some, blocks = bits, expiry = 50 + ) + ).tryGet() + + # Store data blocks + var blkProofs: seq[(bt.Block, Natural, ArchivistProof)] + for i, blk in blocks: + blkProofs.add((blk, i.Natural, tree.getProof(i).tryGet())) + (await repo.putBlocks(treeCid, blkProofs)).tryGet() + + # Store protected manifest on tree overlay + discard (await repo.storeManifest(protManifest)).tryGet() + + # Store verifiable manifest on slot overlays + let verManifestBlk = (await repo.storeVerifiableManifest(verManifest)).tryGet() + let verifiableManifestBytes = verManifestBlk.data.len.NBytes + + # Create slot overlays with expiry in the future (200 > 100) + for slotRoot in slotRoots: + (await repo.putOverlay(slotRoot, status = Completed.some, expiry = 200)).tryGet() + + check repo.quotaUsedBytes > 0.NBytes + check repo.totalBlocks > 0.Natural + + # Fire maintenance: tree overlay (expiry=50) is expired -> drops it + # Protected manifest deleted, data blocks deleted, verifiable manifest kept + maintainer.start() + await mockTimer.invokeCallback() + + # Slot overlays (expiry=200) still alive + for slotRoot in slotRoots: + check (await repo.getOverlay(slotRoot)).isOk + + # Verifiable manifest still exists (slot overlays alive) + check (await repo.getBlock(verManifestBlk.cid)).isOk + + # Only the verifiable manifest block remains + check repo.quotaUsedBytes == verifiableManifestBytes + check repo.totalBlocks == 1.Natural + + # Advance clock past slot expiry + mockClock.set(300) + await mockTimer.invokeCallback() + + # All slot overlays dropped -> verifiable manifest deleted + for slotRoot in slotRoots: + check (await repo.getOverlay(slotRoot)).isErr + + check repo.quotaUsedBytes == 0.NBytes + check repo.totalBlocks == 0.Natural diff --git a/tests/archivist/testmarketplacestorage.nim b/tests/archivist/testmarketplacestorage.nim index 6ca7961c..4e50cb1f 100644 --- a/tests/archivist/testmarketplacestorage.nim +++ b/tests/archivist/testmarketplacestorage.nim @@ -56,10 +56,14 @@ suite "Marketplace storage interface implementation": check metadata.expiry == expiry proc checkSlotExpiry(cid: Cid, slotIndex: uint64, expiry: int64) {.async.} = - # TODO: updateSlotExpiry currently updates the entire dataset expiry, - # not per-slot expiry, so we just check the manifest overlay - discard slotIndex - await checkOverlayExpiry(cid, expiry) + let + localStore = temporary.localStore + manifestBlock = !await localStore.getBlock(cid) + manifest = !Manifest.decode(manifestBlock) + slotCid = manifest.slotRoots[slotIndex] + metadata = !await localStore.getOverlay(slotCid) + + check metadata.expiry == expiry test "updates expiry of slot blocks": let cid = await storeVerifiableData() @@ -99,7 +103,7 @@ suite "Marketplace storage interface implementation": ) await checkOverlayExpiry(cid, expiry) - test "deleteSlot marks overlay as Failure": + test "deleteSlot drops overlay": let cid = await storeVerifiableData() localStore = temporary.localStore @@ -113,12 +117,11 @@ suite "Marketplace storage interface implementation": !await storage.deleteSlot(cid, 0) - # Check that slot overlay status is now Failure + # Check that slot overlay is gone let slotCid = manifest.slotRoots[0] - let metadata = !await localStore.getOverlay(slotCid) - check metadata.status == OverlayStatus.Failure + check (await localStore.getOverlay(slotCid)).isErr - test "deleteSlot marks overlay as Failure for different slot indices": + test "deleteSlot drops overlay for different slot indices": let cid = await storeVerifiableData() localStore = temporary.localStore @@ -132,7 +135,6 @@ suite "Marketplace storage interface implementation": !await storage.deleteSlot(cid, 1) - # Check that slot overlay status is now Failure + # Check that slot overlay is gone let slotCid = manifest.slotRoots[1] - let metadata = !await localStore.getOverlay(slotCid) - check metadata.status == OverlayStatus.Failure + check (await localStore.getOverlay(slotCid)).isErr diff --git a/tests/archivist/utils/testasyncbarrier.nim b/tests/archivist/utils/testasyncbarrier.nim new file mode 100644 index 00000000..86ed38a3 --- /dev/null +++ b/tests/archivist/utils/testasyncbarrier.nim @@ -0,0 +1,270 @@ +import pkg/chronos + +import archivist/utils/asyncbarrier + +import ../../asynctest + +suite "AsyncBarrier": + test "drain returns immediately when no entries": + let barrier = AsyncBarrier.new() + await barrier.drain() + check barrier.count == 0 + + test "single enter and leave": + let barrier = AsyncBarrier.new() + barrier.enter() + check barrier.count == 1 + barrier.leave() + check barrier.count == 0 + + test "multiple enters and leaves": + let barrier = AsyncBarrier.new() + barrier.enter() + barrier.enter() + barrier.enter() + check barrier.count == 3 + barrier.leave() + check barrier.count == 2 + barrier.leave() + barrier.leave() + check barrier.count == 0 + + test "drain waits for all entries to leave": + let barrier = AsyncBarrier.new() + var drained = false + + barrier.enter() + barrier.enter() + + let drainFut = barrier.drain() + proc drainCallback() {.async: (raises: []).} = + try: + await drainFut + except CancelledError: + discard + drained = true + + let drainTask = drainCallback() + + # drain should not complete yet + await sleepAsync(1.millis) + check not drained + + barrier.leave() + await sleepAsync(1.millis) + check not drained + + barrier.leave() + await sleepAsync(1.millis) + check drained + + await drainTask + + test "drain after all leaves returns immediately": + let barrier = AsyncBarrier.new() + barrier.enter() + barrier.leave() + # count is 0, drain should return immediately + await barrier.drain() + + test "multiple drains all wake on leave": + let barrier = AsyncBarrier.new() + var + drained1 = false + drained2 = false + + barrier.enter() + + proc waiter1() {.async: (raises: []).} = + try: + await barrier.drain() + except CancelledError: + discard + drained1 = true + + proc waiter2() {.async: (raises: []).} = + try: + await barrier.drain() + except CancelledError: + discard + drained2 = true + + let + t1 = waiter1() + t2 = waiter2() + + await sleepAsync(1.millis) + check not drained1 + check not drained2 + + barrier.leave() + await sleepAsync(1.millis) + check drained1 + check drained2 + + await t1 + await t2 + + test "reusable after drain": + let barrier = AsyncBarrier.new() + barrier.enter() + barrier.leave() + await barrier.drain() + + # use again + barrier.enter() + check barrier.count == 1 + barrier.leave() + check barrier.count == 0 + await barrier.drain() + + test "cancelled drain does not leak waiter": + let barrier = AsyncBarrier.new() + barrier.enter() + + let drainFut = barrier.drain() + await sleepAsync(1.millis) + check not drainFut.finished() + + await drainFut.cancelAndWait() + check drainFut.cancelled() + + # barrier state is unaffected - leave still works + barrier.leave() + check barrier.count == 0 + + test "drain works after previous drain was cancelled": + let barrier = AsyncBarrier.new() + barrier.enter() + + # first drain - cancelled + let drainFut1 = barrier.drain() + await sleepAsync(1.millis) + await drainFut1.cancelAndWait() + + # second drain - should still block and resolve + var drained = false + proc waiter() {.async: (raises: []).} = + try: + await barrier.drain() + except CancelledError: + discard + drained = true + + let t = waiter() + await sleepAsync(1.millis) + check not drained + + barrier.leave() + await sleepAsync(1.millis) + check drained + await t + + test "enter during drain does not prevent wake": + let barrier = AsyncBarrier.new() + barrier.enter() + + var drained = false + proc waiter() {.async: (raises: []).} = + try: + await barrier.drain() + except CancelledError: + discard + drained = true + + let t = waiter() + await sleepAsync(1.millis) + check not drained + + # new enter while drain is waiting + barrier.enter() + check barrier.count == 2 + + barrier.leave() + await sleepAsync(1.millis) + check not drained # still 1 remaining + + barrier.leave() + await sleepAsync(1.millis) + check drained + await t + +suite "KeyedBarrier": + test "independent keys do not interfere": + let kb = KeyedBarrier[int].new() + kb.enter(1) + kb.enter(2) + check kb.count(1) == 1 + check kb.count(2) == 1 + + kb.leave(1) + check kb.count(1) == 0 + check kb.count(2) == 1 + check 1 notin kb + check 2 in kb + + kb.leave(2) + check 2 notin kb + + test "drain per key": + let kb = KeyedBarrier[int].new() + var drained = false + + kb.enter(1) + kb.enter(2) + + proc waiter() {.async: (raises: []).} = + try: + await kb.drain(1) + except CancelledError: + discard + drained = true + + let t = waiter() + await sleepAsync(1.millis) + check not drained + + # leaving key 2 should not affect key 1 + kb.leave(2) + await sleepAsync(1.millis) + check not drained + + kb.leave(1) + await sleepAsync(1.millis) + check drained + + await t + + test "drain on absent key returns immediately": + let kb = KeyedBarrier[int].new() + await kb.drain(999) + + test "auto-cleanup on leave": + let kb = KeyedBarrier[int].new() + kb.enter(42) + check 42 in kb + kb.leave(42) + check 42 notin kb + + test "count for absent key is 0": + let kb = KeyedBarrier[int].new() + check kb.count(123) == 0 + + test "cancelled drain does not affect keyed barrier state": + let kb = KeyedBarrier[int].new() + kb.enter(1) + + let drainFut = kb.drain(1) + await sleepAsync(1.millis) + check not drainFut.finished() + + await drainFut.cancelAndWait() + check drainFut.cancelled() + + # barrier still tracks key 1 + check 1 in kb + check kb.count(1) == 1 + + # leave cleans up normally + kb.leave(1) + check 1 notin kb diff --git a/tests/circuits/fixtures/input.json b/tests/circuits/fixtures/input.json index 0de9bcfc..8d87be5b 100644 --- a/tests/circuits/fixtures/input.json +++ b/tests/circuits/fixtures/input.json @@ -1,372 +1,372 @@ { - "dataSetRoot": "16074246370508166450132968585287196391860062495017081813239200574579640171677" + "dataSetRoot": "18606430214188827267770032401597500733154450944009012030966280344676725536788" , "entropy": "1234567" , "nCellsPerSlot": 512 , "nSlotsPerDataSet": 11 , "slotIndex": 3 -, "slotRoot": "20744935707483803411869804102043283881376973626291244537230284476834672019997" +, "slotRoot": "66349521849066005241657003280153128449487022754937710836086786170858919323" , "slotProof": - [ "14279309641024220656349577745390262299143357053971618723978902485113885925133" - , "17350220251883387610715080716935498684002984280929482268590417788651882821293" - , "3614172556528990402229172918446087216573760062512459539027101853103043539066" - , "9593656216696187567506330076677122799107266567595923589824071605501987205034" + [ "16433957347359759229665199643363206150915090001687911916184823695206529408787" + , "15506676949812429996631908595864834914098798905091862073421154240216463559020" + , "6076991561843593133956623207180111814529195754530982275084891398093797455878" + , "20875155748622783137645939877909193184025786776054880891695911276432004910168" , "0" , "0" , "0" , "0" ] , "cellData": - [ [ "211066599696340205996365563960462032209214145564176017965177408819390441927" - , "256399834032317991719099034134771774537377713676282398278615627599320306708" - , "40526956212941839024868120947422067322935297516255336725720469887577875470" - , "369406626072040375689238003388146123438765868500054546379159741776926336393" - , "333671948877941061129138970028865844558745266314244224980179694307884999701" - , "12844191661993811401197260475054004253686019503294245287625435635597431176" - , "103242505924551040184986153577926463300854479121445566984435820844932904541" - , "357267598134410031301285377503939045679462829143562392803919346036584141082" - , "162961530392479745020607594774288130869393650464001426863668385541077786641" - , "426633666684068667053061341108297711407154520752589357873877687002123230254" - , "131217200633679697678620573903653316618851166977399592012996945149000115543" - , "347806146382240882424437318028620747118202052207615339887983883245341422889" - , "373560143578047669373240014805175743607410583882636003337120578008437374619" - , "188643112726610812698950916978832639394489168469865816132268079928321994342" - , "261643693073361806247543578456646407192325702146752104760117340650316255422" - , "260425332276127964154119199351807753107358504026338378706506655351595199132" - , "374895089121103633563000003194929058314955925591115574099805066048344387554" - , "251251687633538360627384151887287228926166832908633974282160065378311171354" - , "72870348025150463132527129203816383011245664502016571773976961034605631401" - , "234969517550818492643515432666311755833657223377594670026839098818269671638" - , "250704662734070531273640113211555977086430125469327371276827055724111014200" - , "85287059658255939741261887818611116570376488685014337052369839580946343903" - , "148959658976765873884541474400081762855732313547557243964921157967555048302" - , "402116967301520959272239788104745348344918207829380669065019055837727389479" - , "440503687192139964066297025050080823601005280790824714962651368433530759519" - , "149064344353438643307231355233484617851197634669308957706338957575177745645" - , "249140840255377018814348914718469942883867200392561109698520706525194687651" - , "108796851176515124780842490199733462942992752881710253277179665118758071359" - , "168245155425564161902686596247453762364112240129335852645432169468767513906" - , "129930759770229612264501691941454447321585806001002088749773920103899362070" - , "26204732465033162738545933008873662562758704699080684615280202127289894343" - , "434343986187775289860542785529657444690073100436887472033336117760907652966" - , "361202432740487795596808128962740911600093857340619816047190021218849540225" - , "100616813001075101816809823591611435583084916492802624686700765550893945525" - , "262383566766515164611427346701355047932794351290691325516723194829671679460" - , "223966317895663049002893008659178463136086169222436544014405848127792334099" - , "416071800998333357259662686053338495720384342746822618737948080251761863079" - , "402582378631671531909245563300554883898015688826468151075059467077182712018" - , "271682867846395286938993638577506552857925968097084028550962231439839229096" - , "447239701195120035004067146414333903010840427278848992921567785105217019890" - , "275718367441702584521943480326858762208121719038684001399322597215141179102" - , "86424132534636958411139699704035287483951667275825975536719519441147485752" - , "149332313586183975744469174256094358432564607635406143904268565140988616920" - , "431284330776421588418608279008210842837281123158951642164956884286883748089" - , "328694410745471749523135644660455669483988686888634622076863114197617693825" - , "112671940998917362968156144648543607958275336559773039070338509488400224090" - , "40612851250697989627190554726382690498263128439797780029697069621854862060" - , "235047914228675997216342104196597257178021277585839376175077878186492271543" - , "169718735151210135199527910197065439221144015957220768545119706561079163228" - , "345850109040121443591415752965486014695272086901102608769402892906795715635" - , "107916794601837835951508003838161872232087225679609623116098837565956752373" - , "415195406060373162425374246348197423165008252112453298469572523506488563795" - , "18574536720926634955170276058049993354780447816096707123565164996905722992" - , "77316964375201096801231570737992491072607736322255206149311341101525354519" - , "198566211140075666401818378905444403588640345933666108724809349396921957675" - , "71926050707400318807625168942501384117254391471312636171205913503928815127" - , "303403754792341398295052586417779761162194818661412867094096550880325459639" - , "444796230931706624375881141460151785952798771079111017071124833045933389733" - , "430832727519144759265424205289522948157007336118070755365887670730658782114" - , "75431213985866235726407973044434444984663930761207296437571668004273515965" - , "9242635103653159191249730220870735855877366952081272723035956668095954838" - , "93770678769846326584848478152412123903909949977598807336203128684179492141" - , "438043261966019084676377174988310087831395864304423411701911688757793135582" - , "175357918416563657734972138036003712114814934655676571874083109097142591069" - , "301619681954194702458985259161884119574424456150215738560639417824784261940" - , "376627771252167062559065889174364784087884871999807562320457079200311413098" - , "77407" + [ [ "376218405913623657793820130242823954371824928140112241102426765194751124631" + , "314738819416848774650027734981091450795528277506706691599230133190971367568" + , "185671541438843384018533667993287188552784463976148624025596188573011352773" + , "369999687040716449738176909106781564232273742620873526377044492958801953860" + , "39525314064716138528369181040681116922483814916731706917325010691894397507" + , "302541323054863798970914565712813651458073624994287394144619561394700720934" + , "383695982719976406865576319755097489425201997001423873619063489948394168392" + , "38822223309011740419945322905288718452479467725633601362723768933497869958" + , "158898897386655621268452375097787559583448946268857992017531949889053647907" + , "236092288127036418675425722424991565580750088213017828129064134225446430420" + , "335977300785478885636910729053322368421976818242485685958633465616981327907" + , "45301319286531232943641874324882965377884870550543650570091990173165281071" + , "388953644363320773331950036027658257049224416025199973369040834810517624975" + , "179090684121247908645012095022954366678379615035652152080841921575862849527" + , "72862036521792473294489702156045943231176504393200031577033159444837683459" + , "413966278516198715998742597163251029743719727185811186641172954480245680462" + , "319228121005994819660330485077455869882985383525841466318686456961809616809" + , "444353084405569826087289531412384175839343564931771937199208859800705265053" + , "312931933258657440244621703213958098047316453529511661269661115554246654689" + , "324046707545584499664432514125338347548019803929296260778608312981665136999" + , "232605030859810767881135100702156769560581877553197965969121960680190662662" + , "195552990691139483168354161940453543158121033454942109808480421471170450347" + , "74734187938204104487397624524475192174182971130952358065577650563837633794" + , "442371627545478050209250500478934820654809552790006246404403907109113597328" + , "9598617523629688519083709278332550447898156124591758226092253222648875246" + , "344690490470191053813865730174965086780036867367725707181456248748076810479" + , "196336826937765930451469652657739000056609118134108906692908651802624277652" + , "304083997048994759824930312101343485616462413969423851332740048123084409628" + , "73444200968058379165276591269761835679901128410130486285111106126824387549" + , "41164218953427953677030032164281301613203615322456757352916892168643427436" + , "207579504776374285763667641114627828145077192962839479316901739764136371801" + , "82088187692826676988890393146125034685192561209019305140865159685346757495" + , "423943456028839207066974999222160855378264425689143854762291716361814735416" + , "226708045852403913817586602528347472126068796494111382153788082362944144195" + , "437412910941197733894966244337748970146467298808458787076910652939866520833" + , "390352246839362877179655112186997677144996616242246852190787498680317992738" + , "303351814558427218717384878926541107511358972892215718155073839920488379257" + , "5890060182239622419061464491227879613346478615400415650668235487914021592" + , "418718348373985695698629783903656922567824759542609574186258577328627506403" + , "53846885583601667630411073761920505947167374252286905301414248897440093828" + , "306826929979953876944123206929962405677682208222689356177161887723012386894" + , "148411013792371891786774373191889827749059973272993167877262422473148087635" + , "127328088582820425673528430216399840889260549469319969624868372176926172082" + , "178617939831649929868233133581673934041944609862244411879299222646498583224" + , "266420556349272458856914128132253675462104085415200489974885819785538864923" + , "64753261069521503636939976955809525168465469654303439552001983080616036262" + , "235714577011783282545552023283704157798611276690635835584378225672223007878" + , "421686955680456364126664197384631281591320367522356207256882347485996416515" + , "372767141375487057724571513287482696800102825193528467897205356361158897040" + , "164607501471666790665346020739719094121098265099081469384870172942991747472" + , "173836918820321722867460862782260614388530074559151296552824095205156002658" + , "407246083417425524662312015975827541516082998257748646429630271083101838562" + , "389991969826953086515724742006537835631690026116769444769615911993593462288" + , "445863854711125825107243929835441854187433827073356992737205601122803531410" + , "184547806882331266358125999275667653367692066286808325195573251018217230271" + , "96755304403635090044910011270727026822442147395033306216793040126821093599" + , "348316549898237867477685650420042332333416299717697602846943492962675125583" + , "368929065251254782291227675431844630227494180444988430400933231848805064085" + , "25707069381697103565838390881630223884029886467462653460442087366839706066" + , "13453448982379734755088710230967350622616655629278676855626252106586246786" + , "128145309923602406282902258992095172433009520143900796278410154870899437372" + , "367448239635821145528752813902172652060587933123999861249537692272926671281" + , "13741856404300411205621631477726970794354170418365095541419223480364118497" + , "254556287038616003707407504322985650518049665775309299870685554312496343428" + , "115485459456054538114070418354956387302541146271096599399874210039340195960" + , "442800428570899382838178620688682934670173853840051598091069726281758244009" + , "82358" ] - , [ "424838222746158988229624788694939151178615656210585621868910231014323837551" - , "113188486246955346418956949679485575685258850346101035778277727456423482970" - , "275449978206132565019222655023969430014622832597654643418394602485645803413" - , "407856757018138010439232009766252068440920591566039673660874626878413077905" - , "433369046493777496016384947949950877133856528218602671493669395706908819748" - , "258364166531180422149545015891786023981872586904946376136311798402581278793" - , "111997719221028147522803956659709775148434180015507797582340706052412284571" - , "370086597976426508280413876667101417393598181708677558733730556109327409076" - , "394139601979449259075910238117153992797849553309541269624848742084205563806" - , "224088276319080487199395482893988152025671468715318212801266537036640477323" - , "412710245119809501914481942088314642684754542082140451180970198371889738885" - , "353872602359946553306242341162348980635834907495492814598834657175405697176" - , "252575199506028475372678621140654219936768774012823764047177692104580452933" - , "259093153824033122452869556249315839899366493071746219770487886456301642099" - , "433829976798312333371154167497560676982294392632725612538640639617101218872" - , "69918581382122563555200898078544150952625715196904114153232367538572342772" - , "337937520623192257595352158476909569245988839238498098464935654555688460123" - , "264739973232292969253318643276671532055689422253184191167449284055983944338" - , "326116252818829775096345069850111970510714050346103409479803743342138260656" - , "236666502377413649728378889488706275212721356921124776708954261777813709815" - , "211625935799984260567718228446525455893664313064841539301444509150157287163" - , "60213206239417039999880027112341976360540689886703427811513517396638607512" - , "68310118105957780876770075529546844404225720757669797609686816545988561625" - , "423863085551351065136684030270731679105571943009795949621903966660399419936" - , "388914614294393005039878123500859325222684672184567792659076815268598434245" - , "449456790291560508709069826219925144971979653209755565240911568965768874382" - , "448810363770763694447869940916735951256986784286793489549428379909616059117" - , "93646909783664049092056237949587618925209622020026157405117796611689551192" - , "352210795298632954574896499649181574584074853828419384742874364724522457331" - , "37455517056393404525863484733101879886413925183061645520768912552476716150" - , "386617357584684336812125385078476270301738184058813703112840991226785114117" - , "309940292044597334261558429176136686101590729982259514472573656131579113438" - , "375815246167575100319857872432879650174355611853064771241069582477717074415" - , "332214507344122806007757734266883566559371568252728459951766124888176633706" - , "148990259460952914990881100852534318351247069504848477833147446514732789712" - , "328669527889838880414072022433859139004058211332184916573516704632073044118" - , "39278026039348543645873027549112998051601664395028652771103624299930924528" - , "147717660530589785119644237145092759103012624422229579698752386490700965238" - , "374018518701362017594752095877197725242352803195413267746619111489936980685" - , "19185486483883210730969367354195688373879769005461272861759636600984416877" - , "61866046506558157021682973167090213780467780519546382332208868591026703563" - , "186854966504766517012887726614015646154225796572138017810371160981778288347" - , "87813550250328892091332566928942844770632705056120813488729800874811845697" - , "207775163424060266085108794048371834145545842567796157378282772999548202308" - , "369987573847786237689249538753881486995686208870889713177975415012214427429" - , "240880979016395044518849230927615466120209140082149273390921042537474853143" - , "174902051454932913375934735427101804474275543418199101786687925733405159872" - , "342217255652950822372803598682842961053537267723988087801275319754065261308" - , "403207518067666945448161377960706451817747922771285796668778802535227939962" - , "407191459999036791052261270163259267557900498930952521056725210031161568230" - , "338216472523551728793268225845396218561132966393725938551091882807069657206" - , "118364222678576550870375112494142500603091119946985793934499723872824782886" - , "269721611028665323587192624288165848310917029418261851436925829954710472436" - , "227424498125745236125352117206136621428869662458452610254773560636280935711" - , "334380807433339401906359555583987757411855090694162252260781648609761248049" - , "42470806516174819075107446234449247453971524726021445768611797530804156161" - , "418994916402918322951830716592888390611524984156817012683478842068581638820" - , "363263142412048420546716019774090729399081311227606555141174736853886128407" - , "192292715923468025058557166341646729623133127372303791236447550026886802680" - , "450253878713722337767292128303865371116770532625906889925779639839924402495" - , "412596147086332805611392200560087191411541689130482740065137300252639590489" - , "264059866105067484811456793906835462997849523506366903974724979936196358724" - , "80922384260325792825608274004101667366364502159441286540209512108302572137" - , "69261112192907071823876894642934902051254647002357333297635511793652550535" - , "342810644068354896385837929029331085645928473943142618800192452300937098227" - , "228361826362950356801549793202622774869100858404479869989505302905936946659" - , "89244" + , [ "241157004473245159442240986568309504889899196746578229713232404142091005803" + , "395496832668417307321558036810816797232916421595889698529651962243006427583" + , "431474483222097157434218543836601416123347992295876186514466676899108498487" + , "68060420282303575285665508223718231670566946942933728902759290317334262784" + , "49140562040558135671768899715194701078998944984504923982642853147468950933" + , "313311669660968625380031865652268220934436182386569851157916150423003414976" + , "97836117546139165976439479316796213907792791100438885574353018085787876401" + , "6763283481287492759087548301142293942841936279008117753857130283663279355" + , "209466571068922686862127548047442994386460817886980583830675943023192462653" + , "128541653737197772754606236678969084875131907156835894387654083633436371097" + , "438742879837875619245357588369397870767624549330742306219227503709359909190" + , "445752548697798883058138683941115095721931639931557394130054472171430632433" + , "196165088671406338507496216773832460433532144642930496677434018422900083001" + , "365565569782075543059349613354795850837486329414631742876634959381325750561" + , "108030144575611305356563980566885565016833887733157477721519203320326913335" + , "325984584618670353197402243791764418752871061858617382824986128024707011671" + , "373663543965683693997837823716846029146550751614255124872532581806953357537" + , "337924311509894911135363817375961094769634561834790284518388375862832750644" + , "153794196899089562305152557333738114339437813967698767755983865262702582080" + , "396422603679625909068229877466828437990014089122922360338289795229315956598" + , "276371713466530534578708208879854789159884681335045931827442201136971048111" + , "273953200418028426362647096647063770435030054113505653273826073874899605378" + , "32129062695826194677861587740494315707850212092978207754488281149163079547" + , "221772069175029339993657608747443422610389773750405286185278882298315941214" + , "428987911587389701291997565583957295368257801272647858311093807998411459110" + , "185954083172619583546321684680023284791584246379294657572330983381991135206" + , "295202559818638568917627612027871806387185749343371928833959828231486110473" + , "451766377464294685968084138118417046582587800159608168031907636006096194986" + , "371845278192644706813833046858812091432968198474979186125583121196799912132" + , "142136499715023534153659910986076994637370923737528480561461007018770467586" + , "423399807082760818110972976216216231799478053436783923290450403027665219521" + , "152566745871866741046078904744256904530587476429395937801392976841472096607" + , "340082444605267796740815295444946656667118303953408298764516563848764821983" + , "334372078292431883644416661117566743285855464994942559616043857200297049035" + , "247547121642689723569808526867682208243686750476495306806673315204285504942" + , "414192602713562551320308228372930626530621939166735459935463167848842598697" + , "440600606781608255774101582515426106540088699659579515489927530728495068713" + , "317983586124618484222181726202299510634005007471600178410281228490325259372" + , "229826116943113986547266277805028721917460662878645739734052304806217409242" + , "281347780526304424013890218982565792759375863137073417413006418897116283603" + , "223520056202742097822799582144490429305703089329295156475902801462735333192" + , "229993867036269077021866385189665635306529208645856676533303803824749205547" + , "397865468019191392440745086647241439747591409998839042195802142158722359843" + , "72634714037901782606512022021679914390059294261214278987247077978922653081" + , "276729540649529904843471325162635921683675492811426967532013288350156396692" + , "409039318815892467533279488437251283842216237479760768281499199102751602623" + , "286671245379506053557005054227592360119303746750036690846434468371890463793" + , "295635361196025532440989175760105116760951529109191795332339258401757974835" + , "434485010833031132168803975275705279966147777147537726953670272811033225014" + , "278826601404524072115159011832221103193530637135870251091991860041020675051" + , "228445626912438627369100173895273962305333767890181454460297272514365943870" + , "376398354024967101387383809464371017887399522199590057008679874021883629505" + , "329172413827522578370228688365908150640432538664240456767087544486995623473" + , "293767575752207218149983001236482049608744442001766207561618751059877897026" + , "187326088324785291544730822985079138966918834921360710078233150088557391781" + , "156618964399629895136588841519909824763874177193381390652393293351541276293" + , "373488924194844942764692126419305117410576688798642663864073296622106057447" + , "17882957609964253412528740445381313431007223071643473244775426702361064934" + , "425356200095934964810404931164153113037332523638163337509775023252708342161" + , "347502894403018703717248939226954531868034876491143852311172288429305480236" + , "196702992423962932339402211963496070273511333452256327811845434043376875620" + , "170335815821174792971175245902301645223063201246053812528759477021734822256" + , "210156234155637518517497426826794791494937529761475911024639269143467781294" + , "347016557041841547929015373269797984274841260334636581861005936588702029245" + , "218851635403062261113868967097771081381387623429008999852035401686668648364" + , "333761020235758462244461898596276926173744441649790627473497903147964318973" + , "100136" ] - , [ "359926778925154809567585559910064420821311221299937568759183366972901588855" - , "128688825421597555299113005875649976344934035310192572516197551102045945994" - , "225379354482053502673695304662016054746658452775856157938886424152134693969" - , "321872319934407904743844034025550739031781361848065513098922085967524180784" - , "250375637325405951645070615947051799520095842815922754899017741501395744611" - , "97676493052788080672307677749501730337551225267342432472194527468634722352" - , "140101187036396881926000630022834247990512766860086619783437252676747730662" - , "428833039353549335335605804240430918631639449874968791377641834408506136850" - , "418359203734539413977740838354554804415161215624809316660001820037711273005" - , "197411877418795659129213175603102238904459737200167987167255995825203749339" - , "221646252100316559257470803343058462853641953756978011126414924636869625612" - , "106393540293584181037890192557883541231531964825708650997196071036779482686" - , "121473330828208543539643554911190528237124877123871673169194452404939833883" - , "234055622144947293638512253368547046093971383516706577723613696162225606040" - , "68307451767502390304445005915787226559811433450721625085741437522802389574" - , "446891883436763112014492564462451523127134145501201571918449345324780633462" - , "83718652783543018019599555197511164121642363321504039439786358267060414949" - , "90267297500929836073049162292104311427365986272517761342871530827272320168" - , "398425606698859520268856768787424690872952910789754531465894080258173664751" - , "323570139379118444589557840594603212198136718240764273769210492735883659788" - , "318597103584099056378057647488068323974418467250708490151864712850204121402" - , "6299083430773359277240726214182464517380839990956316267425262319606033077" - , "27638206326925436960316131682014727983280820447721477666884742925275976240" - , "434344186848869917381375812528446841024676181532946456237439060027443649574" - , "64735754118644738348599761514174981324344130078598038275246522384474432918" - , "53068717269762105498508401788249005564862415051761175636612434108259085043" - , "35813044996911619267309099508360887777226716179396659295580849861836012116" - , "67751791392924142809580984450371772015056060429352159361446914484238646676" - , "68534949135677447506316576616742207938855454921330757052467057435206318183" - , "98510151949547604999069864337574320742530406479752176012935179772005228326" - , "342190252152505345443004241184891966319091967630257822491352072978326623645" - , "362701658859425316334005554473186516818256386066010799465369887406035738447" - , "266999116654850467726292928465517542818678046748008340458185725047959981772" - , "227089355966197874086821090531951502393729872265201602128464978982907992285" - , "240800343500959216904535208047288234867926058830277460630902462914796702354" - , "447956858573680756485556898469710354624642653441041335815079102198306530583" - , "89422712944117481549242421245588048728782658978853365197341587057196539094" - , "72610343179362050463955730204044877712105879926618304878262944723464870506" - , "8676698500519447254981838968537883138182811064381378248657969913325524054" - , "180453700216061196739413267121764366438386031039311941313645439527087166894" - , "63346784016053253727621352882430122335280702556586808389293772066791583857" - , "400031453850139978805133735852120986024101930860924735862305444817383365395" - , "230104622290558570218036071349472289358926019290368625724986905348610140188" - , "175689489221336091369196327293045073133701056385414159213338224521167050830" - , "73310331103509697419315970265031228794034932318600293733074730812482185479" - , "371383255255842707875498538452907102684511927320158672347778293876877893808" - , "165319345890230193939972313953881372394171342391835626454759321320435952720" - , "184753541001210613115361457830691571384268642766106935428207829332011259768" - , "378810733004878432563271790625801205570962762395854282745166380274493181314" - , "86321674336629444862383262780020871828941143514651008200999725989892879714" - , "332634533993388248915777870988529817692938793120418377552247997050250349749" - , "41742010257820712511267405684781534740157292266212120171929853763272599516" - , "224101330592139734390658213442359402546038743346294438455635537496290117560" - , "204363902046137087420878796391135987770418514008394389852388361468850216359" - , "296526036888463156867153847102999031430641220536577556044554029990778763710" - , "137568796227115931047082828377698464185467276723279763304686078869351280509" - , "147456720891843338735232645318045656082153590170441596326500204393398792771" - , "297291342309626392877004635010131510068476699687818509485687346071074942006" - , "20748013593486831233186810444485136836664689496258849465507672301203832324" - , "335431726883875036252568773858744219410188302724423071283643221351691013313" - , "50487384098835523033417523223562560857744547945136829388944024807752630716" - , "425952679139710019732649156938676226492714486376811103362264146658191708598" - , "439787938069461539508728805881194071103269524405653997968488488049426387373" - , "279863410796988495259178322026394289028023166886112128504434877538089779477" - , "398941099058270093463626870965433502581301413791423667994257456160377865247" - , "5759692644185723491187478313102540786562625675495805072053262277490225012" - , "115176" + , [ "161727271607716375046017463277032675467852897580227379879234974330735069267" + , "436931903065660785112720893397590174526426983971997354617546638690430763685" + , "392017504230652147245786893747478853495041284244108295094803693038790431637" + , "415321038686697045266739158764608513103594026483882254147103775759997368128" + , "380238313199948054415604748835873743542417360254338173892553353141722288662" + , "399194873686902574069497636877717414238850545727200320274631624384766144707" + , "75931398329133552364165996690114526047263512919434963447008204066080193696" + , "84034105894730397196949280800806664396160394242100398500056138122570541934" + , "76093601501168794869837251823657873743394164861615599801759665833578130798" + , "111628258734988099770340891125519635081706042412329779319122531786451529791" + , "398936343621889345019918589906109538690026662653404775338965468080137186553" + , "125895585167072763506826887402899793618280102534546995588760403601064729008" + , "308103204576302958286001916683972919601967396354654394657588167133357108578" + , "414363523182844374633585173893399460886819264866082707294154869592435628083" + , "66738614147233486589965333904456072129420226802837690945947649695037784241" + , "132043354219419456405550745461957203392937058859243300774001449040519221642" + , "352010914957231155484989004961447787793751594703039747398495399074362950844" + , "89213175644882022915065000776926173879698253894470040052740180616617658777" + , "442556164952391789586961322108847239499170450774494639394706416919685040308" + , "253630744386379155952667345635159581190252638777326230893766941613648055511" + , "349639131380937510732440038289747053773408238867129480929933870880139549035" + , "186601616214284594262403993039285278592418759280757069090994344114646838672" + , "379011397512829403770859532704756792597380611741974182724337785935928632659" + , "79253113700876599174483322184783953189445713242113047777793674435094456561" + , "112126437852389946296315241522494150897747789505415333597795126940254504197" + , "343350033914884642525833060271864529324405675746949908129989474783002862913" + , "227122001013510457498519757868412670488288916214310283000347774847877699282" + , "107662834875709849692490195837475192496354965380431894287990379808197539004" + , "361500604175432943563807341941794744513465901633153734052213505176891784839" + , "223504524304673490990565315404082117528785280419947249898939354210228863897" + , "78660298679765741320861595191013250196743601341664901063056496976795999633" + , "83845284201646013606258397169248603995203335720935715164813389203469747739" + , "169647134590063816111021725745692692999319765561132930984168611455323663570" + , "280014137519601448621612680623704880773177606127090897284679880080570429048" + , "319493693956230249849576001989063681520239743257942569712009999878927472247" + , "51063862200759395242079434817221816971808037184530015501896750275040348628" + , "256095392301821538809333911849885998059887152822568291624336906974340965829" + , "152680867571629986156033493794925552676738699952567584566451278743338640160" + , "403276966179190150576959787330136261689840048114843716430379506435284437126" + , "440340560777726151537398009647923260589429451070397735079562957938722634774" + , "286333167805334041231501541215945435515909864809371658903764343617810781412" + , "238750717505241503542533402016891698595653979079624153040125113866821659797" + , "361643353852142837154987839840950457243073471725403384825504260951177163248" + , "10042456531161572181777013273795943000465958326694229575635016248781631802" + , "358732643100388363227003854386349701725501409872975836421974146987153318101" + , "348493677025466723358656047712200033589989171342759239882067787623540427927" + , "310591777313454971084855274236394192964901191251718701502863815158179031915" + , "4256487263351132251968694121850695116560466267755154654271593908022095551" + , "309690304861683864656162011790064185549622842377996996033063031858428359031" + , "63025847004125694517803691727158373502855189763477703687910149148372205318" + , "108452901436568096217805791902920155511971001363614716336880551579187832248" + , "321891788408649952782235744230469592620475729721216567345672396478919587896" + , "122909225488202402265550310616857955337199606928395713310209018070164326326" + , "408977419166045914073175537818670597523446221113484079257077411494017265291" + , "291565319704338938349322663723631194714440216295326440033577315966042492337" + , "345210733699878107794705094022003175300093518310099323936099507406231186982" + , "411757043749848850796256570107758934211209666977163065846168197943570909456" + , "326037364885896659435791892435550811129438898550985580691572336762305149428" + , "295265165508357362386617098518809569611618704307653551830599785099688313743" + , "182324173521530161555492584546213086541658149131021269925444775127622722530" + , "43089140008175928143043213693074497983003105481104510412542483480105059764" + , "417022216890422232642030568770972794291103494939643089133426735160013203958" + , "64414121262473701832685160135731141022181443819991056119195988275477163878" + , "105685450123892482862046134318105289470249619553362758718569918204172978291" + , "283941709342169467717442892825992169078808383124852681233949136013033263138" + , "427900721646173764826610734736317066575712053087387058334537803840626812318" + , "103690" ] - , [ "199440901482393381753657315848210955092644686957900737971520778206058989647" - , "339215657660349719251187938243949119362753238126598750470832739830379601048" - , "17957417011314891245400567671664723859427624136284133790923936126779445290" - , "294761585889095249928492042608516765584723814657871392207964321318076158536" - , "367304199921887970655898837985855454346911090426896946930048519042744277770" - , "173405546837606747721292792526074597519538685230485266741646923399938591491" - , "13202798104529529703580600642858924379886936325402696094624200032343206719" - , "28211272278315691894282764239983301742024079691520980592618486887749800025" - , "73792448247120972778500624350664849847034095641998271809779791788652649022" - , "386961947078838359430674078072441680475090687247027225632133013772954043342" - , "247859266401821616700765969075270662915024391205665146199401830650793676517" - , "243938047874995926342875119559105623088213951205962677439167259642163766960" - , "14909501249861872673329370269106359532506159818320693170564856401208688898" - , "200331653478898243177761429526240803993101536716611440775588088625522029071" - , "127891684617049394579738365914860024509007913559921966744972525605976847919" - , "202912167983786187592861727924433749852786012202809544200943965898118027816" - , "176370650316309755425558132466370508977563252342126855874617990006444464573" - , "179490319446297562731655155074196383539396893457024237113284509223965454107" - , "118703379899134287650980989454755985628620830085932176414465582081598659194" - , "102025594191113707886629488454876652037341696284939671367279050169222988689" - , "421132375430104331136732058548913808473025321492255503838896123601628815453" - , "328334791815856213206267892694535121226673194052951034497930366807851111845" - , "83012322813281668737061895967682970093636853452224812061110092135287899376" - , "329204708391107220275172926348002826875172581958734129645882445887919889321" - , "410748869385474485539728045765785256294532845070137964737879849265390625591" - , "197274807717335387012872999914232051341799797613667869923402386359379902675" - , "235713095185988155553500217595661312303861624791720350423698435045853678746" - , "150631584359141913552843813384841153102535679219851913018874172414598353488" - , "207783836843813911284913666774420250626019971129676431904589416081127483900" - , "15728034718954665549174230921445077500399069880555428849958014406422697976" - , "69799423545177501667748653663121504155622623013014583690766769624482972893" - , "265665371394145560256710553779588458846778960884758560117805511822299802326" - , "149195925869584039415331014261414953602245337159354350672030141190471260449" - , "162328395279114064180857718453973759378615891406692054752029241989300597156" - , "104643123291849369328362168243087456326274773811561124607060302871149280568" - , "320704123383295724141970902124509237736831907552238934395000039155541269937" - , "77914486216152383968400591704791756847610018757857965758408455442143531631" - , "238365259321088298905088250444146071545398991768186471689605160523129613763" - , "279409375422154593510552116666741774885392805494152031820287626934209978908" - , "195118776021452609708543280509123101181249086555819844047203390788132717252" - , "197977884437087886153482042453896079316138251415359773453987232734849953584" - , "168185043240980087638006965666857387510828226074470344841850764460631595331" - , "231157923359356077977363679818678437836536420121744865399935742538602805912" - , "177903771863742191900138437188329108771172098110036075491750018158192424072" - , "313552174443290416730632310997197097951229162137491595709733420111980331403" - , "273253450712049988786741336540196077743997302924525995219038781650977490211" - , "421908030281055821389377531613150504859996607596444776050212044919345332385" - , "180108184992593746898140529947178182204857361841304042854173884504394805936" - , "37075272799330399065679301151342697855905822543084867570322173216259074746" - , "364885615491975468180698816037289079302391684470748470356247509218051645743" - , "397482868106190800111749908137311511782294652288655656060379563261618687603" - , "192853269627895017416381451198403197013798085262867793715522216458130791820" - , "450480853450142791928572953497280890976328598410525218090104787739409705079" - , "40278654070502961330170439514434669768416784968274956021022493731441898222" - , "251277143131769020481025315216040765839561111684608785317366609258959942695" - , "95094468748825454459610961968601800404132682484160170977941285794444806916" - , "160586633865113902389134480029183924655750088163491531655080014223476604929" - , "211661229493873434581423107377168478957907088187044576032505407590783850232" - , "409651293631434750272174456674594508340952460788494035327119354167465019826" - , "233213211946836553080627522409887799285199986120567245752841080275284294566" - , "143182900674482411759481361336063079267405785923487419697568907351386146653" - , "430050085956999990041799366370428972519385994997821389120583306252090911051" - , "241257468571530133762629460194062384921386438426367660851087853915892684115" - , "106478922860328643074356032194102718325829441005019365153538120054339275205" - , "252933430690486626644000908036895289326526510137514385824014300035301154822" - , "242924628511152862437189415942615812459145003511499320946981326550434266392" - , "107566" + , [ "138256030746461837549397442572994189580701527465369655070213851600518007627" + , "271525602741486033589479644516307374701549260386485121692893555919367723980" + , "337381608165184802786778447932391434412369097299425248801659461600605386178" + , "49701498498310365760361693797812106284070171269016285615924470361252789722" + , "449044135168094544996485091374715792077712970882158828626095572785501177799" + , "439584323181747636274772998229721178940248295089274302614964119595698059987" + , "221129777181643839380392539977605070347501535692415570528742968958469061179" + , "260442488294904536089837556691608496786305506819751327920381397248348032880" + , "156872966588331665382333220548870104268350055018244381452812586212096247279" + , "330028388563134371759747120471868542713404265257557392908609297542915913518" + , "68208896519002443949450373950076461255819926918638085102326435636889002834" + , "252678425784763433055386398667131046783474588561711926671590232565843793296" + , "39797804097181003425685018967587795392789370925981834141937925108570823981" + , "4215309203899245747958768314826702696190731060816998362160400865038349726" + , "136734498997856250258592911803844913371353478919762250837691929695962331708" + , "381677937122396492508579460882855254553577533922865329271401111567053190724" + , "10749675230812229641500092896091014988197056524674840729778549143821979870" + , "135097602282960243337268133382667449145360409230157925352490175714108964095" + , "242873367066105722283825785837901387472900816289547707460309002095940565455" + , "155493268185817930348407141911603982844049521969744923848499407672675777322" + , "415642736863457002375985001594317953276847814461557020757726922490860770148" + , "354136078080086998873962025166643470299269307135348828143330359578329055906" + , "313085855631737401544885149940121258908571920511605787444684542780555855828" + , "8115493393434214117144479399406172998741310440534169294649957094166247923" + , "214543687395433360931792196118626319129162380178692065955585022953154872322" + , "154234128027690913652034697584992881591970243727304110214451008438782477119" + , "14175086290539714529003422615330430531399458272913956844030742715035649280" + , "390709953379081834241435874140250491904820751257013146677897358261818234254" + , "288023834438366455410901385246100453703939618754780502576799536549606208901" + , "39490628023853381122793748880158268059914774659180441073853799477288464720" + , "227342556302937824425029770208610562855918902421760388203050454238568186506" + , "98979226235843591028961570492800217894324047740545872074088316570556888422" + , "176742487279811984830944025154970718757509498643040479007459251457889660974" + , "50047967273252005764639809802617264808102925575335051801884435605468241529" + , "53373921072508713639464897505572245630425484474756416124991863327158673872" + , "103739233791344122266657656377185420712811405445627789230209984340686445829" + , "281870601890020634364055901528050471593459524377279116377201506588288803269" + , "58108578340107960062281622911676585229259002406011458279022033702861683610" + , "153983354618375192568870042530444777536811403489160893433648773055770114459" + , "24038671403248544012297762141095195802669503931504574014429723263878685932" + , "405442044113846320137303776775269426818509780228171595098458284560108809606" + , "279810453411370189602490747308498897392280096477506910130060717199694401246" + , "29620612189783288852331799583581745688796881254192994737159756783938073706" + , "65748776201489341890294761190503004347552972724244124692900449999897997160" + , "84185108927689973339704264748284330345612735279646212757500398200022975393" + , "103680201868116278103585794284234578826520661209314250693300843778354285440" + , "343043430956651179189699467241422954010072228603896277214362733661343082559" + , "300238931628488489385527468761524952820281056919944919495169197311896430182" + , "171825671746780811632820766757844113425087052255261329127541580497346862261" + , "163175795318451916259078815336269324229443680559494691113526999293223844460" + , "288542998077396522505681778139535620236729338612758258043078168171660143672" + , "364125356903005469817382175321894759029889067015408994748822241571441841217" + , "282776836397393777404449767558387174770396078960781774307378089388598105741" + , "219693202287431144006017460242576031372983911761268260483773832278810492239" + , "56259268459827596087901160492357777067986603710333992343924590380487212908" + , "128698229613043526796339362860811759209186471240641977478679949844241161203" + , "290027454532796061600814880155997728395324368788373242317763912665945126949" + , "304384709192191028670602475678897019756633708486706478928956497012091541385" + , "110984661358366566911486526696815003908381000903109697493040276162906245006" + , "248759221245457551072646319932198093621416945225338678371611601838545239699" + , "241874465843306642335156487838842715677606723165906222534038929082869568580" + , "5374595759524642476835141626245636716458666848188818608690122315143873987" + , "415626424605424214024393762971652112770712059960375502675676835407403849381" + , "49086653475957061178690916998263109785161058618763409594238570465187458021" + , "331304514516714303954814609929761562996891476991930828192399479690867705375" + , "375287418064861531006402662263058414400193399475253435941742944272106366126" + , "69542" ] - , [ "10892488375325920610152294701785926476935321890453222549428070867493882259" - , "230776541958253414701326765204413805639078570664616139597900921490475143840" - , "162235550819840758141721889536295480113278275911087429090017541814695333320" - , "318634611531007856220026646570492630940047240387334221027051366284225674524" - , "347695480420330337439096561080975864031317110001559084872978387417651465445" - , "243301070227446762125488369714708670219289121859111929877695012393609726208" - , "312153141205681954392441579373652470334482191990502803524039048245142338874" - , "243769659456658813016931656268556893289414122189859136776671772112510762644" - , "235510946617019540983239794402008700611014664578713646813426116406215382253" - , "394638234040056271265534896894991100558052611842099314808878257003754175212" - , "112730195097163222179949980634599934392634120069300673310070655800491242211" - , "112545144551723145227061757291353149296490850338535641681237178578430772288" - , "399161925498018051746424503488168548076537557369822821644005390567188305750" - , "291823556779130095044851238536860577785281148245623338113991004798525195947" - , "443006765181360772964201996695219533539991249361936300010158089981376772939" - , "74018417655448362012716269253153545524112649147848318337218250865231619883" - , "361038295627629757439073080656763633087074408866229383288288831546300069767" - , "269655542872834422597617091710145830035767834433114567250727497135451412216" - , "58289072717559976781527037332370898163550414718655391280446986067842922181" - , "365399954331278626941053447122605263207706937018525782908294459265663426953" - , "83576501872896181822963149907518188809522341045703363171261203985876068484" - , "203403783686919490357886887779316222746544665267595235714814853282937072937" - , "226090172488558641139782927103632452136731207206963592401497570070700221117" - , "249813560776802008219945831355163226098682741553305882699128946352901227282" - , "236586835155013316983057471119105606475813711035522306026673814184519826069" - , "420611449257527132395951061920868895981487081726689195426673433565077012458" - , "414979562418422585161189066520079202660418565693776869373399931253524536378" - , "115851377630895049619958829726730418778383692593973233290077769966938018584" - , "248071158447148977966329235335508229928647083339924308951291087789494073866" - , "8254100651607835318906499132096926759832050649688561036854000785129084907" - , "91385483239630999205401307763890943367451672856478206313711901403403429289" - , "369346641925770698935046191632374874762045839747680407985443471202350286304" - , "236809023553698844817613980395588655518336237009075203833474757804664254158" - , "8367847400805682648908349286699722579431227561083498361702537964306290078" - , "599241730770400067632779114435635342549228985534229813617556932580328166" - , "347112528350917448294202348829052279076907614831011498643025223836596915573" - , "384244379244118003891043669466323943736726794634167201471569326059716944701" - , "118013777197672343498581960057939216208494837962825017767101107204031333144" - , "27234916267695376599463409893017377196853108034589808756909998459364893467" - , "443519198016088819704735929590164254445884637317806160485888215392818578737" - , "396780482611567392375183169345153676737175342167284140440545202776279411157" - , "420351155303051883480975795437743307852799862858964108014000673383502660760" - , "17379377743250873773932440622865094720355292220433235366224143179854831702" - , "299671454782683147939928632170233327590769402224392134648893444626929909373" - , "143062753141414050359792615867774312517100868919516205179025540179759009492" - , "79497692490953838158801094558761984613913564034406069659969793097043605498" - , "422748645389700647011491406944374966856916994331478229959954030359911549565" - , "101802829812014644970197499895811874607753186302439171072935333706660468030" - , "376428369998893026519415315112012919906032811618495880392785036762185101192" - , "193969030999254195249242252871597931610859408264053152789041067245597391073" - , "262277607928686742238285487190873200602833495734085188071246746209841324139" - , "154099884960502807271641574310268486840763221700920893692135020347157046386" - , "155875061164585018658671842995328931296342883770473498362059838106632382461" - , "248574435283666782825705601259695525637993294311364397935499480206725256362" - , "171325185063038052248966557755722232979612702743265316145563443527135798688" - , "19982746887818897250405980185937061235439217109294376948752373205830077881" - , "363719103724181291346130833008745948141602173373912337315631878022251200824" - , "174596812883797666354579747966720458118607233660798323559531788300018084931" - , "296611197821867585469311917529661698312828606304722408477045992233526328708" - , "115884038550627840768260751168697204665962542002024023842649383174518895165" - , "265597417366164841889730505737916646261040505851159477903649773521314216810" - , "59890857222664166616144499778264545115438678877460924559608721809777680238" - , "150275344313515259978222149421664752546204516114655782705875535407472455999" - , "119762211657733951640135703777940013374803447228667976561992857374027112851" - , "124750313254270944205036764607428674226393862430770588148329786501973600535" - , "223562415856611692667255745278751292230311455355383541649772348032933666931" - , "70851" + , [ "39123829190561861183660183733242133635867533339232738642474830420418257643" + , "37174649578800373610758294659508583535895891558587969014960896362183756233" + , "207569229969176481806026789885614791015947659569784248880676689456568087836" + , "82296684032922076184213320323747973128485733715393876158439279164704509249" + , "111527229046198959047428897909498127413955120461183078636660973555019518861" + , "72587001524964617951038105286581716326808941707785692339994847279369159833" + , "51771406033923099415957249325619479336926872646498026310010726929369461516" + , "265152671434925672945281852111954420407669860105131118320776045599747787440" + , "223017517837954339291931784632296140418370601423757219605966219844712623973" + , "261076479463638074196913354930883177646744205541793747643810002592857756090" + , "387783828269387228322511862093768924472403732804982266841054790407001000318" + , "438225298099950927941157428234752427311021416061602373142009955012434923657" + , "335701546147591724220028525103649653189247443906344005032730980828163577733" + , "90737094620815535844208995276735271315900017571067663789392958375760479008" + , "235873934524644390068542721306250698619830151514965437021645273061867005597" + , "93158217326638440350638622914918872851420263349505320206109636205933107196" + , "184042372480938633778630779166449957677733240160520647425049917857848948406" + , "329400252537956147981383308733351175236411812207024433781214690806268915420" + , "133995911193831935073754989098352939618424202004123755999310711336462440357" + , "265529054864563995028327705804971615140952329208274298734825571374206480188" + , "173897528042241240921952766875335529243047731270673731855006043401333151866" + , "260739915069600709195083941605698846736765477435584229962670335630601970466" + , "171401921503719773665995435091558514673192436995310255941920658936656627636" + , "346392829167608742430282768295235936626033972993698680203358435104055625811" + , "391625910691205162746871333259982974944877087796826413812971317913888646802" + , "384009763135407956440905853269862873389655900532516397777785240721856701713" + , "241990531798436656019524913906646529511488540778528710343209568641928615299" + , "157051334654331651065968929676171887651436526828643220196217574871350785622" + , "352270189669841039247877528832859649317841598461051979759652668011465891568" + , "417977901403489472673737591980697384099152932862321079086926117224934044211" + , "90207023943079751629841795878760420399905667300784801859323514553460055464" + , "83377612623275494305255999438381786862683654879314849483676580707224946556" + , "232077005508620888229258478720522423299074457606730128171934822254385929931" + , "108317574442524328890393042324228515197405603984662510752180069885294853034" + , "334236245681820858576666763120595613245945863233792040782276595831593264267" + , "115227487082090585619981606401630756964770986753939331934724186093240221046" + , "135704135061020416565239659323640515526116680837153449327310403770288996220" + , "422533585457809031752851035394584534875974835999419610902536947553506675466" + , "1970994952906215431786703804252581189883620215773190359856747654421781609" + , "286882824063149641477818923363282944295938454868971480992212004377344508813" + , "366800760771443728276401121803812793410850665623672680090058016253013652836" + , "350262421186205725797117658707471898404527714528482683192757677914940020171" + , "263142239116527075771562086339008294416376465977961141787920329266076113054" + , "1891688623552373296376714072476711345918930670293350449445947359552100105" + , "220593297143937221304009380986123055419843202128717069701373701524573341770" + , "189560068926173621061041209275349227211785974917264051442175334987570569926" + , "389942783260438196800796579051005936150637578365665603675891839955197836860" + , "11039268777535933309761454066968379385803811649178751017683921174521809628" + , "101804830779343249679519503208665210274154642223309969744935313021428436647" + , "427976836071500757088019738220177716154675361488351143100024523789581661294" + , "340152200984249393574062790222940443100537658406052268530670410831869358091" + , "99454559697654930982795138745749463360065638334130260835856093713183607796" + , "396886155136900795735023617950285359476480155376548926721172526900807802732" + , "285373912510002102245297159608488985909135098692355296269840174450070384059" + , "61229070421207552022095926461183597732975710538845760769741778670579471610" + , "398532841083151988050903463941546046818647456660138485890065585460553270029" + , "14242087010600449363366200529242716102819449549447957136832732192340265734" + , "181895206614024541357659462295953616596037608981854733395637141802892449812" + , "76808383121187531214368413693415588785522961317764364771973066323256106717" + , "43708550185311851722847618598117493655501148715925263112055137772889897227" + , "167587120847487421794174745297925163458630632152977961092975082383666984320" + , "406415128537116706408857941555216611036608182780676673731493700483086435223" + , "412897569873740542744872035761409659198035120088201662646852755860136577135" + , "153726232084468805319132005178071672273746389245600604729179943495456529386" + , "110575936082498431398687557847432667364191289309918309424659837348179383694" + , "24387154554490542590275562527179934223944356863752781039864682448646908554" + , "110727" ] ] , "merklePaths": - [ [ "12330511756602656312909435206144412037562550923861053314147193494605624608532" - , "11626412651279744307149389430094868876817959605147995292836995419396445628874" - , "5992799448428980485292103987999399446160713735809250167288027256759377161164" - , "19665782623633007009046423286205585606029554995684266199914536255425604862856" - , "16487082247902739801276638140231448150060608377815926330854457096549924699346" - , "13757776896542890425183206586238760231244468647794123671233758868377423038254" - , "5689382212199289790852870716648320118282913659439556386010574440270030991956" - , "19397819444005071538172083272808000152189643623980212079104170950073560541073" - , "13602141253349313166682170066159161155345481788270338632198566205283140117430" + [ [ "7373747752543321872671006702987703054447808001129167777407466206151522451265" + , "8420726134454359687115045434706519381120587235397847905068479278259503492432" + , "18942830603551638575593850445655121833836976289348882089436275099799063637805" + , "4399578452303778083874951258767944442723965256441309824599581888565983403723" + , "14987716275254692797163785623407414555623098677415753509802959988117217178957" + , "14504305765289705714959523666100275156034056689367568164630385862257567596209" + , "21585154602684431004664192609435970618315270163117104332808296407713696646963" + , "2931501375025361512680880889113088146128601864308065347211197315962956026784" + , "753394286566925219097206250389368333432634432323735599417295414636472248175" , "0" , "0" , "0" @@ -391,15 +391,15 @@ , "0" , "0" ] - , [ "20475873274412005481064639175076772466269555202404881992985142339142752174247" - , "16346160910918020455716115524174351294558871582658635761456705803199256575588" - , "2853750013041818757293788273269978233503065226276819540991640203844566736443" - , "9192572535522846104757923561847416578417599996904208474460700268961782558170" - , "11041850361074018956732434352769265303294549935255362322653487210796196161858" - , "20835509643844784930831626622202658364476409300598072395494952478408974334325" - , "15426115581767819720710837762133134950520914636073261355445708100826108573907" - , "7565353224987902191368863653499353764559862741092477570130316358454603122676" - , "2622681935585012630617774892501744551457568716225188460692779556142778732663" + , [ "20126750780276268127690965733296377677070637077319109102724005804976612746410" + , "14298704079506937424115665774379195626051878750713980674935837916415747264372" + , "18434013679724585603972115803240491189517054395200525089649768928059224533412" + , "2248205094890983687923295083447522917346665635408860192469052025518933886463" + , "15384436141469834918844575336939735185900104023498279758708633350011804221230" + , "14530681811029504601104483116160123805797142572456938135436967673828356935563" + , "9066584305036939167027344222366847339672969594669060551494519060445399840895" + , "21059660862864181741674742678253070960085805242694204828236762991713925793996" + , "4321383376487955227605650522541282456783351259162650469028222578309675166129" , "0" , "0" , "0" @@ -424,15 +424,15 @@ , "0" , "0" ] - , [ "13099868869639061574800067722436547911616384523753701384396275064543709640456" - , "353757809120595355213201328586632712724405232919181040928026587840976194078" - , "17653300914565730132855106316678548541847283141888106932466281385199556950861" - , "15467462085462582082877261755656498905479817107855355753427299990712166382496" - , "8291733777946446853018893495264026584437749231931118771866890345692346711355" - , "15510790697317206014779022286261864844918915222875014882833700758879700055506" - , "5689382212199289790852870716648320118282913659439556386010574440270030991956" - , "19397819444005071538172083272808000152189643623980212079104170950073560541073" - , "13602141253349313166682170066159161155345481788270338632198566205283140117430" + , [ "5720944610738175934791972054315102315741524794129861342502151031247353238022" + , "13964759514893670346288124114605469398041379144080844614386516784465282754556" + , "21599971792671051983116190358720793528318745650813491612372913354983173382690" + , "19074984838142632417524942246453903491596254186349081172920241768853464751785" + , "4143634618854362073393331079769701368365647651895663657135606826129272904514" + , "10120699018002766520605012835043517238241846918467244955580419060582311503402" + , "14194477827915185505458417598612325688837579145207814811446082877899577756037" + , "14845511719423838741057110810816316364397112336760667021406141769423486465287" + , "4321383376487955227605650522541282456783351259162650469028222578309675166129" , "0" , "0" , "0" @@ -457,15 +457,15 @@ , "0" , "0" ] - , [ "123238869181525326412236116167249816873084559218151119452851092131080991962" - , "9610314342084317647296061595824603740114670828357751076517430572434680540425" - , "16802740554584732104294972716558962567961331277692246600846665155168171370476" - , "151083360419914122898584757765086723506432610661508069194962432698872036623" - , "10357032992337239725601662829902169825217513617307319193581711776597892496381" - , "10120699018002766520605012835043517238241846918467244955580419060582311503402" - , "21149604008153751948441881526949680605328007895979738537313721955134548786062" - , "5720106921878932614189421948890362637585879521377362100104826996201092964473" - , "2622681935585012630617774892501744551457568716225188460692779556142778732663" + , [ "10285302562082174938285638104199311170526677488297647842070908768305075159794" + , "9357815062369209476487129340095612795252170063291519616442964135783445311024" + , "14507050748944851192368381763491696453209651823257447903347978564867227281695" + , "20830641738320729484139578353103951192059236196132121539336645254757079614466" + , "12137458272938993395365069993530823235365368235372980480696999645904685829194" + , "18912452177558338907496075944016948101194869472129501293341731076145439018932" + , "15509047188550352526985186698792746340149486344844955562967609109232692051434" + , "14845511719423838741057110810816316364397112336760667021406141769423486465287" + , "4321383376487955227605650522541282456783351259162650469028222578309675166129" , "0" , "0" , "0" @@ -490,15 +490,15 @@ , "0" , "0" ] - , [ "19153513112714782931111012244694922243101677748840856395814929006033044311081" - , "21046138187228318287277629107063936540039891592394801899272249280765102572688" - , "18057980437430910028015917806534512217725128031222973066601095455076586015436" - , "5766677914654397407881589917461473873246279171605373166264025525757502238061" - , "12019967669236656188577515900815533059046454955207846938479617973037184411021" - , "14504305765289705714959523666100275156034056689367568164630385862257567596209" - , "7152002871325824138073253423783370852632926621899161541618248808716037342022" - , "9714587356194206699401761190093056901650105401919163689816999407566849779455" - , "13602141253349313166682170066159161155345481788270338632198566205283140117430" + , [ "21217981719760624184877899649882572628305881461831400142116503211033942407298" + , "7348866387991579504244010054992642796394019938394311297218533889832080995705" + , "21793532448552726990639175253292349994812377250225379673021636008345813367690" + , "18021229518058998625724208625873480268991803541443736945246046877239703762505" + , "5858768953898621623534272836625484139988791502841057904004779672922508038693" + , "1509463060146817208499302461092091752990671336103536081673593318106800164255" + , "10567949269151535789108208893004343030547269413976007251456323997749514164406" + , "2931501375025361512680880889113088146128601864308065347211197315962956026784" + , "753394286566925219097206250389368333432634432323735599417295414636472248175" , "0" , "0" , "0" diff --git a/tests/circuits/fixtures/proof_main.bin b/tests/circuits/fixtures/proof_main.bin index bdeb56cf..98e58b19 100644 Binary files a/tests/circuits/fixtures/proof_main.bin and b/tests/circuits/fixtures/proof_main.bin differ diff --git a/tests/circuits/fixtures/proof_main.r1cs b/tests/circuits/fixtures/proof_main.r1cs index 4df13962..70c0b209 100644 Binary files a/tests/circuits/fixtures/proof_main.r1cs and b/tests/circuits/fixtures/proof_main.r1cs differ diff --git a/tests/circuits/fixtures/proof_main.wasm b/tests/circuits/fixtures/proof_main.wasm index 8e9c3f88..a35e682b 100644 Binary files a/tests/circuits/fixtures/proof_main.wasm and b/tests/circuits/fixtures/proof_main.wasm differ diff --git a/tests/circuits/fixtures/proof_main.zkey b/tests/circuits/fixtures/proof_main.zkey index d59101f3..e727a707 100644 Binary files a/tests/circuits/fixtures/proof_main.zkey and b/tests/circuits/fixtures/proof_main.zkey differ diff --git a/tests/integration/5_minutes/testdatasets.nim b/tests/integration/5_minutes/testdatasets.nim index f388d7ff..4f28bb24 100644 --- a/tests/integration/5_minutes/testdatasets.nim +++ b/tests/integration/5_minutes/testdatasets.nim @@ -60,3 +60,12 @@ suite "Node datasets": fail() except HttpError as error: check "404" in error.msg + + test "cannot upload dataset that would exceed quota": + let smallNode = await testbed.node.storageQuota(1024 * 1024).start() + + try: + discard await testbed.dataset.data(2 * 1024 * 1024).upload(smallNode) + fail() + except HttpError as error: + check "413" in error.msg diff --git a/tests/integration/5_minutes/testupdownload.nim b/tests/integration/5_minutes/testupdownload.nim index fc0de0e8..5a3095a4 100644 --- a/tests/integration/5_minutes/testupdownload.nim +++ b/tests/integration/5_minutes/testupdownload.nim @@ -44,9 +44,9 @@ suite "Uploads and downloads": let exampleData = "some file contents" let exampleDataManifest = %*{ - "treeCid": "zDzSvJTezk7bJNQqFq8k1iHXY84psNuUfZVusA5bBQQUSuyzDSVL", + "treeCid": "zDzSvJTfBxWb7YJRbrX9LMbkm4FkhvBx5pRb6mc9ZntsGcHbZeiq", "datasetSize": 18, - "blockSize": 65536, + "blockSize": 524288, "protected": false, "filename": nil, "mimetype": nil, diff --git a/tests/testbed/builders/node.nim b/tests/testbed/builders/node.nim index 05d613cf..198aef65 100644 --- a/tests/testbed/builders/node.nim +++ b/tests/testbed/builders/node.nim @@ -242,6 +242,7 @@ func circomGraphResolved(builder: NodeBuilder): ?string = proc start*(builder: NodeBuilder): Future[Node] {.async.} = var arguments: seq[string] arguments.add("--disc-port=" & $(await builder.discoveryPortResolved)) + arguments.add("--nat=none") # don't need nat for integration tests for bootstrapNode in await builder.bootstrapNodesResolved: arguments.add("--bootstrap-node=" & bootstrapNode) if builder.logTopics.len > 0: diff --git a/vendor/archivist-contracts b/vendor/archivist-contracts index c0fafaad..6266ce5f 160000 --- a/vendor/archivist-contracts +++ b/vendor/archivist-contracts @@ -1 +1 @@ -Subproject commit c0fafaad1daa303b841cdba6b78ba6c62b2dee6b +Subproject commit 6266ce5fd977b4d8203307a6d39ad66a2a569d82 diff --git a/vendor/archivist-storage-proofs-circuits b/vendor/archivist-storage-proofs-circuits index 21ec7719..963317bd 160000 --- a/vendor/archivist-storage-proofs-circuits +++ b/vendor/archivist-storage-proofs-circuits @@ -1 +1 @@ -Subproject commit 21ec771982a938836b54ea574913ff7964cb83fe +Subproject commit 963317bdbe0bfdcce661f8e84f87ecd8672bca1c diff --git a/vendor/nimble/circom_witnessgen b/vendor/nimble/circom_witnessgen index 15289d5b..6599f3a0 160000 --- a/vendor/nimble/circom_witnessgen +++ b/vendor/nimble/circom_witnessgen @@ -1 +1 @@ -Subproject commit 15289d5b896cea5df7cc91e2c9bec054238bc74a +Subproject commit 6599f3a06c98474c95213f602f32a167503f50b6