diff --git a/.github/workflows/mops-test.yml b/.github/workflows/mops-test.yml index 16863e0..27f6568 100644 --- a/.github/workflows/mops-test.yml +++ b/.github/workflows/mops-test.yml @@ -15,8 +15,11 @@ jobs: - uses: actions/checkout@v4 - uses: ZenVoich/setup-mops@v1 + - name: Install dependencies + run: mops install + - name: make sure moc is installed run: mops toolchain bin moc || mops toolchain use moc latest - name: run tests - run: mops test \ No newline at end of file + run: mops test diff --git a/changelog.md b/changelog.md index 0ff54fa..8b87d5b 100644 --- a/changelog.md +++ b/changelog.md @@ -35,3 +35,11 @@ Before: ## Version 1.0.5 - The performance enhanced ( == less cycles needed for the operations. approx 30-40 %) + +## Version 1.0.6 + +- updated dependencies + +- removed dependency memory-buffer and StableTrieMap through copy and paste + the needed code directly. + \ No newline at end of file diff --git a/makefile b/makefile index ddfd124..567b9d6 100644 --- a/makefile +++ b/makefile @@ -1,16 +1,16 @@ -install-check: +install-check: ifeq (, $(shell which curl)) - @echo No curl is installed, curl will be installed now.... + @echo No curl is installed, curl will be installed now.... @sudo apt-get install curl -y endif -ifeq (,$(shell which $(HOME)/bin/dfx)) +ifeq (,$(shell which $(HOME)/bin/dfx)) @echo No dfx is installed, dfx will be installed now.... curl -fsSL https://internetcomputer.org/install.sh -o install_dfx.sh chmod +x install_dfx.sh ./install_dfx.sh - rm install_dfx.sh + rm install_dfx.sh endif ifeq (, $(shell which nodejs)) @@ -25,7 +25,7 @@ ifeq (, $(shell which mops)) sudo npm i -g ic-mops endif -ifeq (, $(shell which $(HOME)/bin/vessel)) +ifeq (, $(shell which $(HOME)/bin/vessel)) rm installvessel.sh -f echo '#install vessel'>installvessel.sh echo cd $(HOME)/bin>>installvessel.sh @@ -35,12 +35,12 @@ ifeq (, $(shell which $(HOME)/bin/vessel)) chmod +x installvessel.sh ./installvessel.sh rm installvessel.sh -f -endif +endif mops init - mops add memory-buffer mops add test mops add memory-region mops add bench --dev mops update - dfx upgrade \ No newline at end of file +# dfx upgrade + dfxvm update diff --git a/mops.toml b/mops.toml index bad886d..5082d83 100644 --- a/mops.toml +++ b/mops.toml @@ -1,21 +1,19 @@ [package] name = "memory-hashtable" -version = "1.0.5" +version = "1.0.6" description = "Storing/updating/deletion/retrieving of a single blob-value per key. (url: https://github.com/fGhost713/memory-HashTable)" repository = "https://github.com/fGhost713/memory-HashTable" -keywords = [ "storing", "memory", "hashlist", "hashmap", "datastructure" ] +keywords = [ "storing", "memory", "hashlist", "hashmap", "data-structure" ] license = "MIT" [dependencies] -base = "0.10.4" -memory-region = "0.2.0" -StableTrieMap = "https://github.com/NatLabs/StableTrieMap#main@4781cb03efd34b124c22396c69710b374366c797" -itertools = "0.2.1" -test = "1.2.0" -memory-buffer = "0.0.2" +base = "0.16.0" +memory-region = "1.3.2" +itertools = "0.2.2" +test = "2.1.1" [toolchain] -wasmtime = "18.0.1" +wasmtime = "21.0.0" [dev-dependencies] -bench = "1.0.0" \ No newline at end of file +bench = "1.0.0" diff --git a/src/helpers/blobify.mo b/src/helpers/blobify.mo new file mode 100644 index 0000000..2d8c55f --- /dev/null +++ b/src/helpers/blobify.mo @@ -0,0 +1,182 @@ + +/// Blobify is a module that provides a generic interface for converting +/// values to and from blobs. It is intended to be used for serializing +/// and deserializing values that will be stored in persistent stable memory. +/// +/// The Blobify module provides a default implementation for the following +/// types: +/// - Nat +/// - Nat8 +/// - Nat16 +/// - Nat32 +/// - Nat64 +/// - Blob +/// - Bool +/// - Text +/// - Principal + +import TextModule "mo:base/Text"; +import BlobModule "mo:base/Blob"; +import ArrayModule "mo:base/Array"; +import NatModule "mo:base/Nat"; +import Nat8Module "mo:base/Nat8"; +import Nat16Module "mo:base/Nat16"; +import Nat32Module "mo:base/Nat32"; +import Nat64Module "mo:base/Nat64"; +import PrincipalModule "mo:base/Principal"; + +import Debug "mo:base/Debug"; + +module { + + let Base = { + Array = ArrayModule; + Blob = BlobModule; + Nat = NatModule; + Nat8 = Nat8Module; + Nat16 = Nat16Module; + Nat32 = Nat32Module; + Nat64 = Nat64Module; + Text = TextModule; + Principal = PrincipalModule; + }; + + public type Blobify = { + to_blob : (A) -> Blob; + from_blob : (Blob) -> A; + }; + + // Default blobify helpers return blobs in little-endian format. + public let Nat : Blobify = { + to_blob = func(n : Nat) : Blob { + if (n == 0) { return "\00" }; + var num = n; + var nbytes = 0; + + while (num > 0) { + num /= 255; + nbytes += 1; + }; + + num := n; + + let arr = ArrayModule.tabulate( + nbytes, + func(_ : Nat) : Nat8 { + let tmp = num % 255; + num /= 255; + Nat8Module.fromNat(tmp); + }, + ); + + Base.Blob.fromArray(arr); + }; + from_blob = func(blob : Blob) : Nat { + var n = 0; + let bytes = Base.Blob.toArray(blob); + + var j = bytes.size(); + + while (j > 0){ + let byte = bytes.get(j - 1); + n *= 255; + n += Base.Nat8.toNat(byte); + + j -= 1; + }; + + n; + }; + }; + + public let Nat8 : Blobify = { + to_blob = func(n : Nat8) : Blob { Base.Blob.fromArray([n]) }; + from_blob = func(blob : Blob) : Nat8 { Base.Blob.toArray(blob)[0] }; + }; + + public let Nat16 : Blobify = { + to_blob = func(n : Nat16) : Blob { + Base.Blob.fromArray([ + Base.Nat8.fromNat16(n & 0xff), + Base.Nat8.fromNat16(n >> 8), + ]) + }; + from_blob = func(blob : Blob) : Nat16 { + let bytes = Base.Blob.toArray(blob); + + let _n16 = Base.Nat16.fromNat8(bytes[1] << 8) + | Base.Nat16.fromNat8(bytes[0]); + }; + }; + + public let Nat32 : Blobify = { + to_blob = func(n : Nat32) : Blob{ + Base.Blob.fromArray([ + Base.Nat8.fromNat(Base.Nat32.toNat(n & 0xff)), + Base.Nat8.fromNat(Base.Nat32.toNat((n >> 8) & 0xff)), + Base.Nat8.fromNat(Base.Nat32.toNat((n >> 16) & 0xff)), + Base.Nat8.fromNat(Base.Nat32.toNat(n >> 24)), + ]) + }; + from_blob = func(blob : Blob) : Nat32 { + let bytes = Base.Blob.toArray(blob); + + let _n32 = Base.Nat32.fromNat(Base.Nat8.toNat(bytes[3] << 24)) + | Base.Nat32.fromNat(Base.Nat8.toNat(bytes[2] << 16)) + | Base.Nat32.fromNat(Base.Nat8.toNat(bytes[1] << 8)) + | Base.Nat32.fromNat(Base.Nat8.toNat(bytes[0])); + }; + }; + + public let Nat64 : Blobify = { + to_blob = func(n : Nat64) : Blob { + Base.Blob.fromArray([ + Base.Nat8.fromNat(Base.Nat64.toNat(n & 0xff)), + Base.Nat8.fromNat(Base.Nat64.toNat((n >> 8) & 0xff)), + Base.Nat8.fromNat(Base.Nat64.toNat((n >> 16) & 0xff)), + Base.Nat8.fromNat(Base.Nat64.toNat((n >> 24) & 0xff)), + Base.Nat8.fromNat(Base.Nat64.toNat((n >> 32) & 0xff)), + Base.Nat8.fromNat(Base.Nat64.toNat((n >> 40) & 0xff)), + Base.Nat8.fromNat(Base.Nat64.toNat((n >> 48) & 0xff)), + Base.Nat8.fromNat(Base.Nat64.toNat(n >> 56)), + ]) + }; + from_blob = func(blob : Blob) : Nat64 { + let bytes = Base.Blob.toArray(blob); + + let _n64 = Base.Nat64.fromNat(Base.Nat8.toNat(bytes[7] << 56)) + | Base.Nat64.fromNat(Base.Nat8.toNat(bytes[6] << 48)) + | Base.Nat64.fromNat(Base.Nat8.toNat(bytes[5] << 40)) + | Base.Nat64.fromNat(Base.Nat8.toNat(bytes[4] << 32)) + | Base.Nat64.fromNat(Base.Nat8.toNat(bytes[3] << 24)) + | Base.Nat64.fromNat(Base.Nat8.toNat(bytes[2] << 16)) + | Base.Nat64.fromNat(Base.Nat8.toNat(bytes[1] << 8)) + | Base.Nat64.fromNat(Base.Nat8.toNat(bytes[0])); + }; + }; + + public let Blob : Blobify = { + to_blob = func(b : Blob) : Blob = b; + from_blob = func(blob : Blob) : Blob = blob; + }; + + public let Bool : Blobify = { + to_blob = func(b : Bool) : Blob = Base.Blob.fromArray([if (b) 1 else 0]); + from_blob = func(blob : Blob) : Bool { + blob == Base.Blob.fromArray([1]); + }; + }; + + public let Text : Blobify = { + to_blob = func(t : Text) : Blob = TextModule.encodeUtf8(t); + from_blob = func(blob : Blob) : Text { + let ?text = TextModule.decodeUtf8(blob) else Debug.trap("from_blob() on Blobify.Text failed to decodeUtf8"); + text; + }; + }; + + public let Principal : Blobify = { + to_blob = func(p : Principal) : Blob { Base.Principal.toBlob(p) }; + from_blob = func(blob : Blob) : Principal { Base.Principal.fromBlob(blob) }; + }; +}; diff --git a/src/helpers/stableTrieMap.mo b/src/helpers/stableTrieMap.mo new file mode 100644 index 0000000..8f07431 --- /dev/null +++ b/src/helpers/stableTrieMap.mo @@ -0,0 +1,197 @@ +// copied from here: https://github.com/NatLabs/StableTrieMap/blob/main/src/lib.mo + +import Trie "mo:base/Trie"; +import Hash "mo:base/Hash"; +import Iter "mo:base/Iter"; +import List "mo:base/List"; + +module { + public type StableTrieMap = { + var trie : Trie.Trie; + var _size : Nat; + + }; + + public func new() : StableTrieMap { + { + var trie = Trie.empty(); + var _size = 0; + }; + }; + + public func size(self : StableTrieMap) : Nat { + self._size; + }; + + public func replace( + self : StableTrieMap, + keyEq : (K, K) -> Bool, + keyHash : (K) -> Hash.Hash, + key : K, + val : V, + ) : ?V { + let keyObj = { key; hash = keyHash(key) }; + + let (updatedMap, prevVal) = Trie.put(self.trie, keyObj, keyEq, val); + + self.trie := updatedMap; + + switch (prevVal) { + case (null) { self._size += 1 }; + case (_) {}; + }; + + prevVal; + }; + + public let put = func( + self : StableTrieMap, + keyEq : (K, K) -> Bool, + keyHash : (K) -> Hash.Hash, + key : K, + val : V, + ) { ignore replace(self, keyEq, keyHash, key, val) }; + + public func get( + self : StableTrieMap, + keyEq : (K, K) -> Bool, + keyHash : (K) -> Hash.Hash, + key : K, + ) : ?V { + let keyObj = { key; hash = keyHash(key) }; + Trie.find(self.trie, keyObj, keyEq); + }; + + public func remove( + self : StableTrieMap, + keyEq : (K, K) -> Bool, + keyHash : (K) -> Hash.Hash, + key : K, + ) : ?V { + let keyObj = { key; hash = keyHash(key) }; + + let (updatedMap, prevVal) = Trie.remove(self.trie, keyObj, keyEq); + self.trie := updatedMap; + + switch (prevVal) { + case (?_) { self._size -= 1 }; + case (null) {}; + }; + + prevVal; + }; + + public func delete( + self : StableTrieMap, + keyEq : (K, K) -> Bool, + keyHash : (K) -> Hash.Hash, + key : K, + ) { + ignore remove(self, keyEq, keyHash, key); + }; + + public func entries(self : StableTrieMap) : Iter.Iter<(K, V)> { + object { + var stack = ?(self.trie, null) : List.List>; + + public func next() : ?(K, V) { + switch stack { + case null { null }; + case (?(trie, stack2)) { + switch trie { + case (#empty) { + stack := stack2; + next(); + }; + case (#leaf({ keyvals = null })) { + stack := stack2; + next(); + }; + case (#leaf({ size = c; keyvals = ?((k, v), kvs) })) { + stack := ?(#leaf({ size = c -1; keyvals = kvs }), stack2); + ?(k.key, v); + }; + case (#branch(br)) { + stack := ?(br.left, ?(br.right, stack2)); + next(); + }; + }; + }; + }; + }; + }; + }; + + public func keys(self : StableTrieMap) : Iter.Iter { + Iter.map<(K, V), K>(entries(self), func((key, _)) { key }); + }; + + public func vals(self : StableTrieMap) : Iter.Iter { + Iter.map<(K, V), V>(entries(self), func((_, val)) { val }); + }; + + public func fromEntries( + _entries : Iter.Iter<(K, V)>, + keyEq : (K, K) -> Bool, + keyHash : (K) -> Hash.Hash, + ) : StableTrieMap { + let triemap = new(); + + for ((key, val) in _entries) { + put(triemap, keyEq, keyHash, key, val); + }; + + triemap; + }; + + public func clone( + self : StableTrieMap, + keyEq : (K, K) -> Bool, + keyHash : (K) -> Hash.Hash, + ) : StableTrieMap { + fromEntries(entries(self), keyEq, keyHash); + }; + + public func clear(self : StableTrieMap) { + self.trie := Trie.empty(); + self._size := 0; + }; + + // additional helper functions + public func isEmpty(self : StableTrieMap) : Bool { + self._size == 0; + }; + + public func containsKey( + self : StableTrieMap, + keyEq : (K, K) -> Bool, + keyHash : (K) -> Hash.Hash, + key : K, + ) : Bool { + switch (get(self, keyEq, keyHash, key)) { + case (?v) true; + case (_) false; + }; + }; + + /// Adds the given `defaultVal` if the key does not exist in the map + /// and updates the value of an existing key + public func putOrUpdate( + self : StableTrieMap, + keyEq : (K, K) -> Bool, + keyHash : (K) -> Hash.Hash, + key : K, + defaultVal : V, + update : (V) -> V, + ) { + switch (get(self, keyEq, keyHash, key)) { + case (?val) { + put(self, keyEq, keyHash, key, update(val)); + }; + case (_) { + put(self, keyEq, keyHash, key, defaultVal); + }; + }; + }; + +}; diff --git a/src/lib.mo b/src/lib.mo index e7ba3a4..67a91b6 100644 --- a/src/lib.mo +++ b/src/lib.mo @@ -1,21 +1,14 @@ import HashTableTypes "types/hashTableTypes"; -import LibKeyInfo "modules/libKeyInfo"; -import LibWrappedBlob "modules/libWrappedBlob"; -import Option "mo:base/Option"; -import Blob "mo:base/Blob"; import Nat8 "mo:base/Nat8"; -import List "mo:base/List"; -import Iter "mo:base/Iter"; import Nat64 "mo:base/Nat64"; -import Debug "mo:base/Debug"; import Array "mo:base/Array"; -import BlobifyModule "mo:memory-buffer/Blobify"; +import BlobifyModule "helpers/blobify"; import { MemoryRegion } "mo:memory-region"; -import StableTrieMap "mo:StableTrieMap"; +import StableTrieMap "helpers/stableTrieMap"; import MemoryHashTableModule "/modules/memoryHashTable"; module { - + public type MemoryStorage = HashTableTypes.MemoryStorage; private type KeyInfo = HashTableTypes.KeyInfo; private type WrappedBlob = HashTableTypes.WrappedBlob; @@ -39,4 +32,4 @@ module { }; return newItem; }; -}; \ No newline at end of file +}; diff --git a/src/modules/libIndexMapping.mo b/src/modules/libIndexMapping.mo index 777f0ec..2729b5a 100644 --- a/src/modules/libIndexMapping.mo +++ b/src/modules/libIndexMapping.mo @@ -1,16 +1,10 @@ import Blob "mo:base/Blob"; import HashTableTypes "../types/hashTableTypes"; -import StableTrieMap "mo:StableTrieMap"; +import StableTrieMap "../helpers/stableTrieMap"; import Nat32 "mo:base/Nat32"; import Nat64 "mo:base/Nat64"; -import Region "mo:base/Region"; -import Iter "mo:base/Iter"; import List "mo:base/List"; -import Binary "../helpers/binary"; -import Itertools "mo:itertools/Iter"; -import Option "mo:base/Option"; -import Debug "mo:base/Debug"; -import { MemoryRegion } "mo:memory-region"; + module { diff --git a/src/modules/libKeyInfo.mo b/src/modules/libKeyInfo.mo index e5cea1c..ab481f5 100644 --- a/src/modules/libKeyInfo.mo +++ b/src/modules/libKeyInfo.mo @@ -1,15 +1,11 @@ import Blob "mo:base/Blob"; import HashTableTypes "../types/hashTableTypes"; -import StableTrieMap "mo:StableTrieMap"; import Nat32 "mo:base/Nat32"; import Nat64 "mo:base/Nat64"; import Region "mo:base/Region"; import Iter "mo:base/Iter"; import List "mo:base/List"; -import Binary "../helpers/binary"; -import Itertools "mo:itertools/Iter"; import Option "mo:base/Option"; -import Debug "mo:base/Debug"; import { MemoryRegion } "mo:memory-region"; import libIndexMapping "libIndexMapping"; @@ -67,7 +63,7 @@ module { public func delete_keyinfo(memoryStorage : MemoryStorage, keyInfoAddress : Nat64) { let keyInfoSize : Nat64 = Region.loadNat64(memoryStorage.memory_region.region, keyInfoAddress); - + // Deleting by giving the memory free MemoryRegion.deallocate(memoryStorage.memory_region, Nat64.toNat(keyInfoAddress), Nat64.toNat(keyInfoSize)); }; diff --git a/src/modules/libWrappedBlob.mo b/src/modules/libWrappedBlob.mo index 3cf09ae..c41519a 100644 --- a/src/modules/libWrappedBlob.mo +++ b/src/modules/libWrappedBlob.mo @@ -1,18 +1,11 @@ import Blob "mo:base/Blob"; import HashTableTypes "../types/hashTableTypes"; -import StableTrieMap "mo:StableTrieMap"; +import StableTrieMap "../helpers/stableTrieMap"; import Nat32 "mo:base/Nat32"; import Nat64 "mo:base/Nat64"; import Region "mo:base/Region"; -import Iter "mo:base/Iter"; import Nat "mo:base/Nat"; import List "mo:base/List"; -import Binary "../helpers/binary"; -import Itertools "mo:itertools/Iter"; -import Option "mo:base/Option"; -import Result "mo:base/Result"; -import Debug "mo:base/Debug"; -import Array "mo:base/Array"; import libKeyInfo "libKeyInfo"; import { MemoryRegion } "mo:memory-region"; import libIndexMapping "libIndexMapping"; @@ -90,7 +83,7 @@ module { }; }; case (_) { // The key was not used before - + let addresses = add_new_item_internal_and_update_key_mappings(key, memoryStorage, blobToStoreOrUpdate); return addresses.1; diff --git a/src/modules/memoryHashTable.mo b/src/modules/memoryHashTable.mo index f907b25..cefde72 100644 --- a/src/modules/memoryHashTable.mo +++ b/src/modules/memoryHashTable.mo @@ -2,10 +2,6 @@ import HashTableTypes "../types/hashTableTypes"; import LibKeyInfo "libKeyInfo"; import LibWrappedBlob "libWrappedBlob"; import Option "mo:base/Option"; -import Result "mo:base/Result"; -import BlobifyModule "mo:memory-buffer/Blobify"; -import { MemoryRegion } "mo:memory-region"; -import StableTrieMap "mo:StableTrieMap"; module { @@ -25,9 +21,9 @@ module { public func get(key : Blob) : ?Blob { let memoryAddressesOrNull = LibKeyInfo.get_memory_addresses(key, memoryStorage); - + switch (memoryAddressesOrNull) { - case (?memoryAddresses) { + case (?memoryAddresses) { let wrappedBlobAddress = memoryAddresses.1; let internalBlob:Blob = LibWrappedBlob.get_internal_blob_from_memory( memoryStorage, wrappedBlobAddress); @@ -41,7 +37,7 @@ module { // Delete value by key public func delete(key : Blob) { - LibWrappedBlob.delete(key, memoryStorage); + LibWrappedBlob.delete(key, memoryStorage); }; }; diff --git a/src/types/hashTableTypes.mo b/src/types/hashTableTypes.mo index 884a9c1..f838824 100644 --- a/src/types/hashTableTypes.mo +++ b/src/types/hashTableTypes.mo @@ -1,5 +1,5 @@ import { MemoryRegion } "mo:memory-region"; -import StableTrieMap "mo:StableTrieMap"; +import StableTrieMap "../helpers/stableTrieMap"; import List "mo:base/List"; import Nat64 "mo:base/Nat64"; import Nat32 "mo:base/Nat32";