|
| 1 | +"""Changed-path Bloom filters used by commit-graph files.""" |
| 2 | +from __future__ import annotations |
| 3 | + |
| 4 | +import struct |
| 5 | + |
| 6 | +from . import objects as objs |
| 7 | +from . import workdir |
| 8 | +from .repo import Repository |
| 9 | + |
| 10 | + |
| 11 | +BLOOM_HASH_VERSION = 1 |
| 12 | +BLOOM_NUM_HASHES = 7 |
| 13 | +BLOOM_BITS_PER_ENTRY = 10 |
| 14 | +BLOOM_MAX_CHANGED_PATHS = 512 |
| 15 | +_BLOOM_SEED0 = 0x293AE76F |
| 16 | +_BLOOM_SEED1 = 0x7E646E2C |
| 17 | + |
| 18 | + |
| 19 | +def _murmur3_x86_32(data: bytes, seed: int) -> int: |
| 20 | + c1 = 0xCC9E2D51 |
| 21 | + c2 = 0x1B873593 |
| 22 | + h = seed & 0xFFFFFFFF |
| 23 | + rounded = len(data) & ~3 |
| 24 | + for pos in range(0, rounded, 4): |
| 25 | + k = struct.unpack_from("<I", data, pos)[0] |
| 26 | + k = (k * c1) & 0xFFFFFFFF |
| 27 | + k = ((k << 15) | (k >> 17)) & 0xFFFFFFFF |
| 28 | + k = (k * c2) & 0xFFFFFFFF |
| 29 | + h ^= k |
| 30 | + h = ((h << 13) | (h >> 19)) & 0xFFFFFFFF |
| 31 | + h = (h * 5 + 0xE6546B64) & 0xFFFFFFFF |
| 32 | + |
| 33 | + k = 0 |
| 34 | + tail = data[rounded:] |
| 35 | + if len(tail) == 3: |
| 36 | + k ^= tail[2] << 16 |
| 37 | + if len(tail) >= 2: |
| 38 | + k ^= tail[1] << 8 |
| 39 | + if len(tail) >= 1: |
| 40 | + k ^= tail[0] |
| 41 | + k = (k * c1) & 0xFFFFFFFF |
| 42 | + k = ((k << 15) | (k >> 17)) & 0xFFFFFFFF |
| 43 | + k = (k * c2) & 0xFFFFFFFF |
| 44 | + h ^= k |
| 45 | + |
| 46 | + h ^= len(data) |
| 47 | + h ^= h >> 16 |
| 48 | + h = (h * 0x85EBCA6B) & 0xFFFFFFFF |
| 49 | + h ^= h >> 13 |
| 50 | + h = (h * 0xC2B2AE35) & 0xFFFFFFFF |
| 51 | + h ^= h >> 16 |
| 52 | + return h & 0xFFFFFFFF |
| 53 | + |
| 54 | + |
| 55 | +def _collect_tree_paths(repo: Repository, tree_sha: str, prefix: str, out: set[str]) -> None: |
| 56 | + stack = [(prefix, tree_sha)] |
| 57 | + while stack: |
| 58 | + cur_prefix, cur_tree = stack.pop() |
| 59 | + try: |
| 60 | + entries = workdir._tree_entries(repo, cur_tree) |
| 61 | + except KeyError: |
| 62 | + continue |
| 63 | + for entry in entries: |
| 64 | + path = f"{cur_prefix}{entry.name}" |
| 65 | + if entry.is_dir(): |
| 66 | + stack.append((path + "/", entry.sha)) |
| 67 | + else: |
| 68 | + out.add(path) |
| 69 | + |
| 70 | + |
| 71 | +def _diff_trees(repo: Repository, old_tree: str | None, new_tree: str | None, prefix: str, out: set[str]) -> None: |
| 72 | + if old_tree == new_tree: |
| 73 | + return |
| 74 | + if old_tree is None: |
| 75 | + if new_tree is not None: |
| 76 | + _collect_tree_paths(repo, new_tree, prefix, out) |
| 77 | + return |
| 78 | + if new_tree is None: |
| 79 | + _collect_tree_paths(repo, old_tree, prefix, out) |
| 80 | + return |
| 81 | + try: |
| 82 | + old_entries = {entry.name: entry for entry in workdir._tree_entries(repo, old_tree)} |
| 83 | + new_entries = {entry.name: entry for entry in workdir._tree_entries(repo, new_tree)} |
| 84 | + except KeyError: |
| 85 | + return |
| 86 | + for name in set(old_entries) | set(new_entries): |
| 87 | + old = old_entries.get(name) |
| 88 | + new = new_entries.get(name) |
| 89 | + path = f"{prefix}{name}" |
| 90 | + if old is None: |
| 91 | + if new is not None and new.is_dir(): |
| 92 | + _collect_tree_paths(repo, new.sha, path + "/", out) |
| 93 | + else: |
| 94 | + out.add(path) |
| 95 | + continue |
| 96 | + if new is None: |
| 97 | + if old.is_dir(): |
| 98 | + _collect_tree_paths(repo, old.sha, path + "/", out) |
| 99 | + else: |
| 100 | + out.add(path) |
| 101 | + continue |
| 102 | + if (old.mode, old.sha) == (new.mode, new.sha): |
| 103 | + continue |
| 104 | + if old.is_dir() and new.is_dir(): |
| 105 | + _diff_trees(repo, old.sha, new.sha, path + "/", out) |
| 106 | + else: |
| 107 | + out.add(path) |
| 108 | + |
| 109 | + |
| 110 | +def _with_parent_dirs(paths: set[str]) -> list[str]: |
| 111 | + expanded: set[str] = set() |
| 112 | + for path in paths: |
| 113 | + parts = path.split("/") |
| 114 | + for i in range(1, len(parts)): |
| 115 | + expanded.add("/".join(parts[:i])) |
| 116 | + expanded.add(path) |
| 117 | + return sorted(expanded) |
| 118 | + |
| 119 | + |
| 120 | +def changed_paths_for_commit(repo: Repository, commit: objs.Commit) -> list[str]: |
| 121 | + previous_tree = None |
| 122 | + if commit.parents: |
| 123 | + try: |
| 124 | + parent_type, parent_data = objs.read_object(repo, commit.parents[0]) |
| 125 | + except KeyError: |
| 126 | + parent_type, parent_data = "", b"" |
| 127 | + if parent_type == "commit": |
| 128 | + previous_tree = objs.parse_commit(parent_data).tree |
| 129 | + changed: set[str] = set() |
| 130 | + _diff_trees(repo, previous_tree, commit.tree, "", changed) |
| 131 | + return _with_parent_dirs(changed) |
| 132 | + |
| 133 | + |
| 134 | +def bloom_filter_for_paths(paths: list[str]) -> bytes: |
| 135 | + if not paths: |
| 136 | + return b"\0" |
| 137 | + if len(paths) > BLOOM_MAX_CHANGED_PATHS: |
| 138 | + return b"\xff" |
| 139 | + byte_count = max(1, (len(paths) * BLOOM_BITS_PER_ENTRY + 7) // 8) |
| 140 | + bit_count = byte_count * 8 |
| 141 | + out = bytearray(byte_count) |
| 142 | + for path in paths: |
| 143 | + data = path.encode("utf-8") |
| 144 | + h0 = _murmur3_x86_32(data, _BLOOM_SEED0) |
| 145 | + h1 = _murmur3_x86_32(data, _BLOOM_SEED1) |
| 146 | + for i in range(BLOOM_NUM_HASHES): |
| 147 | + bit = (h0 + i * h1) % bit_count |
| 148 | + out[bit // 8] |= 1 << (bit % 8) |
| 149 | + return bytes(out) |
| 150 | + |
| 151 | + |
| 152 | +def bloom_maybe_contains(filter_data: bytes, path: str) -> bool: |
| 153 | + if filter_data == b"\xff": |
| 154 | + return True |
| 155 | + if not filter_data or filter_data == b"\0": |
| 156 | + return False |
| 157 | + bit_count = len(filter_data) * 8 |
| 158 | + data = path.encode("utf-8") |
| 159 | + h0 = _murmur3_x86_32(data, _BLOOM_SEED0) |
| 160 | + h1 = _murmur3_x86_32(data, _BLOOM_SEED1) |
| 161 | + for i in range(BLOOM_NUM_HASHES): |
| 162 | + bit = (h0 + i * h1) % bit_count |
| 163 | + if not (filter_data[bit // 8] & (1 << (bit % 8))): |
| 164 | + return False |
| 165 | + return True |
| 166 | + |
| 167 | + |
| 168 | +def build_commit_graph_bloom_chunks( |
| 169 | + repo: Repository, |
| 170 | + shas: list[str], |
| 171 | + commits: dict[str, objs.Commit], |
| 172 | +) -> tuple[bytes, bytes]: |
| 173 | + offsets = bytearray() |
| 174 | + filters = bytearray() |
| 175 | + end = 0 |
| 176 | + for sha in shas: |
| 177 | + filter_data = bloom_filter_for_paths(changed_paths_for_commit(repo, commits[sha])) |
| 178 | + filters += filter_data |
| 179 | + end += len(filter_data) |
| 180 | + offsets += struct.pack(">I", end) |
| 181 | + bdat = struct.pack(">III", BLOOM_HASH_VERSION, BLOOM_NUM_HASHES, BLOOM_BITS_PER_ENTRY) + bytes(filters) |
| 182 | + return bytes(offsets), bdat |
| 183 | + |
| 184 | + |
| 185 | +def read_commit_graph_bloom_filters(bidx: bytes, bdat: bytes, commit_count: int) -> list[bytes]: |
| 186 | + if len(bidx) != commit_count * 4: |
| 187 | + raise ValueError("invalid BIDX chunk length") |
| 188 | + if len(bdat) < 12: |
| 189 | + raise ValueError("invalid BDAT chunk length") |
| 190 | + version, hashes, bits_per_entry = struct.unpack(">III", bdat[:12]) |
| 191 | + if (version, hashes, bits_per_entry) != ( |
| 192 | + BLOOM_HASH_VERSION, |
| 193 | + BLOOM_NUM_HASHES, |
| 194 | + BLOOM_BITS_PER_ENTRY, |
| 195 | + ): |
| 196 | + raise ValueError("unsupported commit-graph Bloom settings") |
| 197 | + data = bdat[12:] |
| 198 | + filters: list[bytes] = [] |
| 199 | + previous = 0 |
| 200 | + for i in range(commit_count): |
| 201 | + end = struct.unpack(">I", bidx[i * 4 : i * 4 + 4])[0] |
| 202 | + if end < previous or end > len(data): |
| 203 | + raise ValueError("invalid BIDX offset") |
| 204 | + filters.append(data[previous:end]) |
| 205 | + previous = end |
| 206 | + if previous != len(data): |
| 207 | + raise ValueError("unused BDAT filter bytes") |
| 208 | + return filters |
0 commit comments