diff --git a/.github/workflows/build_and_test_tidesdb.yml b/.github/workflows/build_and_test_tidesdb.yml new file mode 100644 index 0000000..74c7b44 --- /dev/null +++ b/.github/workflows/build_and_test_tidesdb.yml @@ -0,0 +1,186 @@ +name: TidesDB Python CI + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + workflow_dispatch: + +jobs: + build-and-test: + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + name: Linux x64 + - os: macos-latest + name: macOS x64 + - os: windows-latest + name: Windows x64 + + runs-on: ${{ matrix.os }} + name: ${{ matrix.name }} + + steps: + - name: Checkout tidesdb-python repo + uses: actions/checkout@v4 + with: + repository: tidesdb/tidesdb-python + path: tidesdb-python + + - name: Checkout tidesdb repo + uses: actions/checkout@v4 + with: + repository: tidesdb/tidesdb + path: tidesdb + + - name: Install dependencies (Linux) + if: runner.os == 'Linux' + run: | + sudo apt update + sudo apt install -y libzstd-dev liblz4-dev libsnappy-dev build-essential cmake pkg-config + + - name: Install dependencies (macOS) + if: runner.os == 'macOS' + run: brew install zstd lz4 snappy + + - name: Setup MSYS2 (Windows) + if: runner.os == 'Windows' + uses: msys2/setup-msys2@v2 + with: + msystem: MINGW64 + update: true + install: >- + mingw-w64-x86_64-gcc + mingw-w64-x86_64-cmake + mingw-w64-x86_64-make + mingw-w64-x86_64-zstd + mingw-w64-x86_64-lz4 + mingw-w64-x86_64-snappy + mingw-w64-x86_64-python + mingw-w64-x86_64-python-pip + + - name: Configure and build TidesDB (Linux) + if: runner.os == 'Linux' + run: | + cd tidesdb + cmake -S . -B build -DTIDESDB_BUILD_TESTS=OFF -DTIDESDB_WITH_SANITIZER=OFF + cmake --build build --config Release + sudo cmake --install build + sudo ldconfig + + - name: Configure and build TidesDB (macOS) + if: runner.os == 'macOS' + run: | + cd tidesdb + export HOMEBREW_PREFIX=$(brew --prefix) + cmake -S . -B build \ + -DTIDESDB_BUILD_TESTS=OFF \ + -DTIDESDB_WITH_SANITIZER=OFF \ + -DCMAKE_PREFIX_PATH="${HOMEBREW_PREFIX}" + cmake --build build --config Release + sudo cmake --install build + + - name: Create CMake config files for MSYS2 packages (Windows) + if: runner.os == 'Windows' + shell: msys2 {0} + run: | + MINGW_PREFIX_WIN=$(cygpath -m /mingw64) + + mkdir -p /mingw64/lib/cmake/lz4 + mkdir -p /mingw64/lib/cmake/zstd + mkdir -p /mingw64/lib/cmake/Snappy + mkdir -p /mingw64/lib/cmake/PThreads4W + + cat > /mingw64/lib/cmake/lz4/lz4-config.cmake << EOF + if(NOT TARGET lz4::lz4) + add_library(lz4::lz4 SHARED IMPORTED) + set_target_properties(lz4::lz4 PROPERTIES + IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/liblz4.dll" + IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/liblz4.dll.a" + INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" + ) + endif() + EOF + + cat > /mingw64/lib/cmake/zstd/zstd-config.cmake << EOF + if(NOT TARGET zstd::libzstd_shared) + add_library(zstd::libzstd_shared SHARED IMPORTED) + set_target_properties(zstd::libzstd_shared PROPERTIES + IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libzstd.dll" + IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libzstd.dll.a" + INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" + ) + endif() + EOF + + cat > /mingw64/lib/cmake/Snappy/Snappy-config.cmake << EOF + if(NOT TARGET Snappy::snappy) + add_library(Snappy::snappy SHARED IMPORTED) + set_target_properties(Snappy::snappy PROPERTIES + IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libsnappy.dll" + IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libsnappy.dll.a" + INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" + ) + endif() + EOF + + cat > /mingw64/lib/cmake/PThreads4W/PThreads4W-config.cmake << EOF + if(NOT TARGET PThreads4W::PThreads4W) + add_library(PThreads4W::PThreads4W SHARED IMPORTED) + set_target_properties(PThreads4W::PThreads4W PROPERTIES + IMPORTED_LOCATION "${MINGW_PREFIX_WIN}/bin/libwinpthread-1.dll" + IMPORTED_IMPLIB "${MINGW_PREFIX_WIN}/lib/libpthread.dll.a" + INTERFACE_INCLUDE_DIRECTORIES "${MINGW_PREFIX_WIN}/include" + ) + endif() + EOF + + - name: Configure and build TidesDB (Windows) + if: runner.os == 'Windows' + shell: msys2 {0} + run: | + cd tidesdb + cmake -G "MinGW Makefiles" \ + -DCMAKE_MAKE_PROGRAM=mingw32-make \ + -DCMAKE_PREFIX_PATH=/mingw64 \ + -DTIDESDB_WITH_SANITIZER=OFF \ + -DTIDESDB_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -S . -B build + cmake --build build --config Release + cmake --install build --prefix /mingw64 + cp build/libtidesdb.dll /mingw64/bin/ + + - name: Install Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Run Python tests (Linux) + if: runner.os == 'Linux' + run: | + cd tidesdb-python + pip install -e ".[dev]" + pytest -v + + - name: Run Python tests (macOS) + if: runner.os == 'macOS' + run: | + cd tidesdb-python + pip install -e ".[dev]" + pytest -v + + - name: Run Python tests (Windows) + if: runner.os == 'Windows' + shell: msys2 {0} + run: | + export PATH="/mingw64/bin:/mingw64/lib:$PATH" + cd tidesdb-python + python -m venv venv + source venv/bin/activate + pip install pytest pytest-cov + pip install -e . + pytest -v --no-cov \ No newline at end of file diff --git a/.github/workflows/tidesdb-python-build-test.yml b/.github/workflows/tidesdb-python-build-test.yml deleted file mode 100644 index 752617f..0000000 --- a/.github/workflows/tidesdb-python-build-test.yml +++ /dev/null @@ -1,180 +0,0 @@ -name: TidesDB Python Workflow - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - workflow_dispatch: - -jobs: - test: - name: Test on ${{ matrix.os }} with Python ${{ matrix.python-version }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ['3.11'] - - steps: - - name: Checkout Python bindings - uses: actions/checkout@v4 - - - name: Checkout TidesDB main repo - uses: actions/checkout@v4 - with: - repository: tidesdb/tidesdb - ref: master - path: tidesdb-core - - - name: Show TidesDB version - working-directory: tidesdb-core - run: | - echo "Building TidesDB from commit:" - git log -1 --oneline - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - # Ubuntu dependencies - - name: Install dependencies (Ubuntu) - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y \ - build-essential \ - cmake \ - libzstd-dev \ - liblz4-dev \ - libsnappy-dev \ - libssl-dev - - # macOS dependencies - - name: Install dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew install \ - cmake \ - zstd \ - lz4 \ - snappy \ - openssl@3 - - # Windows dependencies - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: | - choco install cmake --installargs 'ADD_CMAKE_TO_PATH=System' - vcpkg install zstd:x64-windows lz4:x64-windows snappy:x64-windows openssl:x64-windows - shell: powershell - - # Build and install TidesDB C library (Ubuntu/macOS) - - name: Build TidesDB (Unix) - if: runner.os != 'Windows' - working-directory: tidesdb-core - run: | - rm -rf build - cmake -S . -B build \ - -DCMAKE_BUILD_TYPE=Release \ - -DTIDESDB_WITH_SANITIZER=OFF \ - -DTIDESDB_BUILD_TESTS=OFF - cmake --build build - sudo cmake --install build - - # Build and install TidesDB C library (Windows) - - name: Build TidesDB (Windows) - if: runner.os == 'Windows' - working-directory: tidesdb-core - run: | - cmake -S . -B build ` - -DCMAKE_BUILD_TYPE=Release ` - -DTIDESDB_WITH_SANITIZER=OFF ` - -DTIDESDB_BUILD_TESTS=OFF ` - -DCMAKE_TOOLCHAIN_FILE="C:/vcpkg/scripts/buildsystems/vcpkg.cmake" - cmake --build build --config Release - cmake --install build --config Release - shell: powershell - - # Update library paths (Ubuntu) - - name: Update library cache (Ubuntu) - if: runner.os == 'Linux' - run: sudo ldconfig - - # Update library paths (macOS) - - name: Set library path (macOS) - if: runner.os == 'macOS' - run: | - echo "DYLD_LIBRARY_PATH=/usr/local/lib:$DYLD_LIBRARY_PATH" >> $GITHUB_ENV - - # Update library paths (Windows) - - name: Set library path (Windows) - if: runner.os == 'Windows' - run: | - echo "$env:ProgramFiles\TidesDB\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - shell: powershell - - # Install Python dependencies - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install pytest pytest-cov - - # Install Python bindings - - name: Install TidesDB Python bindings - run: | - pip install -e . - - # Verify library can be loaded - - name: Verify TidesDB library - run: | - python -c "from tidesdb import TidesDB; print('TidesDB library loaded successfully')" - - # Run tests with verbose output - - name: Run tests - run: | - pytest -v --tb=short -x - timeout-minutes: 60 - - # Upload coverage - - name: Upload coverage to Codecov - if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' - uses: codecov/codecov-action@v4 - with: - file: ./coverage.xml - flags: unittests - name: codecov-umbrella - fail_ci_if_error: false - - - package: - name: Build Package - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install build tools - run: | - python -m pip install --upgrade pip - pip install build twine - - - name: Build package - run: | - python -m build - - - name: Check package - run: | - twine check dist/* - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: python-package - path: dist/ \ No newline at end of file diff --git a/.gitignore b/.gitignore index b88b8df..595919d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,14 +18,39 @@ wheels/ *.egg-info/ .installed.cfg *.egg -.pytest_cache/ -.coverage +*.manifest +*.spec +pip-log.txt +pip-delete-this-directory.txt htmlcov/ .tox/ -.vscode/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +*.mo +*.pot +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ .idea/ +.vscode/ *.swp *.swo -.DS_Store -Thumbs.db -test_db/ \ No newline at end of file +*~ +.mypy_cache/ +.dmypy.json +dmypy.json +.ruff_cache/ +test_db/ +*.db diff --git a/LICENSE b/LICENSE index fa0086a..a612ad9 100644 --- a/LICENSE +++ b/LICENSE @@ -370,4 +370,4 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice --------------------------------------------------------- This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. \ No newline at end of file + defined by the Mozilla Public License, v. 2.0. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 6fbfdd1..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,15 +0,0 @@ -# Include documentation -include README.md -include LICENSE - -# Include package metadata -include pyproject.toml - -# Exclude unnecessary files -global-exclude __pycache__ -global-exclude *.py[co] -global-exclude .DS_Store -global-exclude *.so -global-exclude *.dylib -global-exclude *.dll -global-exclude test_* \ No newline at end of file diff --git a/README.md b/README.md index 7ca4f0f..b156965 100644 --- a/README.md +++ b/README.md @@ -1,567 +1,28 @@ -# tidesdb-python +# TidesDB Python -Official Python bindings for TidesDB v1. - -TidesDB is a fast and efficient key-value storage engine library written in C. The underlying data structure is based on a log-structured merge-tree (LSM-tree). This Python binding provides a Pythonic interface to TidesDB with full support for all v1 features. +Official Python package for [TidesDB](https://tidesdb.com). ## Features -- **ACID Transactions** - Atomic, consistent, isolated, and durable transactions across column families -- **Optimized Concurrency** - Multiple concurrent readers, writers don't block readers -- **Column Families** - Isolated key-value stores with independent configuration -- **Bidirectional Iterators** - Iterate forward and backward over sorted key-value pairs -- **TTL Support** - Time-to-live for automatic key expiration -- **Compression** - Snappy, LZ4, or ZSTD compression support -- **Bloom Filters** - Reduce disk reads with configurable false positive rates -- **Background Compaction** - Automatic or manual SSTable compaction with parallel execution -- **Sync Modes** - Three durability levels: NONE, BACKGROUND, FULL -- **Custom Comparators** - Register custom key comparison functions -- **Error Handling** - Detailed error codes for production use -- **Context Managers** - Pythonic resource management with `with` statements -- **Python Iterator Protocol** - Use iterators in for loops and comprehensions - -## Installation - -### Prerequisites - -You must have the TidesDB v1 shared C library installed on your system. - -**Building TidesDB:** -```bash -# Clone TidesDB repository -git clone https://github.com/tidesdb/tidesdb.git -cd tidesdb - -# Build and install (compile with sanitizer and tests OFF for bindings) -rm -rf build && cmake -S . -B build -DTIDESDB_WITH_SANITIZER=OFF -DTIDESDB_BUILD_TESTS=OFF -cmake --build build -sudo cmake --install build -``` - -**Dependencies:** -- Snappy -- LZ4 -- Zstandard -- OpenSSL - -**On Ubuntu/Debian:** -```bash -sudo apt install libzstd-dev liblz4-dev libsnappy-dev libssl-dev -``` - -**On macOS:** -```bash -brew install zstd lz4 snappy openssl -``` - -### Install Python Package - -```bash -pip install tidesdb -``` - -Or install from source: -```bash -git clone https://github.com/tidesdb/tidesdb-python.git -cd tidesdb-python -pip install -e . -``` - -## Quick Start - -```python -from tidesdb import TidesDB, ColumnFamilyConfig - -# Open database (with optional parameters) -with TidesDB("./mydb", enable_debug_logging=False, max_open_file_handles=1024) as db: - # Create column family - db.create_column_family("users") - - # Write data - with db.begin_txn() as txn: - txn.put("users", b"user:1", b"Alice") - txn.put("users", b"user:2", b"Bob") - txn.commit() - - # Read data - with db.begin_read_txn() as txn: - value = txn.get("users", b"user:1") - print(f"Value: {value.decode()}") # Output: Value: Alice -``` - -## Usage - -### Opening and Closing a Database - -```python -from tidesdb import TidesDB - -# Using context manager (recommended) -with TidesDB("./mydb") as db: - # Use database - pass - -# Manual open/close with options -db = TidesDB( - "./mydb", - enable_debug_logging=False, - max_open_file_handles=1024 # 0 = unlimited, >0 = cache up to N open files -) -# Use database -db.close() -``` - -### Creating and Dropping Column Families - -```python -from tidesdb import ColumnFamilyConfig, CompressionAlgo, SyncMode - -# Create with default configuration -db.create_column_family("my_cf") - -# Create with custom configuration -config = ColumnFamilyConfig( - memtable_flush_size=128 * 1024 * 1024, # 128MB - max_sstables_before_compaction=128, # Trigger compaction at 128 SSTables - compaction_threads=4, # Use 4 threads for parallel compaction - max_level=12, - probability=0.25, - compressed=True, - compress_algo=CompressionAlgo.LZ4, - bloom_filter_fp_rate=0.01, # 1% false positive rate - enable_background_compaction=True, - background_compaction_interval=1000000, # Check every 1 second (microseconds) - use_sbha=True, - sync_mode=SyncMode.BACKGROUND, - sync_interval=1000 # Sync every 1 second (milliseconds) -) - -db.create_column_family("my_cf", config) - -# Drop a column family -db.drop_column_family("my_cf") -``` - -### CRUD Operations - -All operations are performed through transactions for ACID guarantees. - -#### Writing Data - -```python -# Simple write -with db.begin_txn() as txn: - txn.put("my_cf", b"key", b"value") - txn.commit() - -# Multiple operations -with db.begin_txn() as txn: - txn.put("my_cf", b"key1", b"value1") - txn.put("my_cf", b"key2", b"value2") - txn.put("my_cf", b"key3", b"value3") - txn.commit() -``` - -#### Writing with TTL - -```python -import time - -with db.begin_txn() as txn: - # Expire in 10 seconds - ttl = int(time.time()) + 10 - txn.put("my_cf", b"temp_key", b"temp_value", ttl) - txn.commit() - -# TTL examples -ttl = -1 # No expiration -ttl = int(time.time()) + 300 # Expire in 5 minutes -ttl = int(time.time()) + 3600 # Expire in 1 hour -``` - -#### Reading Data - -```python -with db.begin_read_txn() as txn: - value = txn.get("my_cf", b"key") - print(f"Value: {value.decode()}") -``` - -#### Deleting Data - -```python -with db.begin_txn() as txn: - txn.delete("my_cf", b"key") - txn.commit() -``` - -#### Transaction Rollback - -```python -# Manual rollback -with db.begin_txn() as txn: - txn.put("my_cf", b"key", b"value") - txn.rollback() # Changes not applied - -# Automatic rollback on exception -try: - with db.begin_txn() as txn: - txn.put("my_cf", b"key", b"value") - raise ValueError("Error!") -except ValueError: - pass # Transaction automatically rolled back -``` - -### Iterating Over Data - -```python -# Forward iteration -with db.begin_read_txn() as txn: - with txn.new_iterator("my_cf") as it: - it.seek_to_first() - - while it.valid(): - key = it.key() - value = it.value() - print(f"Key: {key}, Value: {value}") - it.next() - -# Backward iteration -with db.begin_read_txn() as txn: - with txn.new_iterator("my_cf") as it: - it.seek_to_last() - - while it.valid(): - key = it.key() - value = it.value() - print(f"Key: {key}, Value: {value}") - it.prev() - -# Using Python iterator protocol -with db.begin_read_txn() as txn: - with txn.new_iterator("my_cf") as it: - it.seek_to_first() - - for key, value in it: - print(f"Key: {key}, Value: {value}") - -# Get all items as list -with db.begin_read_txn() as txn: - with txn.new_iterator("my_cf") as it: - it.seek_to_first() - items = list(it) # List of (key, value) tuples -``` - -### Column Family Statistics - -```python -stats = db.get_column_family_stats("my_cf") - -print(f"Column Family: {stats.name}") -print(f"Comparator: {stats.comparator_name}") -print(f"Number of SSTables: {stats.num_sstables}") -print(f"Total SSTable Size: {stats.total_sstable_size} bytes") -print(f"Memtable Size: {stats.memtable_size} bytes") -print(f"Memtable Entries: {stats.memtable_entries}") -print(f"Compression: {stats.config.compressed}") -print(f"Bloom Filter FP Rate: {stats.config.bloom_filter_fp_rate}") -print(f"Sync Mode: {stats.config.sync_mode}") -``` - -### Listing Column Families - -```python -cf_list = db.list_column_families() -print(f"Column families: {cf_list}") -``` - -### Compaction - -```python -# Automatic background compaction (set during CF creation) -config = ColumnFamilyConfig( - enable_background_compaction=True, - max_sstables_before_compaction=512, # Trigger at 512 SSTables - compaction_threads=4 # Use 4 threads -) -db.create_column_family("my_cf", config) - -# Manual compaction (requires minimum 2 SSTables) -cf = db.get_column_family("my_cf") -cf.compact() -``` - -### Sync Modes - -```python -from tidesdb import SyncMode - -# TDB_SYNC_NONE - Fastest, least durable (OS handles flushing) -config = ColumnFamilyConfig(sync_mode=SyncMode.NONE) - -# TDB_SYNC_BACKGROUND - Balanced (fsync every N milliseconds) -config = ColumnFamilyConfig( - sync_mode=SyncMode.BACKGROUND, - sync_interval=1000 # Sync every 1 second -) - -# TDB_SYNC_FULL - Most durable (fsync on every write) -config = ColumnFamilyConfig(sync_mode=SyncMode.FULL) -``` - -### Compression Algorithms - -```python -from tidesdb import CompressionAlgo - -# No compression (set compressed=False) -config = ColumnFamilyConfig( - compressed=False -) - -# Snappy (fast, default) -config = ColumnFamilyConfig( - compressed=True, - compress_algo=CompressionAlgo.SNAPPY # Value: 0 -) - -# LZ4 (very fast) -config = ColumnFamilyConfig( - compressed=True, - compress_algo=CompressionAlgo.LZ4 # Value: 1 -) - -# Zstandard (high compression) -config = ColumnFamilyConfig( - compressed=True, - compress_algo=CompressionAlgo.ZSTD # Value: 2 -) -``` - -## Working with Python Objects - -### Using Pickle - -```python -import pickle - -# Store Python objects -user_data = { - "name": "John Doe", - "age": 30, - "email": "john@example.com" -} - -with db.begin_txn() as txn: - key = b"user:123" - value = pickle.dumps(user_data) - txn.put("users", key, value) - txn.commit() - -# Retrieve Python objects -with db.begin_read_txn() as txn: - key = b"user:123" - value = txn.get("users", key) - user_data = pickle.loads(value) - print(user_data) -``` - -### Using JSON - -```python -import json - -# Store JSON -data = {"name": "Alice", "score": 95} - -with db.begin_txn() as txn: - key = b"player:1" - value = json.dumps(data).encode() - txn.put("players", key, value) - txn.commit() - -# Retrieve JSON -with db.begin_read_txn() as txn: - key = b"player:1" - value = txn.get("players", key) - data = json.loads(value.decode()) - print(data) -``` - -## Error Handling - -```python -from tidesdb import TidesDBException, ErrorCode - -try: - with db.begin_read_txn() as txn: - value = txn.get("my_cf", b"nonexistent_key") -except TidesDBException as e: - print(f"Error: {e}") - print(f"Error code: {e.code}") - - if e.code == ErrorCode.TDB_ERR_NOT_FOUND: - print("Key not found") - elif e.code == ErrorCode.TDB_ERR_MEMORY: - print("Out of memory") - # ... handle other errors -``` - -**Error Codes:** -- `TDB_SUCCESS` (0) - Operation successful -- `TDB_ERR_MEMORY` (-2) - Memory allocation failed -- `TDB_ERR_INVALID_ARGS` (-3) - Invalid arguments -- `TDB_ERR_IO` (-4) - I/O error -- `TDB_ERR_NOT_FOUND` (-5) - Key not found -- `TDB_ERR_EXISTS` (-6) - Resource already exists -- `TDB_ERR_CORRUPT` (-7) - Data corruption -- `TDB_ERR_LOCK` (-8) - Lock acquisition failed -- `TDB_ERR_TXN_COMMITTED` (-9) - Transaction already committed -- `TDB_ERR_TXN_ABORTED` (-10) - Transaction aborted -- `TDB_ERR_READONLY` (-11) - Write on read-only transaction -- `TDB_ERR_FULL` (-12) - Database full -- `TDB_ERR_INVALID_NAME` (-13) - Invalid name -- `TDB_ERR_INVALID_CF` (-16) - Invalid column family -- `TDB_ERR_THREAD` (-17) - Thread operation failed -- `TDB_ERR_CHECKSUM` (-18) - Checksum verification failed -- `TDB_ERR_KEY_DELETED` (-19) - Key is deleted (tombstone) -- `TDB_ERR_KEY_EXPIRED` (-20) - Key has expired (TTL) - -## Complete Example - -```python -from tidesdb import TidesDB, ColumnFamilyConfig, CompressionAlgo, SyncMode -import time -import json - -# Open database -with TidesDB("./example_db") as db: - # Create column family with custom configuration - config = ColumnFamilyConfig( - memtable_flush_size=64 * 1024 * 1024, - compressed=True, - compress_algo=CompressionAlgo.LZ4, - bloom_filter_fp_rate=0.01, - enable_background_compaction=True, - sync_mode=SyncMode.BACKGROUND, - sync_interval=1000 - ) - - db.create_column_family("users", config) - - # Write data - users = [ - {"id": 1, "name": "Alice", "email": "alice@example.com"}, - {"id": 2, "name": "Bob", "email": "bob@example.com"}, - {"id": 3, "name": "Charlie", "email": "charlie@example.com"}, - ] - - with db.begin_txn() as txn: - for user in users: - key = f"user:{user['id']}".encode() - value = json.dumps(user).encode() - txn.put("users", key, value) - - # Add temporary session data with TTL - session_key = b"session:abc123" - session_value = b"session_data" - ttl = int(time.time()) + 3600 # Expire in 1 hour - txn.put("users", session_key, session_value, ttl) - - txn.commit() - - # Read data - with db.begin_read_txn() as txn: - value = txn.get("users", b"user:1") - user = json.loads(value.decode()) - print(f"User: {user}") - - # Iterate over all users - print("\nAll users:") - with db.begin_read_txn() as txn: - with txn.new_iterator("users") as it: - it.seek_to_first() - for key, value in it: - if key.startswith(b"user:"): - user = json.loads(value.decode()) - print(f" {user['name']} - {user['email']}") - - # Get statistics - stats = db.get_column_family_stats("users") - print(f"\nDatabase Statistics:") - print(f" Memtable Size: {stats.memtable_size} bytes") - print(f" Memtable Entries: {stats.memtable_entries}") - print(f" Number of SSTables: {stats.num_sstables}") - - # Clean up - db.drop_column_family("users") -``` - -## Performance Tips - -1. **Batch operations** in transactions for better performance -2. **Use appropriate sync mode** for your durability requirements -3. **Enable background compaction** for automatic maintenance -4. **Adjust memtable flush size** based on your workload -5. **Use compression** to reduce disk usage and I/O -6. **Configure bloom filters** to reduce unnecessary disk reads -7. **Set appropriate TTL** to automatically expire old data -8. **Use parallel compaction** for faster SSTable merging -9. **Use context managers** to ensure proper resource cleanup - -## Testing - -```bash -# Run tests -python -m pytest test_tidesdb.py -v - -# Run with coverage -python -m pytest test_tidesdb.py --cov=tidesdb --cov-report=html -``` - -## Type Hints - -The package includes full type hints for better IDE support: - -```python -from tidesdb import TidesDB, ColumnFamilyConfig, Transaction -from typing import Optional - -def get_user(db: TidesDB, user_id: int) -> Optional[bytes]: - with db.begin_read_txn() as txn: - try: - key = f"user:{user_id}".encode() - return txn.get("users", key) - except TidesDBException: - return None -``` - -## Concurrency - -TidesDB is designed for high concurrency: - -- **Multiple readers can read concurrently** - No blocking between readers -- **Writers don't block readers** - Readers can access data during writes -- **Writers block other writers** - Only one writer per column family at a time -- **Read transactions** (`begin_read_txn`) acquire read locks -- **Write transactions** (`begin_txn`) acquire write locks on commit -- **Different column families** can be written concurrently +- MVCC with five isolation levels from READ UNCOMMITTED to SERIALIZABLE +- Column families (isolated key-value stores with independent configuration) +- Bidirectional iterators with forward/backward traversal and seek support +- TTL (time to live) support with automatic key expiration +- LZ4, LZ4 Fast, ZSTD, Snappy, or no compression +- Bloom filters with configurable false positive rates +- Global block CLOCK cache for hot blocks +- Savepoints for partial transaction rollback +- Six built-in comparators plus custom registration ## License Multiple licenses apply: -``` -Mozilla Public License Version 2.0 (TidesDB) - --- AND -- - -BSD 3 Clause (Snappy) -BSD 2 (LZ4) -BSD 2 (xxHash - Yann Collet) -BSD (Zstandard) -Apache 2.0 (OpenSSL 3.0+) / OpenSSL License (OpenSSL 1.x) -``` +- Mozilla Public License Version 2.0 (TidesDB) +- BSD 3-Clause (Snappy) +- BSD 2-Clause (LZ4) +- BSD 2-Clause (xxHash - Yann Collet) +- BSD (Zstandard) ## Contributing @@ -569,14 +30,6 @@ Contributions are welcome! Please feel free to submit a Pull Request. ## Support -For issues, questions, or discussions: -- GitHub Issues: https://github.com/tidesdb/tidesdb-python/issues -- Discord Community: https://discord.gg/tWEmjR66cy -- Main TidesDB Repository: https://github.com/tidesdb/tidesdb - -## Links - -- [TidesDB Main Repository](https://github.com/tidesdb/tidesdb) -- [TidesDB Documentation](https://github.com/tidesdb/tidesdb#readme) -- [Other Language Bindings](https://github.com/tidesdb/tidesdb#bindings) -- [PyPI Package](https://pypi.org/project/tidesdb/) \ No newline at end of file +- [Discord](https://discord.gg/tWEmjR66cy) +- [GitHub Issues](https://github.com/tidesdb/tidesdb-python/issues) +- [Documentation](https://tidesdb.com/reference/python/) diff --git a/creadme.md b/creadme.md deleted file mode 100644 index 97ff8bd..0000000 --- a/creadme.md +++ /dev/null @@ -1,789 +0,0 @@ -
-

-
- -TidesDB is a fast and efficient key value storage engine library written in C. -The underlying data structure is based on a log-structured merge-tree (LSM-tree). - -It is not a full-featured database, but rather a library that can be used to build a database atop of or used as a standalone key-value/column store. - -[![Linux Build Status](https://github.com/tidesdb/tidesdb/actions/workflows/build_and_test_tidesdb.yml/badge.svg)](https://github.com/tidesdb/tidesdb/actions/workflows/build_and_test_tidesdb.yml) - -## Features -- [x] **ACID Transactions** - Atomic, consistent, isolated, and durable. Transactions support multiple operations across column families. -- [x] **Optimized Concurrency** - Writers don't block readers, readers don't block readers. Column families use reader-writer locks allowing multiple concurrent readers. Only writers block other writers on the same column family. -- [x] **Column Families** - Isolated key-value stores. Each column family has its own memtable, SSTables, and WAL. -- [x] **Atomic Transactions** - Commit or rollback multiple operations atomically. Failed transactions automatically rollback. -- [x] **Bidirectional Iterators** - Iterate forward and backward over key-value pairs with merge-sort across memtable and SSTables. -- [x] **Write-Ahead Log (WAL)** - Durability through WAL. Automatic recovery on startup reconstructs memtable from WAL. -- [x] **Background Compaction** - Automatic background compaction when SSTable count reaches threshold. Configurable compaction interval and capacity. -- [x] **Bloom Filters** - Reduce disk reads by checking key existence before reading SSTables. Configurable false positive rate. -- [x] **Compression** - Snappy, LZ4, or ZSTD compression for SSTables and WAL entries. Configurable per column family. -- [x] **TTL Support** - Time-to-live for key-value pairs. Expired entries automatically skipped during reads. -- [x] **Custom Comparators** - Register custom key comparison functions. Built-in comparators `memcmp, string, numeric`. -- [x] **Sync Modes** - Three sync modes `NONE (fastest), BACKGROUND (balanced), FULL (most durable)`. -- [x] **Configurable** - Per-column-family configuration `memtable size, compaction settings, compression, bloom filters, sync mode`. -- [x] **Simple API** - Clean, easy-to-use C API. Returns 0 on success, -1 on error. -- [x] **Skip List Memtable** - Lock-free skip list for in-memory storage with configurable max level and probability. -- [x] **Cross-Platform** - Linux, macOS, and Windows support with platform abstraction layer. -- [x] **Sorted Binary Hash Array (SBHA)** - Fast SSTable lookups. Direct key-to-block offset mapping without full SSTable scans. -- [x] **Tombstones** - Efficient deletion through tombstone markers. Removed during compaction. -- [x] **Streamlined Serialization** - Compact binary format with versioning and bit-packed flags. -- [x] **LRU File Handle Cache** - Configurable LRU cache for open file handles. Limits system resources while maintaining performance. Set `max_open_file_handles` to control cache size (0 = disabled). - -## Building -Using cmake to build the shared library. - -### Unix (Linux/macOS) -```bash -rm -rf build && cmake -S . -B build -cmake --build build -cmake --install build - -# Production build -rm -rf build && cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DTIDESDB_WITH_SANITIZER=OFF -DTIDESDB_BUILD_TESTS=OFF -cmake --build build --config Release -sudo cmake --install build - -# On linux run ldconfig to update the shared library cache -ldconfig -``` - -### Windows - -#### Option 1 MinGW-w64 (Recommended for Windows) -MinGW-w64 provides a GCC-based toolchain with better C11 support and POSIX compatibility. - -**Prerequisites** -- Install [MinGW-w64](https://www.mingw-w64.org/) -- Install [CMake](https://cmake.org/download/) -- Install [vcpkg](https://vcpkg.io/en/getting-started.html) for dependencies - -**Build Steps** -```powershell -# Clean previous build -Remove-Item -Recurse -Force build -ErrorAction SilentlyContinue - -# Configure with MinGW -cmake -S . -B build -G "MinGW Makefiles" -DCMAKE_C_COMPILER=gcc -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake - -# Build -cmake --build build - -# Run tests -cd build -ctest --verbose # or use --output-on-failure to only show failures -``` - -#### Option 2 MSVC (Visual Studio) -**Prerequisites** -- Install [Visual Studio 2019 or later](https://visualstudio.microsoft.com/) with C++ development tools -- Install [CMake](https://cmake.org/download/) -- Install [vcpkg](https://vcpkg.io/en/getting-started.html) for dependencies - -**Build Steps** -```powershell -# Clean previous build -Remove-Item -Recurse -Force build -ErrorAction SilentlyContinue - -# Configure with MSVC -cmake -S . -B build -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake - -# Build (Debug or Release) -cmake --build build --config Debug -# or -cmake --build build --config Release - -# Run tests -cd build -ctest -C Debug --verbose -# or -ctest -C Release --verbose - -``` - -**Note** MSVC requires Visual Studio 2019 16.8 or later for C11 atomics support (`/experimental:c11atomics`). Both Debug and Release builds are fully supported. - -## Requirements -You need cmake and a C compiler. -You also require the `snappy`, `lz4`, `zstd`, and `openssl` libraries. - -### Dependencies -- [Snappy](https://github.com/google/snappy) - Compression -- [LZ4](https://github.com/lz4/lz4) - Compression -- [Zstandard](https://github.com/facebook/zstd) - Compression -- [OpenSSL](https://www.openssl.org/) - Cryptographic hashing (SHA1) - -### Linux -```bash -sudo apt install libzstd-dev -sudo apt install liblz4-dev -sudo apt install libsnappy-dev -sudo apt install libssl-dev -``` - -### MacOS -```bash -brew install zstd -brew install lz4 -brew install snappy -brew install openssl -``` - -### Windows -Windows using vcpkg -```bash -vcpkg install zstd -vcpkg install lz4 -vcpkg install snappy -vcpkg install openssl -``` - -## Bindings -Bindings are in the works in various languages. - - -## Discord Community -Join the [TidesDB Discord Community](https://discord.gg/tWEmjR66cy) to ask questions, work on development, and discuss the future of TidesDB. - -## Include -```c -#include /* You can use other components of TidesDB such as skip list, bloom filter etc.. under tidesdb/ - this also prevents collisions. */ -``` - -## Error Codes -TidesDB provides detailed error codes for production use. All functions return `0` on success or a negative error code on failure. - -| Error Code | Value | Description | -|------------|-------|-------------| -| `TDB_SUCCESS` | 0 | operation successful | -| `TDB_ERROR` | -1 | generic error | -| `TDB_ERR_MEMORY` | -2 | memory allocation failed | -| `TDB_ERR_INVALID_ARGS` | -3 | invalid arguments passed to function | -| `TDB_ERR_IO` | -4 | I/O error (file operations) | -| `TDB_ERR_NOT_FOUND` | -5 | key not found | -| `TDB_ERR_EXISTS` | -6 | resource already exists | -| `TDB_ERR_CORRUPT` | -7 | data corruption detected | -| `TDB_ERR_LOCK` | -8 | lock acquisition failed | -| `TDB_ERR_TXN_COMMITTED` | -9 | transaction already committed | -| `TDB_ERR_TXN_ABORTED` | -10 | transaction aborted | -| `TDB_ERR_READONLY` | -11 | write operation on read-only transaction | -| `TDB_ERR_FULL` | -12 | database or resource full | -| `TDB_ERR_INVALID_NAME` | -13 | invalid name (too long or empty) | -| `TDB_ERR_COMPARATOR_NOT_FOUND` | -14 | comparator not found in registry | -| `TDB_ERR_MAX_COMPARATORS` | -15 | maximum number of comparators reached | -| `TDB_ERR_INVALID_CF` | -16 | invalid column family | -| `TDB_ERR_THREAD` | -17 | thread creation or operation failed | -| `TDB_ERR_CHECKSUM` | -18 | checksum verification failed | - -**Example error handling** -```c -int result = tidesdb_txn_put(txn, "my_cf", key, key_size, value, value_size, -1); -if (result != TDB_SUCCESS) -{ - switch (result) - { - case TDB_ERR_MEMORY: - fprintf(stderr, "out of memory\n"); - break; - case TDB_ERR_INVALID_ARGS: - fprintf(stderr, "invalid arguments\n"); - break; - case TDB_ERR_READONLY: - fprintf(stderr, "cannot write to read-only transaction\n"); - break; - default: - fprintf(stderr, "operation failed with error code: %d\n", result); - break; - } - return -1; -} -``` - -## Usage -TidesDB v1 uses a simplified API. All functions return `0` on success and a negative error code on failure. - -### Opening a database -To open a database you pass a config struct and a pointer to the database. -```c -tidesdb_config_t config = { - .db_path = "./mydb", - .enable_debug_logging = 0 /* Optional enable debug logging */ -}; - -tidesdb_t *db = NULL; -if (tidesdb_open(&config, &db) != 0) -{ - /* Handle error */ - return -1; -} - -/* Close the database */ -if (tidesdb_close(db) != 0) -{ - /* Handle error */ - return -1; -} -``` - -### Debug Logging -TidesDB provides runtime debug logging that can be enabled/disabled dynamically. - -**Enable at startup** -```c -tidesdb_config_t config = { - .db_path = "./mydb", - .enable_debug_logging = 1 /* Enable debug logging */ -}; - -tidesdb_t *db = NULL; -tidesdb_open(&config, &db); -``` - -**Enable/disable at runtime** -```c -extern int _tidesdb_debug_enabled; /* Global debug flag */ - -/* Enable debug logging */ -_tidesdb_debug_enabled = 1; - -/* Your operations here - debug logs will be written to stderr */ - -/* Disable debug logging */ -_tidesdb_debug_enabled = 0; -``` - -**Output** -Debug logs are written to **stderr** with the format -``` -[TidesDB DEBUG] filename:line: message -``` - -**Redirect to file** -```bash -./your_program 2> tidesdb_debug.log # Redirect stderr to file -``` - -### Creating a column family -Column families are isolated key-value stores. Use the config struct for customization or use defaults. - -```c -/* Create with default configuration */ -tidesdb_column_family_config_t cf_config = tidesdb_default_column_family_config(); - -if (tidesdb_create_column_family(db, "my_cf", &cf_config) != 0) -{ - /* Handle error */ - return -1; -} -``` - -**Custom configuration example** -```c -tidesdb_column_family_config_t cf_config = { - .memtable_flush_size = 128 * 1024 * 1024, /* 128MB */ - .max_sstables_before_compaction = 512, /* trigger compaction at 512 SSTables (min 2 required) */ - .compaction_threads = 4, /* use 4 threads for parallel compaction (0 = single-threaded) */ - .max_level = 12, /* skip list max level */ - .probability = 0.25f, /* skip list probability */ - .compressed = 1, /* enable compression */ - .compress_algo = COMPRESS_LZ4, /* use LZ4 */ - .bloom_filter_fp_rate = 0.01, /* 1% false positive rate */ - .enable_background_compaction = 1, /* enable background compaction */ - .background_compaction_interval = 1000000, /* check every 1000000 microseconds (1 second) */ - .use_sbha = 1, /* use sorted binary hash array */ - .sync_mode = TDB_SYNC_BACKGROUND, /* background fsync */ - .sync_interval = 1000, /* fsync every 1000ms (1 second) */ - .comparator_name = NULL /* NULL = use default "memcmp" */ -}; - -if (tidesdb_create_column_family(db, "my_cf", &cf_config) != 0) -{ - /* Handle error */ - return -1; -} -``` - -**Using custom comparator** -```c -/* Register custom comparator first (see examples/custom_comparator.c) */ -tidesdb_register_comparator("reverse", my_reverse_compare); - -tidesdb_column_family_config_t cf_config = tidesdb_default_column_family_config(); -cf_config.comparator_name = "reverse"; /* use registered comparator */ - -if (tidesdb_create_column_family(db, "sorted_cf", &cf_config) != 0) -{ - /* Handle error */ - return -1; -} -``` - - -### Dropping a column family - -```c -if (tidesdb_drop_column_family(db, "my_cf") != 0) -{ - /* Handle error */ - return -1; -} -``` - -### Getting a column family -Retrieve a column family pointer to use in operations. -```c -tidesdb_column_family_t *cf = tidesdb_get_column_family(db, "my_cf"); -if (cf == NULL) -{ - /* Column family not found */ - return -1; -} -``` - -### Listing column families -Get all column family names in the database. -```c -char **names = NULL; -int count = 0; - -if (tidesdb_list_column_families(db, &names, &count) == 0) -{ - printf("Found %d column families:\n", count); - for (int i = 0; i < count; i++) - { - printf(" - %s\n", names[i]); - free(names[i]); /* Free each name */ - } - free(names); /* Free the array */ -} -``` - -### Column family statistics -Get detailed statistics about a column family. -```c -tidesdb_column_family_stat_t *stats = NULL; - -if (tidesdb_get_column_family_stats(db, "my_cf", &stats) == 0) -{ - printf("Column Family: %s\n", stats->name); - printf("Comparator: %s\n", stats->comparator_name); - printf("SSTables: %d\n", stats->num_sstables); - printf("Total SSTable Size: %zu bytes\n", stats->total_sstable_size); - printf("Memtable Size: %zu bytes\n", stats->memtable_size); - printf("Memtable Entries: %d\n", stats->memtable_entries); - printf("Compression: %s\n", stats->config.compressed ? "enabled" : "disabled"); - printf("Bloom Filter FP Rate: %.4f\n", stats->config.bloom_filter_fp_rate); - - free(stats); -} -``` - -**Statistics include** -- Column family name and comparator -- Number of SSTables and total size -- Memtable size and entry count -- Full configuration (compression, bloom filters, sync mode, etc.) - -### Transactions -All operations in TidesDB v1 are done through transactions for ACID guarantees. - -**Basic transaction** -```c -tidesdb_txn_t *txn = NULL; -if (tidesdb_txn_begin(db, &txn) != 0) -{ - return -1; -} - -/* Put a key-value pair */ -const uint8_t *key = (uint8_t *)"mykey"; -const uint8_t *value = (uint8_t *)"myvalue"; - -if (tidesdb_txn_put(txn, "my_cf", key, 5, value, 7, -1) != 0) -{ - tidesdb_txn_free(txn); - return -1; -} - -/* Commit the transaction */ -if (tidesdb_txn_commit(txn) != 0) -{ - tidesdb_txn_free(txn); - return -1; -} - -tidesdb_txn_free(txn); -``` - -**With TTL (time-to-live)** -```c -tidesdb_txn_t *txn = NULL; -tidesdb_txn_begin(db, &txn); - -const uint8_t *key = (uint8_t *)"temp_key"; -const uint8_t *value = (uint8_t *)"temp_value"; - -/* TTL is Unix timestamp (seconds since epoch) - absolute expiration time */ -time_t ttl = time(NULL) + 60; /* Expires 60 seconds from now */ - -/* Use -1 for no expiration */ -tidesdb_txn_put(txn, "my_cf", key, 8, value, 10, ttl); -tidesdb_txn_commit(txn); -tidesdb_txn_free(txn); -``` - -**TTL Examples** -```c -/* No expiration */ -time_t ttl = -1; - -/* Expire in 5 minutes */ -time_t ttl = time(NULL) + (5 * 60); - -/* Expire in 1 hour */ -time_t ttl = time(NULL) + (60 * 60); - -/* Expire at specific time (e.g., midnight) */ -time_t ttl = 1730592000; /* Specific Unix timestamp */ -``` - -**Getting a key-value pair** -```c -tidesdb_txn_t *txn = NULL; -tidesdb_txn_begin_read(db, &txn); /* Read-only transaction */ - -const uint8_t *key = (uint8_t *)"mykey"; -uint8_t *value = NULL; -size_t value_size = 0; - -if (tidesdb_txn_get(txn, "my_cf", key, 5, &value, &value_size) == 0) -{ - /* Use value */ - printf("Value: %.*s\n", (int)value_size, value); - free(value); -} - -tidesdb_txn_free(txn); -``` - -**Deleting a key-value pair** -```c -tidesdb_txn_t *txn = NULL; -tidesdb_txn_begin(db, &txn); - -const uint8_t *key = (uint8_t *)"mykey"; -tidesdb_txn_delete(txn, "my_cf", key, 5); - -tidesdb_txn_commit(txn); -tidesdb_txn_free(txn); -``` - -**Multi-operation transaction** -```c -tidesdb_txn_t *txn = NULL; -tidesdb_txn_begin(db, &txn); - -/* Multiple operations in one transaction */ -tidesdb_txn_put(txn, "my_cf", (uint8_t *)"key1", 4, (uint8_t *)"value1", 6, -1); -tidesdb_txn_put(txn, "my_cf", (uint8_t *)"key2", 4, (uint8_t *)"value2", 6, -1); -tidesdb_txn_delete(txn, "my_cf", (uint8_t *)"old_key", 7); - -/* Commit atomically - all or nothing */ -if (tidesdb_txn_commit(txn) != 0) -{ - /* On error, transaction is automatically rolled back */ - tidesdb_txn_free(txn); - return -1; -} - -tidesdb_txn_free(txn); -``` - -**Transaction rollback** -```c -tidesdb_txn_t *txn = NULL; -tidesdb_txn_begin(db, &txn); - -tidesdb_txn_put(txn, "my_cf", (uint8_t *)"key", 3, (uint8_t *)"value", 5, -1); - -/* Decide to rollback instead of commit */ -tidesdb_txn_rollback(txn); -tidesdb_txn_free(txn); -/* No changes were applied */ -``` - -### Iterators -Iterators provide efficient forward and backward traversal over key-value pairs. - -**Forward iteration** -```c -tidesdb_txn_t *txn = NULL; -tidesdb_txn_begin_read(db, &txn); - -tidesdb_iter_t *iter = NULL; -if (tidesdb_iter_new(txn, "my_cf", &iter) != 0) -{ - tidesdb_txn_free(txn); - return -1; -} - -/* Seek to first entry */ -tidesdb_iter_seek_to_first(iter); - -while (tidesdb_iter_valid(iter)) -{ - uint8_t *key = NULL; - size_t key_size = 0; - uint8_t *value = NULL; - size_t value_size = 0; - - if (tidesdb_iter_key(iter, &key, &key_size) == 0 && - tidesdb_iter_value(iter, &value, &value_size) == 0) - { - /* Use key and value */ - printf("Key: %.*s, Value: %.*s\n", - (int)key_size, key, (int)value_size, value); - free(key); - free(value); - } - - tidesdb_iter_next(iter); -} - -tidesdb_iter_free(iter); -tidesdb_txn_free(txn); -``` - -**Backward iteration** -```c -tidesdb_txn_t *txn = NULL; -tidesdb_txn_begin_read(db, &txn); - -tidesdb_iter_t *iter = NULL; -tidesdb_iter_new(txn, "my_cf", &iter); - -/* Seek to last entry */ -tidesdb_iter_seek_to_last(iter); - -while (tidesdb_iter_valid(iter)) -{ - /* Process entries in reverse order */ - tidesdb_iter_prev(iter); -} - -tidesdb_iter_free(iter); -tidesdb_txn_free(txn); -``` - -**Iterator Compaction Resilience** - -TidesDB iterators automatically handle compaction that occurs during iteration: - -- **Automatic Snapshot Refresh** - When an iterator detects that compaction has occurred (SSTable count changed), it automatically refreshes its internal snapshot with the new compacted SSTables -- **Seamless Continuation** - The iterator continues traversing data from the new SSTables without requiring manual intervention -- **No Blocking** - Compaction doesn't wait for iterators, and iterators don't block compaction -- **Read Lock Protection** - Each iteration operation holds a read lock, preventing SSTables from being freed during access - -```c -tidesdb_iter_t *iter = NULL; -tidesdb_iter_new(txn, "my_cf", &iter); -tidesdb_iter_seek_to_first(iter); - -while (tidesdb_iter_valid(iter)) -{ - /* If compaction occurs here, iterator automatically refreshes */ - /* and continues reading from the new compacted SSTables */ - - uint8_t *key = NULL, *value = NULL; - size_t key_size = 0, value_size = 0; - - tidesdb_iter_key(iter, &key, &key_size); - tidesdb_iter_value(iter, &value, &value_size); - - /* Process data */ - - free(key); - free(value); - tidesdb_iter_next(iter); /* Detects and handles compaction if it occurred */ -} - -tidesdb_iter_free(iter); -``` - -This design ensures iterators remain valid and functional even during active compaction, providing a robust and concurrent iteration experience. - -### Custom Comparators -Register custom key comparison functions for specialized sorting. - -**Register a comparator** -```c -/* Define your comparison function */ -int my_reverse_compare(const uint8_t *key1, size_t key1_size, - const uint8_t *key2, size_t key2_size, void *ctx) -{ - int result = memcmp(key1, key2, key1_size < key2_size ? key1_size : key2_size); - return -result; /* reverse order */ -} - -/* Register it before creating column families */ -tidesdb_register_comparator("reverse", my_reverse_compare); - -/* Use in column family */ -tidesdb_column_family_config_t cf_config = tidesdb_default_column_family_config(); -cf_config.comparator_name = "reverse"; -tidesdb_create_column_family(db, "sorted_cf", &cf_config); -``` - -**Built-in comparators** -- `"memcmp"` - Binary comparison (default) -- `"string"` - Lexicographic string comparison -- `"numeric"` - Numeric comparison for uint64_t keys - -See `examples/custom_comparator.c` for more examples. - -### Sync Modes -Control durability vs performance tradeoff. - -```c -tidesdb_column_family_config_t cf_config = tidesdb_default_column_family_config(); - -/* TDB_SYNC_NONE - Fastest, least durable (OS handles flushing) */ -cf_config.sync_mode = TDB_SYNC_NONE; - -/* TDB_SYNC_BACKGROUND - Balanced (fsync every N milliseconds in background) */ -cf_config.sync_mode = TDB_SYNC_BACKGROUND; -cf_config.sync_interval = 1000; /* fsync every 1000ms (1 second) */ - -/* TDB_SYNC_FULL - Most durable (fsync on every write) */ -cf_config.sync_mode = TDB_SYNC_FULL; - -tidesdb_create_column_family(db, "my_cf", &cf_config); -``` - -## Background Compaction -TidesDB v1 features automatic background compaction with optional parallel execution. - -**Automatic background compaction** runs when SSTable count reaches the configured threshold - -```c -tidesdb_column_family_config_t cf_config = tidesdb_default_column_family_config(); -cf_config.enable_background_compaction = 1; /* Enable background compaction */ -cf_config.background_compaction_interval = TDB_DEFAULT_BACKGROUND_COMPACTION_INTERVAL; /* Check every 1000000 microseconds (1 second) */ -cf_config.max_sstables_before_compaction = TDB_DEFAULT_MAX_SSTABLES; /* Trigger at 128 SSTables (default) */ -cf_config.compaction_threads = TDB_DEFAULT_COMPACTION_THREADS; /* Use 4 threads for parallel compaction */ - -tidesdb_create_column_family(db, "my_cf", &cf_config); -/* Background thread automatically compacts when threshold is reached */ -``` - -**Configuration Options** -- `enable_background_compaction` - Enable/disable automatic background compaction (default: enabled) -- `background_compaction_interval` - Interval in microseconds between compaction checks (default: 1000000 = 1 second) -- `max_sstables_before_compaction` - SSTable count threshold to trigger compaction (default: 512, minimum: 2) -- `compaction_threads` - Number of threads for parallel compaction (default: 4, set to 0 for single-threaded) - -**Parallel Compaction** -- Set `compaction_threads > 0` to enable parallel compaction -- Uses semaphore-based thread pool for concurrent SSTable pair merging -- Each thread compacts one pair of SSTables independently -- Automatically limits threads to available CPU cores -- Set `compaction_threads = 0` for single-threaded compaction (default 4 threads) - -**Manual compaction** can be triggered at any time (requires minimum 2 SSTables) - -```c -tidesdb_compact(cf); /* Automatically uses parallel compaction if compaction_threads > 0 */ -``` - -**Benefits** -- Removes tombstones and expired TTL entries -- Merges duplicate keys (keeps latest version) -- Reduces SSTable count -- Background compaction runs in separate thread (non-blocking) -- Parallel compaction significantly speeds up large compactions -- Manual compaction requires minimum 2 SSTables to merge - -## Concurrency Model - -TidesDB is designed for high concurrency with minimal blocking - -**Reader-Writer Locks** -- Each column family has a reader-writer lock -- **Multiple readers can read concurrently** - no blocking between readers -- **Writers don't block readers** - readers can access data while writes are in progress -- **Writers block other writers** - only one writer per column family at a time - -**Transaction Isolation** -- Read transactions (`tidesdb_txn_begin_read`) acquire read locks -- Write transactions (`tidesdb_txn_begin`) acquire write locks on commit -- Transactions are isolated - changes not visible until commit - -**Optimal for** -- Read-heavy workloads (unlimited concurrent readers) -- Mixed read/write workloads (readers never wait for writers) -- Multi-column-family applications (different CFs can be written concurrently) - -**Example - Concurrent Operations** -```c -/* Thread 1 Reading */ -tidesdb_txn_t *read_txn; -tidesdb_txn_begin_read(db, &read_txn); -tidesdb_txn_get(read_txn, "my_cf", key, key_size, &value, &value_size); -/* Can read while Thread 2 is writing */ - -/* Thread 2 Writing */ -tidesdb_txn_t *write_txn; -tidesdb_txn_begin(db, &write_txn); -tidesdb_txn_put(write_txn, "my_cf", key, key_size, value, value_size, -1); -tidesdb_txn_commit(write_txn); /* Briefly blocks other writers only */ - -/* Thread 3 Reading different CF */ -tidesdb_txn_t *other_txn; -tidesdb_txn_begin_read(db, &other_txn); -tidesdb_txn_get(other_txn, "other_cf", key, key_size, &value, &value_size); -/* No blocking different column family */ -``` - -## License -Multiple - -``` -Mozilla Public License Version 2.0 (TidesDB) - --- AND -- -BSD 3 Clause (Snappy) -BSD 2 (LZ4) -BSD 2 (xxHash - Yann Collet) -BSD (Zstandard) -Apache 2.0 (OpenSSL 3.0+) / OpenSSL License (OpenSSL 1.x) -``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 76c8321..6fcf685 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,19 @@ [build-system] -requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"] +requires = ["setuptools>=61.0", "wheel"] build-backend = "setuptools.build_meta" [project] name = "tidesdb" -version = "1.0.0" -description = "Official Python bindings for TidesDB v1+" -readme = "README_PYTHON.md" -requires-python = ">=3.7" +version = "0.5.0" +description = "Official Python bindings for TidesDB - A high-performance embedded key-value storage engine" +readme = "README.md" +requires-python = ">=3.10" license = {text = "MPL-2.0"} authors = [ - {name = "TidesDB Team", email = "support@tidesdb.com"} + {name = "TidesDB Lead/Creator", email = "me@alexpadula.com"} ] maintainers = [ - {name = "TidesDB Team", email = "support@tidesdb.com"} + {name = "TidesDB Lead/Creator", email = "me@alexpadula.com"} ] keywords = [ "database", @@ -21,7 +21,9 @@ keywords = [ "lsm-tree", "embedded", "storage-engine", - "tidesdb" + "tidesdb", + "mvcc", + "transactions" ] classifiers = [ "Development Status :: 4 - Beta", @@ -29,20 +31,19 @@ classifiers = [ "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", "Operating System :: OS Independent", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Database", + "Topic :: Database :: Database Engines/Servers", "Topic :: Software Development :: Libraries :: Python Modules", "Typing :: Typed" ] [project.urls] -Homepage = "https://github.com/tidesdb/tidesdb-python" -Documentation = "https://github.com/tidesdb/tidesdb" +Homepage = "https://tidesdb.com" +Documentation = "https://tidesdb.com/reference/python/" Repository = "https://github.com/tidesdb/tidesdb-python" "Bug Tracker" = "https://github.com/tidesdb/tidesdb-python/issues" Discord = "https://discord.gg/tWEmjR66cy" @@ -52,18 +53,24 @@ dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", "black>=23.0.0", - "flake8>=6.0.0", + "ruff>=0.1.0", "mypy>=1.0.0", ] +[tool.setuptools.packages.find] +where = ["src"] + [tool.black] line-length = 100 -target-version = ['py37', 'py38', 'py39', 'py310', 'py311', 'py312', 'py313'] +target-version = ['py310', 'py311', 'py312', 'py313'] include = '\.pyi?$' -[tool.isort] -profile = "black" -line_length = 100 +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "N", "UP", "B", "C4"] [tool.pytest.ini_options] testpaths = ["tests"] @@ -73,7 +80,7 @@ python_functions = "test_*" addopts = "-v --cov=tidesdb --cov-report=html --cov-report=term" [tool.mypy] -python_version = "3.7" +python_version = "3.10" warn_return_any = true warn_unused_configs = true -disallow_untyped_defs = true \ No newline at end of file +disallow_untyped_defs = true diff --git a/setup.py b/setup.py deleted file mode 100644 index 89970a3..0000000 --- a/setup.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -TidesDB Python Bindings Setup -""" - -from setuptools import setup, find_packages -from pathlib import Path - -# Read the README file -this_directory = Path(__file__).parent -long_description = (this_directory / "README.md").read_text() if (this_directory / "README.md").exists() else "" - -setup( - name="tidesdb", - version="1.0.0", - author="TidesDB Authors", - author_email="me@alexpadula.com", - description="Official Python bindings for TidesDB v1+", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/tidesdb/tidesdb-python", - project_urls={ - "Bug Tracker": "https://github.com/tidesdb/tidesdb-python/issues", - "Documentation": "https://github.com/tidesdb/tidesdb", - "Source Code": "https://github.com/tidesdb/tidesdb-python", - "Discord": "https://discord.gg/tWEmjR66cy", - }, - packages=find_packages(), - py_modules=["tidesdb"], - classifiers=[ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: Database", - "Topic :: Software Development :: Libraries :: Python Modules", - ], - python_requires=">=3.7", - install_requires=[ - # No external dependencies - uses ctypes with system library - ], - extras_require={ - "dev": [ - "pytest>=7.0.0", - "pytest-cov>=4.0.0", - "black>=23.0.0", - "flake8>=6.0.0", - "mypy>=1.0.0", - ], - }, - keywords="database, key-value, lsm-tree, embedded, storage-engine", - license="MPL-2.0", -) \ No newline at end of file diff --git a/src/tidesdb/__init__.py b/src/tidesdb/__init__.py new file mode 100644 index 0000000..fa3c074 --- /dev/null +++ b/src/tidesdb/__init__.py @@ -0,0 +1,45 @@ +""" +TidesDB Python Bindings + +Official Python bindings for TidesDB v7+ - A high-performance embedded key-value storage engine. + +Copyright (C) TidesDB +Licensed under the Mozilla Public License, v. 2.0 +""" + +from .tidesdb import ( + TidesDB, + Transaction, + Iterator, + ColumnFamily, + Config, + ColumnFamilyConfig, + Stats, + CacheStats, + CompressionAlgorithm, + SyncMode, + LogLevel, + IsolationLevel, + TidesDBError, + default_config, + default_column_family_config, +) + +__version__ = "7.3.1" +__all__ = [ + "TidesDB", + "Transaction", + "Iterator", + "ColumnFamily", + "Config", + "ColumnFamilyConfig", + "Stats", + "CacheStats", + "CompressionAlgorithm", + "SyncMode", + "LogLevel", + "IsolationLevel", + "TidesDBError", + "default_config", + "default_column_family_config", +] diff --git a/src/tidesdb/tidesdb.py b/src/tidesdb/tidesdb.py new file mode 100644 index 0000000..e922b6a --- /dev/null +++ b/src/tidesdb/tidesdb.py @@ -0,0 +1,1143 @@ +""" +TidesDB Python Bindings v7+ + +Copyright (C) TidesDB +Original Author: Alex Gaetano Padula + +Licensed under the Mozilla Public License, v. 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.mozilla.org/en-US/MPL/2.0/ + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +from __future__ import annotations + +import ctypes +import os +import sys +from ctypes import ( + POINTER, + Structure, + c_char, + c_char_p, + c_double, + c_float, + c_int, + c_size_t, + c_uint8, + c_uint64, + c_void_p, +) +from dataclasses import dataclass +from enum import IntEnum +from typing import Iterator as TypingIterator + + +def _load_library() -> ctypes.CDLL: + """Load the TidesDB shared library.""" + if sys.platform == "win32": + lib_names = ["tidesdb.dll", "libtidesdb.dll"] + elif sys.platform == "darwin": + lib_names = ["libtidesdb.dylib", "libtidesdb.so"] + else: + lib_names = ["libtidesdb.so", "libtidesdb.so.1"] + + search_paths = [ + "", + "/usr/local/lib/", + "/usr/lib/", + "/opt/homebrew/lib/", + "/mingw64/lib/", + ] + + for path in search_paths: + for lib_name in lib_names: + try: + return ctypes.CDLL(path + lib_name) + except OSError: + continue + + raise RuntimeError( + "Could not load TidesDB library. " + "Please ensure libtidesdb is installed and in your library path. " + "On Linux: /usr/local/lib or set LD_LIBRARY_PATH. " + "On macOS: /usr/local/lib or /opt/homebrew/lib or set DYLD_LIBRARY_PATH. " + "On Windows: ensure tidesdb.dll is in PATH or current directory." + ) + + +_lib = _load_library() + + +TDB_MAX_COMPARATOR_NAME = 64 +TDB_MAX_COMPARATOR_CTX = 256 + +TDB_SUCCESS = 0 +TDB_ERR_MEMORY = -1 +TDB_ERR_INVALID_ARGS = -2 +TDB_ERR_NOT_FOUND = -3 +TDB_ERR_IO = -4 +TDB_ERR_CORRUPTION = -5 +TDB_ERR_EXISTS = -6 +TDB_ERR_CONFLICT = -7 +TDB_ERR_TOO_LARGE = -8 +TDB_ERR_MEMORY_LIMIT = -9 +TDB_ERR_INVALID_DB = -10 +TDB_ERR_UNKNOWN = -11 +TDB_ERR_LOCKED = -12 + + +class CompressionAlgorithm(IntEnum): + """Compression algorithm types.""" + + NO_COMPRESSION = 0 + SNAPPY_COMPRESSION = 1 + LZ4_COMPRESSION = 2 + ZSTD_COMPRESSION = 3 + LZ4_FAST_COMPRESSION = 4 + + +class SyncMode(IntEnum): + """Sync modes for durability.""" + + SYNC_NONE = 0 + SYNC_FULL = 1 + SYNC_INTERVAL = 2 + + +class LogLevel(IntEnum): + """Logging levels.""" + + LOG_DEBUG = 0 + LOG_INFO = 1 + LOG_WARN = 2 + LOG_ERROR = 3 + LOG_FATAL = 4 + LOG_NONE = 99 + + +class IsolationLevel(IntEnum): + """Transaction isolation levels.""" + + READ_UNCOMMITTED = 0 + READ_COMMITTED = 1 + REPEATABLE_READ = 2 + SNAPSHOT = 3 + SERIALIZABLE = 4 + + +class TidesDBError(Exception): + """Base exception for TidesDB errors.""" + + def __init__(self, message: str, code: int = TDB_ERR_UNKNOWN): + super().__init__(message) + self.code = code + + @classmethod + def from_code(cls, code: int, context: str = "") -> TidesDBError: + """Create exception from error code.""" + error_messages = { + TDB_ERR_MEMORY: "memory allocation failed", + TDB_ERR_INVALID_ARGS: "invalid arguments", + TDB_ERR_NOT_FOUND: "not found", + TDB_ERR_IO: "I/O error", + TDB_ERR_CORRUPTION: "data corruption", + TDB_ERR_EXISTS: "already exists", + TDB_ERR_CONFLICT: "transaction conflict", + TDB_ERR_TOO_LARGE: "key or value too large", + TDB_ERR_MEMORY_LIMIT: "memory limit exceeded", + TDB_ERR_INVALID_DB: "invalid database handle", + TDB_ERR_UNKNOWN: "unknown error", + TDB_ERR_LOCKED: "database is locked", + } + + msg = error_messages.get(code, "unknown error") + if context: + msg = f"{context}: {msg} (code: {code})" + else: + msg = f"{msg} (code: {code})" + + return cls(msg, code) + + +class _CColumnFamilyConfig(Structure): + """C structure for tidesdb_column_family_config_t.""" + + _fields_ = [ + ("write_buffer_size", c_size_t), + ("level_size_ratio", c_size_t), + ("min_levels", c_int), + ("dividing_level_offset", c_int), + ("klog_value_threshold", c_size_t), + ("compression_algo", c_int), + ("enable_bloom_filter", c_int), + ("bloom_fpr", c_double), + ("enable_block_indexes", c_int), + ("index_sample_ratio", c_int), + ("block_index_prefix_len", c_int), + ("sync_mode", c_int), + ("sync_interval_us", c_uint64), + ("comparator_name", c_char * TDB_MAX_COMPARATOR_NAME), + ("comparator_ctx_str", c_char * TDB_MAX_COMPARATOR_CTX), + ("comparator_fn_cached", c_void_p), + ("comparator_ctx_cached", c_void_p), + ("skip_list_max_level", c_int), + ("skip_list_probability", c_float), + ("default_isolation_level", c_int), + ("min_disk_space", c_uint64), + ("l1_file_count_trigger", c_int), + ("l0_queue_stall_threshold", c_int), + ] + + +class _CConfig(Structure): + """C structure for tidesdb_config_t.""" + + _fields_ = [ + ("db_path", c_char_p), + ("num_flush_threads", c_int), + ("num_compaction_threads", c_int), + ("log_level", c_int), + ("block_cache_size", c_size_t), + ("max_open_sstables", c_size_t), + ] + + +class _CStats(Structure): + """C structure for tidesdb_stats_t.""" + + _fields_ = [ + ("num_levels", c_int), + ("memtable_size", c_size_t), + ("level_sizes", POINTER(c_size_t)), + ("level_num_sstables", POINTER(c_int)), + ("config", POINTER(_CColumnFamilyConfig)), + ] + + +class _CCacheStats(Structure): + """C structure for tidesdb_cache_stats_t.""" + + _fields_ = [ + ("enabled", c_int), + ("total_entries", c_size_t), + ("total_bytes", c_size_t), + ("hits", c_uint64), + ("misses", c_uint64), + ("hit_rate", c_double), + ("num_partitions", c_size_t), + ] + + +_lib.tidesdb_default_column_family_config.argtypes = [] +_lib.tidesdb_default_column_family_config.restype = _CColumnFamilyConfig + +_lib.tidesdb_default_config.argtypes = [] +_lib.tidesdb_default_config.restype = _CConfig + +_lib.tidesdb_open.argtypes = [POINTER(_CConfig), POINTER(c_void_p)] +_lib.tidesdb_open.restype = c_int + +_lib.tidesdb_close.argtypes = [c_void_p] +_lib.tidesdb_close.restype = c_int + +_lib.tidesdb_create_column_family.argtypes = [c_void_p, c_char_p, POINTER(_CColumnFamilyConfig)] +_lib.tidesdb_create_column_family.restype = c_int + +_lib.tidesdb_drop_column_family.argtypes = [c_void_p, c_char_p] +_lib.tidesdb_drop_column_family.restype = c_int + +_lib.tidesdb_get_column_family.argtypes = [c_void_p, c_char_p] +_lib.tidesdb_get_column_family.restype = c_void_p + +_lib.tidesdb_list_column_families.argtypes = [c_void_p, POINTER(POINTER(c_char_p)), POINTER(c_int)] +_lib.tidesdb_list_column_families.restype = c_int + +_lib.tidesdb_txn_begin.argtypes = [c_void_p, POINTER(c_void_p)] +_lib.tidesdb_txn_begin.restype = c_int + +_lib.tidesdb_txn_begin_with_isolation.argtypes = [c_void_p, c_int, POINTER(c_void_p)] +_lib.tidesdb_txn_begin_with_isolation.restype = c_int + +_lib.tidesdb_txn_put.argtypes = [ + c_void_p, + c_void_p, + POINTER(c_uint8), + c_size_t, + POINTER(c_uint8), + c_size_t, + c_int, +] +_lib.tidesdb_txn_put.restype = c_int + +_lib.tidesdb_txn_get.argtypes = [ + c_void_p, + c_void_p, + POINTER(c_uint8), + c_size_t, + POINTER(POINTER(c_uint8)), + POINTER(c_size_t), +] +_lib.tidesdb_txn_get.restype = c_int + +_lib.tidesdb_txn_delete.argtypes = [c_void_p, c_void_p, POINTER(c_uint8), c_size_t] +_lib.tidesdb_txn_delete.restype = c_int + +_lib.tidesdb_txn_commit.argtypes = [c_void_p] +_lib.tidesdb_txn_commit.restype = c_int + +_lib.tidesdb_txn_rollback.argtypes = [c_void_p] +_lib.tidesdb_txn_rollback.restype = c_int + +_lib.tidesdb_txn_free.argtypes = [c_void_p] +_lib.tidesdb_txn_free.restype = None + +_lib.tidesdb_txn_savepoint.argtypes = [c_void_p, c_char_p] +_lib.tidesdb_txn_savepoint.restype = c_int + +_lib.tidesdb_txn_rollback_to_savepoint.argtypes = [c_void_p, c_char_p] +_lib.tidesdb_txn_rollback_to_savepoint.restype = c_int + +_lib.tidesdb_txn_release_savepoint.argtypes = [c_void_p, c_char_p] +_lib.tidesdb_txn_release_savepoint.restype = c_int + +_lib.tidesdb_iter_new.argtypes = [c_void_p, c_void_p, POINTER(c_void_p)] +_lib.tidesdb_iter_new.restype = c_int + +_lib.tidesdb_iter_seek_to_first.argtypes = [c_void_p] +_lib.tidesdb_iter_seek_to_first.restype = c_int + +_lib.tidesdb_iter_seek_to_last.argtypes = [c_void_p] +_lib.tidesdb_iter_seek_to_last.restype = c_int + +_lib.tidesdb_iter_seek.argtypes = [c_void_p, POINTER(c_uint8), c_size_t] +_lib.tidesdb_iter_seek.restype = c_int + +_lib.tidesdb_iter_seek_for_prev.argtypes = [c_void_p, POINTER(c_uint8), c_size_t] +_lib.tidesdb_iter_seek_for_prev.restype = c_int + +_lib.tidesdb_iter_valid.argtypes = [c_void_p] +_lib.tidesdb_iter_valid.restype = c_int + +_lib.tidesdb_iter_next.argtypes = [c_void_p] +_lib.tidesdb_iter_next.restype = c_int + +_lib.tidesdb_iter_prev.argtypes = [c_void_p] +_lib.tidesdb_iter_prev.restype = c_int + +_lib.tidesdb_iter_key.argtypes = [c_void_p, POINTER(POINTER(c_uint8)), POINTER(c_size_t)] +_lib.tidesdb_iter_key.restype = c_int + +_lib.tidesdb_iter_value.argtypes = [c_void_p, POINTER(POINTER(c_uint8)), POINTER(c_size_t)] +_lib.tidesdb_iter_value.restype = c_int + +_lib.tidesdb_iter_free.argtypes = [c_void_p] +_lib.tidesdb_iter_free.restype = None + +_lib.tidesdb_compact.argtypes = [c_void_p] +_lib.tidesdb_compact.restype = c_int + +_lib.tidesdb_flush_memtable.argtypes = [c_void_p] +_lib.tidesdb_flush_memtable.restype = c_int + +_lib.tidesdb_get_stats.argtypes = [c_void_p, POINTER(POINTER(_CStats))] +_lib.tidesdb_get_stats.restype = c_int + +_lib.tidesdb_free_stats.argtypes = [POINTER(_CStats)] +_lib.tidesdb_free_stats.restype = None + +_lib.tidesdb_get_cache_stats.argtypes = [c_void_p, POINTER(_CCacheStats)] +_lib.tidesdb_get_cache_stats.restype = c_int + + +@dataclass +class Config: + """Configuration for opening a TidesDB instance.""" + + db_path: str + num_flush_threads: int = 2 + num_compaction_threads: int = 2 + log_level: LogLevel = LogLevel.LOG_INFO + block_cache_size: int = 64 * 1024 * 1024 + max_open_sstables: int = 256 + + +@dataclass +class ColumnFamilyConfig: + """Configuration for a column family.""" + + write_buffer_size: int = 64 * 1024 * 1024 + level_size_ratio: int = 10 + min_levels: int = 5 + dividing_level_offset: int = 2 + klog_value_threshold: int = 512 + compression_algorithm: CompressionAlgorithm = CompressionAlgorithm.LZ4_COMPRESSION + enable_bloom_filter: bool = True + bloom_fpr: float = 0.01 + enable_block_indexes: bool = True + index_sample_ratio: int = 1 + block_index_prefix_len: int = 16 + sync_mode: SyncMode = SyncMode.SYNC_INTERVAL + sync_interval_us: int = 128000 + comparator_name: str = "memcmp" + skip_list_max_level: int = 12 + skip_list_probability: float = 0.25 + default_isolation_level: IsolationLevel = IsolationLevel.READ_COMMITTED + min_disk_space: int = 100 * 1024 * 1024 + l1_file_count_trigger: int = 4 + l0_queue_stall_threshold: int = 20 + + def _to_c_struct(self) -> _CColumnFamilyConfig: + """Convert to C structure.""" + c_config = _CColumnFamilyConfig() + c_config.write_buffer_size = self.write_buffer_size + c_config.level_size_ratio = self.level_size_ratio + c_config.min_levels = self.min_levels + c_config.dividing_level_offset = self.dividing_level_offset + c_config.klog_value_threshold = self.klog_value_threshold + c_config.compression_algo = int(self.compression_algorithm) + c_config.enable_bloom_filter = 1 if self.enable_bloom_filter else 0 + c_config.bloom_fpr = self.bloom_fpr + c_config.enable_block_indexes = 1 if self.enable_block_indexes else 0 + c_config.index_sample_ratio = self.index_sample_ratio + c_config.block_index_prefix_len = self.block_index_prefix_len + c_config.sync_mode = int(self.sync_mode) + c_config.sync_interval_us = self.sync_interval_us + c_config.skip_list_max_level = self.skip_list_max_level + c_config.skip_list_probability = self.skip_list_probability + c_config.default_isolation_level = int(self.default_isolation_level) + c_config.min_disk_space = self.min_disk_space + c_config.l1_file_count_trigger = self.l1_file_count_trigger + c_config.l0_queue_stall_threshold = self.l0_queue_stall_threshold + + name_bytes = self.comparator_name.encode("utf-8")[:TDB_MAX_COMPARATOR_NAME - 1] + name_bytes = name_bytes + b"\x00" * (TDB_MAX_COMPARATOR_NAME - len(name_bytes)) + c_config.comparator_name = name_bytes + + return c_config + + +@dataclass +class Stats: + """Statistics about a column family.""" + + num_levels: int + memtable_size: int + level_sizes: list[int] + level_num_sstables: list[int] + config: ColumnFamilyConfig | None = None + + +@dataclass +class CacheStats: + """Statistics about the block cache.""" + + enabled: bool + total_entries: int + total_bytes: int + hits: int + misses: int + hit_rate: float + num_partitions: int + + +def default_config() -> Config: + """Get default database configuration.""" + return Config(db_path="") + + +def default_column_family_config() -> ColumnFamilyConfig: + """Get default column family configuration from C library.""" + c_config = _lib.tidesdb_default_column_family_config() + return ColumnFamilyConfig( + write_buffer_size=c_config.write_buffer_size, + level_size_ratio=c_config.level_size_ratio, + min_levels=c_config.min_levels, + dividing_level_offset=c_config.dividing_level_offset, + klog_value_threshold=c_config.klog_value_threshold, + compression_algorithm=CompressionAlgorithm(c_config.compression_algo), + enable_bloom_filter=bool(c_config.enable_bloom_filter), + bloom_fpr=c_config.bloom_fpr, + enable_block_indexes=bool(c_config.enable_block_indexes), + index_sample_ratio=c_config.index_sample_ratio, + block_index_prefix_len=c_config.block_index_prefix_len, + sync_mode=SyncMode(c_config.sync_mode), + sync_interval_us=c_config.sync_interval_us, + comparator_name=c_config.comparator_name.decode("utf-8").rstrip("\x00"), + skip_list_max_level=c_config.skip_list_max_level, + skip_list_probability=c_config.skip_list_probability, + default_isolation_level=IsolationLevel(c_config.default_isolation_level), + min_disk_space=c_config.min_disk_space, + l1_file_count_trigger=c_config.l1_file_count_trigger, + l0_queue_stall_threshold=c_config.l0_queue_stall_threshold, + ) + + +class Iterator: + """Iterator for traversing key-value pairs in a column family.""" + + def __init__(self, iter_ptr: c_void_p) -> None: + self._iter = iter_ptr + self._closed = False + + def seek_to_first(self) -> None: + """Position iterator at the first key.""" + if self._closed: + raise TidesDBError("Iterator is closed") + result = _lib.tidesdb_iter_seek_to_first(self._iter) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to seek to first") + + def seek_to_last(self) -> None: + """Position iterator at the last key.""" + if self._closed: + raise TidesDBError("Iterator is closed") + result = _lib.tidesdb_iter_seek_to_last(self._iter) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to seek to last") + + def seek(self, key: bytes) -> None: + """Position iterator at the first key >= target key.""" + if self._closed: + raise TidesDBError("Iterator is closed") + key_buf = (c_uint8 * len(key)).from_buffer_copy(key) if key else None + result = _lib.tidesdb_iter_seek(self._iter, key_buf, len(key)) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to seek") + + def seek_for_prev(self, key: bytes) -> None: + """Position iterator at the last key <= target key.""" + if self._closed: + raise TidesDBError("Iterator is closed") + key_buf = (c_uint8 * len(key)).from_buffer_copy(key) if key else None + result = _lib.tidesdb_iter_seek_for_prev(self._iter, key_buf, len(key)) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to seek for prev") + + def valid(self) -> bool: + """Check if iterator is positioned at a valid entry.""" + if self._closed: + return False + return bool(_lib.tidesdb_iter_valid(self._iter)) + + def next(self) -> None: + """Move iterator to the next entry.""" + if self._closed: + raise TidesDBError("Iterator is closed") + # next() returns NOT_FOUND when reaching the end, which is not an error + _lib.tidesdb_iter_next(self._iter) + + def prev(self) -> None: + """Move iterator to the previous entry.""" + if self._closed: + raise TidesDBError("Iterator is closed") + # prev() returns NOT_FOUND when reaching the beginning, which is not an error + _lib.tidesdb_iter_prev(self._iter) + + def key(self) -> bytes: + """Get the current key.""" + if self._closed: + raise TidesDBError("Iterator is closed") + + key_ptr = POINTER(c_uint8)() + key_size = c_size_t() + + result = _lib.tidesdb_iter_key(self._iter, ctypes.byref(key_ptr), ctypes.byref(key_size)) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to get key") + + return ctypes.string_at(key_ptr, key_size.value) + + def value(self) -> bytes: + """Get the current value.""" + if self._closed: + raise TidesDBError("Iterator is closed") + + value_ptr = POINTER(c_uint8)() + value_size = c_size_t() + + result = _lib.tidesdb_iter_value( + self._iter, ctypes.byref(value_ptr), ctypes.byref(value_size) + ) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to get value") + + return ctypes.string_at(value_ptr, value_size.value) + + def close(self) -> None: + """Free iterator resources.""" + if not self._closed and self._iter: + _lib.tidesdb_iter_free(self._iter) + self._closed = True + + def __enter__(self) -> Iterator: + return self + + def __exit__(self, exc_type: type | None, exc_val: Exception | None, exc_tb: object) -> bool: + self.close() + return False + + def __iter__(self) -> TypingIterator[tuple[bytes, bytes]]: + return self + + def __next__(self) -> tuple[bytes, bytes]: + if not self.valid(): + raise StopIteration + key = self.key() + value = self.value() + self.next() + return key, value + + def __del__(self) -> None: + if _lib is None: + return + self.close() + + +class ColumnFamily: + """Column family handle.""" + + def __init__(self, cf_ptr: c_void_p, name: str) -> None: + self._cf = cf_ptr + self.name = name + + def compact(self) -> None: + """Manually trigger compaction for this column family.""" + result = _lib.tidesdb_compact(self._cf) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to compact column family") + + def flush_memtable(self) -> None: + """Manually trigger memtable flush for this column family.""" + result = _lib.tidesdb_flush_memtable(self._cf) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to flush memtable") + + def get_stats(self) -> Stats: + """Get statistics for this column family.""" + stats_ptr = POINTER(_CStats)() + result = _lib.tidesdb_get_stats(self._cf, ctypes.byref(stats_ptr)) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to get stats") + + c_stats = stats_ptr.contents + + level_sizes = [] + level_num_sstables = [] + + if c_stats.num_levels > 0: + if c_stats.level_sizes: + for i in range(c_stats.num_levels): + level_sizes.append(c_stats.level_sizes[i]) + if c_stats.level_num_sstables: + for i in range(c_stats.num_levels): + level_num_sstables.append(c_stats.level_num_sstables[i]) + + config = None + if c_stats.config: + c_cfg = c_stats.config.contents + config = ColumnFamilyConfig( + write_buffer_size=c_cfg.write_buffer_size, + level_size_ratio=c_cfg.level_size_ratio, + min_levels=c_cfg.min_levels, + dividing_level_offset=c_cfg.dividing_level_offset, + klog_value_threshold=c_cfg.klog_value_threshold, + compression_algorithm=CompressionAlgorithm(c_cfg.compression_algo), + enable_bloom_filter=bool(c_cfg.enable_bloom_filter), + bloom_fpr=c_cfg.bloom_fpr, + enable_block_indexes=bool(c_cfg.enable_block_indexes), + index_sample_ratio=c_cfg.index_sample_ratio, + block_index_prefix_len=c_cfg.block_index_prefix_len, + sync_mode=SyncMode(c_cfg.sync_mode), + sync_interval_us=c_cfg.sync_interval_us, + comparator_name=c_cfg.comparator_name.decode("utf-8").rstrip("\x00"), + skip_list_max_level=c_cfg.skip_list_max_level, + skip_list_probability=c_cfg.skip_list_probability, + default_isolation_level=IsolationLevel(c_cfg.default_isolation_level), + min_disk_space=c_cfg.min_disk_space, + l1_file_count_trigger=c_cfg.l1_file_count_trigger, + l0_queue_stall_threshold=c_cfg.l0_queue_stall_threshold, + ) + + stats = Stats( + num_levels=c_stats.num_levels, + memtable_size=c_stats.memtable_size, + level_sizes=level_sizes, + level_num_sstables=level_num_sstables, + config=config, + ) + + _lib.tidesdb_free_stats(stats_ptr) + return stats + + +class Transaction: + """Transaction for atomic operations.""" + + def __init__(self, txn_ptr: c_void_p) -> None: + self._txn = txn_ptr + self._closed = False + self._committed = False + self._freed = False + + def put(self, cf: ColumnFamily, key: bytes, value: bytes, ttl: int = -1) -> None: + """ + Put a key-value pair in the transaction. + + Args: + cf: Column family handle + key: Key as bytes + value: Value as bytes + ttl: Time-to-live as Unix timestamp (seconds since epoch), or -1 for no expiration + """ + if self._closed: + raise TidesDBError("Transaction is closed") + if self._committed: + raise TidesDBError("Transaction already committed") + + key_buf = (c_uint8 * len(key)).from_buffer_copy(key) if key else None + value_buf = (c_uint8 * len(value)).from_buffer_copy(value) if value else None + + result = _lib.tidesdb_txn_put( + self._txn, cf._cf, key_buf, len(key), value_buf, len(value), ttl + ) + + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to put key-value pair") + + def get(self, cf: ColumnFamily, key: bytes) -> bytes: + """ + Get a value from the transaction. + + Args: + cf: Column family handle + key: Key as bytes + + Returns: + Value as bytes + + Raises: + TidesDBError: If key not found or other error + """ + if self._closed: + raise TidesDBError("Transaction is closed") + + key_buf = (c_uint8 * len(key)).from_buffer_copy(key) if key else None + value_ptr = POINTER(c_uint8)() + value_size = c_size_t() + + result = _lib.tidesdb_txn_get( + self._txn, cf._cf, key_buf, len(key), ctypes.byref(value_ptr), ctypes.byref(value_size) + ) + + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to get value") + + value = ctypes.string_at(value_ptr, value_size.value) + + if sys.platform == "win32": + libc = ctypes.CDLL("msvcrt") + else: + libc = ctypes.CDLL(None) + libc.free(ctypes.cast(value_ptr, c_void_p)) + + return value + + def delete(self, cf: ColumnFamily, key: bytes) -> None: + """ + Delete a key-value pair in the transaction. + + Args: + cf: Column family handle + key: Key as bytes + """ + if self._closed: + raise TidesDBError("Transaction is closed") + if self._committed: + raise TidesDBError("Transaction already committed") + + key_buf = (c_uint8 * len(key)).from_buffer_copy(key) if key else None + + result = _lib.tidesdb_txn_delete(self._txn, cf._cf, key_buf, len(key)) + + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to delete key") + + def commit(self) -> None: + """Commit the transaction.""" + if self._closed: + raise TidesDBError("Transaction is closed") + if self._committed: + raise TidesDBError("Transaction already committed") + + result = _lib.tidesdb_txn_commit(self._txn) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to commit transaction") + + self._committed = True + + def rollback(self) -> None: + """Rollback the transaction.""" + if self._closed: + raise TidesDBError("Transaction is closed") + if self._committed: + raise TidesDBError("Transaction already committed") + + result = _lib.tidesdb_txn_rollback(self._txn) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to rollback transaction") + + def savepoint(self, name: str) -> None: + """Create a savepoint within the transaction.""" + if self._closed: + raise TidesDBError("Transaction is closed") + if self._committed: + raise TidesDBError("Transaction already committed") + + result = _lib.tidesdb_txn_savepoint(self._txn, name.encode("utf-8")) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to create savepoint") + + def rollback_to_savepoint(self, name: str) -> None: + """Rollback the transaction to a savepoint.""" + if self._closed: + raise TidesDBError("Transaction is closed") + if self._committed: + raise TidesDBError("Transaction already committed") + + result = _lib.tidesdb_txn_rollback_to_savepoint(self._txn, name.encode("utf-8")) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to rollback to savepoint") + + def release_savepoint(self, name: str) -> None: + """Release a savepoint without rolling back.""" + if self._closed: + raise TidesDBError("Transaction is closed") + if self._committed: + raise TidesDBError("Transaction already committed") + + result = _lib.tidesdb_txn_release_savepoint(self._txn, name.encode("utf-8")) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to release savepoint") + + def new_iterator(self, cf: ColumnFamily) -> Iterator: + """ + Create a new iterator for the column family within this transaction. + + Args: + cf: Column family handle + + Returns: + Iterator instance + """ + if self._closed: + raise TidesDBError("Transaction is closed") + + iter_ptr = c_void_p() + result = _lib.tidesdb_iter_new(self._txn, cf._cf, ctypes.byref(iter_ptr)) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to create iterator") + + return Iterator(iter_ptr) + + def close(self) -> None: + """Free transaction resources.""" + if not self._closed and self._txn: + _lib.tidesdb_txn_free(self._txn) + self._closed = True + + def __enter__(self) -> Transaction: + return self + + def __exit__(self, exc_type: type | None, exc_val: Exception | None, exc_tb: object) -> bool: + if exc_type is not None and not self._committed: + try: + self.rollback() + except TidesDBError: + pass + self.close() + return False + + def __del__(self) -> None: + if _lib is None: + return + self.close() + + +class TidesDB: + """TidesDB database instance.""" + + def __init__(self, config: Config) -> None: + """ + Open a TidesDB database. + + Args: + config: Database configuration + """ + self._db: c_void_p | None = None + self._closed = False + + os.makedirs(config.db_path, exist_ok=True) + abs_path = os.path.abspath(config.db_path) + + self._path_bytes = abs_path.encode("utf-8") + + c_config = _CConfig( + db_path=self._path_bytes, + num_flush_threads=config.num_flush_threads, + num_compaction_threads=config.num_compaction_threads, + log_level=int(config.log_level), + block_cache_size=config.block_cache_size, + max_open_sstables=config.max_open_sstables, + ) + + db_ptr = c_void_p() + result = _lib.tidesdb_open(ctypes.byref(c_config), ctypes.byref(db_ptr)) + + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to open database") + + self._db = db_ptr + + @classmethod + def open( + cls, + path: str, + num_flush_threads: int = 2, + num_compaction_threads: int = 2, + log_level: LogLevel = LogLevel.LOG_INFO, + block_cache_size: int = 64 * 1024 * 1024, + max_open_sstables: int = 256, + ) -> TidesDB: + """ + Convenience method to open a database with individual parameters. + + Args: + path: Path to the database directory + num_flush_threads: Number of flush threads + num_compaction_threads: Number of compaction threads + log_level: Logging level + block_cache_size: Size of block cache in bytes + max_open_sstables: Maximum number of open SSTables + + Returns: + TidesDB instance + """ + config = Config( + db_path=path, + num_flush_threads=num_flush_threads, + num_compaction_threads=num_compaction_threads, + log_level=log_level, + block_cache_size=block_cache_size, + max_open_sstables=max_open_sstables, + ) + return cls(config) + + def close(self) -> None: + """Close the database.""" + if _lib is None: + return + if not self._closed and self._db: + db_ptr = self._db + self._db = None + self._closed = True + result = _lib.tidesdb_close(db_ptr) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to close database") + + def create_column_family( + self, name: str, config: ColumnFamilyConfig | None = None + ) -> None: + """ + Create a new column family. + + Args: + name: Name of the column family + config: Configuration for the column family, or None for defaults + """ + if self._closed: + raise TidesDBError("Database is closed") + + if config is None: + config = default_column_family_config() + + c_config = config._to_c_struct() + + result = _lib.tidesdb_create_column_family( + self._db, name.encode("utf-8"), ctypes.byref(c_config) + ) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to create column family") + + def drop_column_family(self, name: str) -> None: + """ + Drop a column family and all its data. + + Args: + name: Name of the column family + """ + if self._closed: + raise TidesDBError("Database is closed") + + result = _lib.tidesdb_drop_column_family(self._db, name.encode("utf-8")) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to drop column family") + + def get_column_family(self, name: str) -> ColumnFamily: + """ + Get a column family handle. + + Args: + name: Name of the column family + + Returns: + ColumnFamily instance + """ + if self._closed: + raise TidesDBError("Database is closed") + + cf_ptr = _lib.tidesdb_get_column_family(self._db, name.encode("utf-8")) + if not cf_ptr: + raise TidesDBError(f"Column family not found: {name}", TDB_ERR_NOT_FOUND) + + return ColumnFamily(cf_ptr, name) + + def list_column_families(self) -> list[str]: + """ + List all column families in the database. + + Returns: + List of column family names + """ + if self._closed: + raise TidesDBError("Database is closed") + + names_ptr = POINTER(c_char_p)() + count = c_int() + + result = _lib.tidesdb_list_column_families( + self._db, ctypes.byref(names_ptr), ctypes.byref(count) + ) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to list column families") + + if count.value == 0: + return [] + + names = [] + + try: + if sys.platform == "win32": + libc = ctypes.CDLL("msvcrt") + elif sys.platform == "darwin": + libc = ctypes.CDLL("libc.dylib") + else: + libc = ctypes.CDLL("libc.so.6") + libc.free.argtypes = [c_void_p] + libc.free.restype = None + except OSError: + libc = None + + raw_array = ctypes.cast(names_ptr, POINTER(c_void_p)) + + for i in range(count.value): + str_ptr = raw_array[i] + if str_ptr: + char_ptr = ctypes.cast(str_ptr, c_char_p) + names.append(char_ptr.value.decode("utf-8")) + if libc: + libc.free(str_ptr) + + if libc: + libc.free(ctypes.cast(names_ptr, c_void_p)) + + return names + + def begin_txn(self) -> Transaction: + """ + Begin a new transaction with default isolation level. + + Returns: + Transaction instance + """ + if self._closed: + raise TidesDBError("Database is closed") + + txn_ptr = c_void_p() + result = _lib.tidesdb_txn_begin(self._db, ctypes.byref(txn_ptr)) + + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to begin transaction") + + return Transaction(txn_ptr) + + def begin_txn_with_isolation(self, isolation: IsolationLevel) -> Transaction: + """ + Begin a new transaction with the specified isolation level. + + Args: + isolation: Transaction isolation level + + Returns: + Transaction instance + """ + if self._closed: + raise TidesDBError("Database is closed") + + txn_ptr = c_void_p() + result = _lib.tidesdb_txn_begin_with_isolation( + self._db, int(isolation), ctypes.byref(txn_ptr) + ) + + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to begin transaction with isolation") + + return Transaction(txn_ptr) + + def get_cache_stats(self) -> CacheStats: + """ + Get statistics about the block cache. + + Returns: + CacheStats instance + """ + if self._closed: + raise TidesDBError("Database is closed") + + c_stats = _CCacheStats() + result = _lib.tidesdb_get_cache_stats(self._db, ctypes.byref(c_stats)) + if result != TDB_SUCCESS: + raise TidesDBError.from_code(result, "failed to get cache stats") + + return CacheStats( + enabled=bool(c_stats.enabled), + total_entries=c_stats.total_entries, + total_bytes=c_stats.total_bytes, + hits=c_stats.hits, + misses=c_stats.misses, + hit_rate=c_stats.hit_rate, + num_partitions=c_stats.num_partitions, + ) + + def __enter__(self) -> TidesDB: + return self + + def __exit__(self, exc_type: type | None, exc_val: Exception | None, exc_tb: object) -> bool: + self.close() + return False + + def __del__(self) -> None: + if _lib is None: + return + if not self._closed: + try: + self.close() + except (TidesDBError, OSError): + pass diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_tidesdb.py b/tests/test_tidesdb.py index 3f07c23..1dd2602 100644 --- a/tests/test_tidesdb.py +++ b/tests/test_tidesdb.py @@ -1,548 +1,352 @@ """ -TidesDB Python Bindings Tests +Tests for TidesDB Python bindings. -Copyright (C) TidesDB +These tests require the TidesDB shared library to be installed. """ -import unittest import os import shutil +import tempfile import time -import pickle -from pathlib import Path -from tidesdb import ( - TidesDB, - ColumnFamilyConfig, - CompressionAlgo, - SyncMode, - TidesDBException, - ErrorCode -) - - -class TestTidesDB(unittest.TestCase): - """Test suite for TidesDB Python bindings.""" - - def setUp(self): - """Set up test database.""" - # Use absolute path to avoid encoding issues - self.test_db_path = os.path.abspath("test_db") - if os.path.exists(self.test_db_path): - shutil.rmtree(self.test_db_path) - - def tearDown(self): - """Clean up test database.""" - if os.path.exists(self.test_db_path): - shutil.rmtree(self.test_db_path) - - def test_open_close(self): - """Test opening and closing database.""" - db = TidesDB(self.test_db_path) - self.assertIsNotNone(db) + +import pytest + +import tidesdb + + +@pytest.fixture +def temp_db_path(): + """Create a temporary directory for test database.""" + path = tempfile.mkdtemp(prefix="tidesdb_test_") + yield path + shutil.rmtree(path, ignore_errors=True) + + +@pytest.fixture +def db(temp_db_path): + """Create a test database.""" + database = tidesdb.TidesDB.open(temp_db_path) + yield database + database.close() + + +@pytest.fixture +def cf(db): + """Create a test column family.""" + db.create_column_family("test_cf") + cf = db.get_column_family("test_cf") + yield cf + try: + db.drop_column_family("test_cf") + except tidesdb.TidesDBError: + pass + + +class TestOpenClose: + """Tests for database open/close operations.""" + + def test_open_close(self, temp_db_path): + """Test basic open and close.""" + db = tidesdb.TidesDB.open(temp_db_path) + assert db is not None + db.close() + + def test_open_with_config(self, temp_db_path): + """Test open with custom configuration.""" + config = tidesdb.Config( + db_path=temp_db_path, + num_flush_threads=4, + num_compaction_threads=4, + log_level=tidesdb.LogLevel.LOG_WARN, + block_cache_size=32 * 1024 * 1024, + max_open_sstables=128, + ) + db = tidesdb.TidesDB(config) + assert db is not None db.close() - - def test_context_manager(self): + + def test_context_manager(self, temp_db_path): """Test database as context manager.""" - with TidesDB(self.test_db_path) as db: - self.assertIsNotNone(db) - # Create a CF with background compaction disabled - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - # Database should be closed after context - - def test_create_drop_column_family(self): - """Test creating and dropping column families.""" - with TidesDB(self.test_db_path) as db: - # Create with background compaction disabled - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Verify it exists - cf_list = db.list_column_families() - self.assertIn("test_cf", cf_list) - - # Drop it - db.drop_column_family("test_cf") - - # Verify it's gone - cf_list = db.list_column_families() - self.assertNotIn("test_cf", cf_list) - - def test_create_column_family_with_config(self): - """Test creating column family with custom configuration.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig( - memtable_flush_size=128 * 1024 * 1024, # 128MB - max_sstables_before_compaction=512, - compaction_threads=4, - compressed=True, - compress_algo=CompressionAlgo.LZ4, - bloom_filter_fp_rate=0.01, - enable_background_compaction=True, - sync_mode=SyncMode.BACKGROUND, - sync_interval=1000 - ) - - db.create_column_family("custom_cf", config) - - # Verify configuration - stats = db.get_column_family_stats("custom_cf") - self.assertEqual(stats.config.memtable_flush_size, 128 * 1024 * 1024) - self.assertEqual(stats.config.compress_algo, CompressionAlgo.LZ4) - self.assertTrue(stats.config.compressed) - self.assertTrue(stats.config.enable_background_compaction) - self.assertEqual(stats.config.sync_mode, SyncMode.BACKGROUND) - - def test_list_column_families(self): + with tidesdb.TidesDB.open(temp_db_path) as db: + assert db is not None + + +class TestColumnFamilies: + """Tests for column family operations.""" + + def test_create_drop_column_family(self, db): + """Test creating and dropping a column family.""" + db.create_column_family("test_cf") + cf = db.get_column_family("test_cf") + assert cf is not None + assert cf.name == "test_cf" + db.drop_column_family("test_cf") + + def test_create_with_config(self, db): + """Test creating column family with custom config.""" + config = tidesdb.default_column_family_config() + config.write_buffer_size = 32 * 1024 * 1024 + config.compression_algorithm = tidesdb.CompressionAlgorithm.LZ4_COMPRESSION + config.enable_bloom_filter = True + config.bloom_fpr = 0.01 + + db.create_column_family("custom_cf", config) + cf = db.get_column_family("custom_cf") + assert cf is not None + + stats = cf.get_stats() + assert stats.config is not None + assert stats.config.enable_bloom_filter is True + + db.drop_column_family("custom_cf") + + def test_list_column_families(self, db): """Test listing column families.""" - with TidesDB(self.test_db_path) as db: - # Create multiple column families with background compaction disabled - config = ColumnFamilyConfig(enable_background_compaction=False) - cf_names = ["cf1", "cf2", "cf3"] - for name in cf_names: - db.create_column_family(name, config) - - # List them - cf_list = db.list_column_families() - # Check that all created column families are present - # (there may be additional ones from previous test runs) - for name in cf_names: - self.assertIn(name, cf_list) - - def test_transaction_put_get_delete(self): - """Test basic CRUD operations with transactions.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Put data - with db.begin_txn() as txn: - txn.put("test_cf", b"key1", b"value1") - txn.put("test_cf", b"key2", b"value2") - txn.commit() - - # Get data - with db.begin_read_txn() as txn: - value1 = txn.get("test_cf", b"key1") - self.assertEqual(value1, b"value1") - - value2 = txn.get("test_cf", b"key2") - self.assertEqual(value2, b"value2") - - # Delete data - with db.begin_txn() as txn: - txn.delete("test_cf", b"key1") - txn.commit() - - # Verify deletion - with db.begin_read_txn() as txn: - with self.assertRaises(TidesDBException): - txn.get("test_cf", b"key1") - - def test_transaction_with_ttl(self): - """Test transactions with TTL.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Put with TTL (2 seconds from now) - ttl = int(time.time()) + 2 - with db.begin_txn() as txn: - txn.put("test_cf", b"temp_key", b"temp_value", ttl) - txn.commit() - - # Verify it exists - with db.begin_read_txn() as txn: - value = txn.get("test_cf", b"temp_key") - self.assertEqual(value, b"temp_value") - - # Wait for expiration - time.sleep(3) - - # Verify it's expired - with db.begin_read_txn() as txn: - with self.assertRaises(TidesDBException): - txn.get("test_cf", b"temp_key") - - def test_multi_operation_transaction(self): - """Test transaction with multiple operations.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Multiple operations in one transaction - with db.begin_txn() as txn: - for i in range(10): - key = f"key{i}".encode() - value = f"value{i}".encode() - txn.put("test_cf", key, value) - txn.commit() - - # Verify all were written - with db.begin_read_txn() as txn: - for i in range(10): - key = f"key{i}".encode() - expected_value = f"value{i}".encode() - value = txn.get("test_cf", key) - self.assertEqual(value, expected_value) - - def test_transaction_rollback(self): + db.create_column_family("cf1") + db.create_column_family("cf2") + + names = db.list_column_families() + assert "cf1" in names + assert "cf2" in names + + db.drop_column_family("cf1") + db.drop_column_family("cf2") + + def test_get_nonexistent_column_family(self, db): + """Test getting a non-existent column family.""" + with pytest.raises(tidesdb.TidesDBError): + db.get_column_family("nonexistent") + + +class TestTransactions: + """Tests for transaction operations.""" + + def test_put_get(self, db, cf): + """Test basic put and get.""" + with db.begin_txn() as txn: + txn.put(cf, b"key1", b"value1") + txn.commit() + + with db.begin_txn() as txn: + value = txn.get(cf, b"key1") + assert value == b"value1" + + def test_delete(self, db, cf): + """Test delete operation.""" + with db.begin_txn() as txn: + txn.put(cf, b"key1", b"value1") + txn.commit() + + with db.begin_txn() as txn: + txn.delete(cf, b"key1") + txn.commit() + + with db.begin_txn() as txn: + with pytest.raises(tidesdb.TidesDBError): + txn.get(cf, b"key1") + + def test_rollback(self, db, cf): """Test transaction rollback.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Put some data and rollback - with db.begin_txn() as txn: - txn.put("test_cf", b"rollback_key", b"rollback_value") - txn.rollback() - - # Verify data wasn't written - with db.begin_read_txn() as txn: - with self.assertRaises(TidesDBException): - txn.get("test_cf", b"rollback_key") - - def test_transaction_auto_rollback_on_exception(self): - """Test transaction automatically rolls back on exception.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Exception in context manager should trigger rollback - try: - with db.begin_txn() as txn: - txn.put("test_cf", b"error_key", b"error_value") - raise ValueError("Test error") - except ValueError: - pass - - # Verify data wasn't written - with db.begin_read_txn() as txn: - with self.assertRaises(TidesDBException): - txn.get("test_cf", b"error_key") - - def test_iterator_forward(self): + with db.begin_txn() as txn: + txn.put(cf, b"key1", b"value1") + txn.rollback() + + with db.begin_txn() as txn: + with pytest.raises(tidesdb.TidesDBError): + txn.get(cf, b"key1") + + def test_multiple_operations(self, db, cf): + """Test multiple operations in one transaction.""" + with db.begin_txn() as txn: + txn.put(cf, b"key1", b"value1") + txn.put(cf, b"key2", b"value2") + txn.put(cf, b"key3", b"value3") + txn.delete(cf, b"key2") + txn.commit() + + with db.begin_txn() as txn: + assert txn.get(cf, b"key1") == b"value1" + assert txn.get(cf, b"key3") == b"value3" + with pytest.raises(tidesdb.TidesDBError): + txn.get(cf, b"key2") + + def test_isolation_level(self, db, cf): + """Test transaction with specific isolation level.""" + txn = db.begin_txn_with_isolation(tidesdb.IsolationLevel.SERIALIZABLE) + txn.put(cf, b"key1", b"value1") + txn.commit() + txn.close() + + +class TestSavepoints: + """Tests for savepoint operations.""" + + def test_savepoint_rollback(self, db, cf): + """Test savepoint and rollback to savepoint.""" + with db.begin_txn() as txn: + txn.put(cf, b"key1", b"value1") + txn.savepoint("sp1") + txn.put(cf, b"key2", b"value2") + txn.rollback_to_savepoint("sp1") + txn.commit() + + with db.begin_txn() as txn: + assert txn.get(cf, b"key1") == b"value1" + with pytest.raises(tidesdb.TidesDBError): + txn.get(cf, b"key2") + + def test_release_savepoint(self, db, cf): + """Test releasing a savepoint.""" + with db.begin_txn() as txn: + txn.put(cf, b"key1", b"value1") + txn.savepoint("sp1") + txn.put(cf, b"key2", b"value2") + txn.release_savepoint("sp1") + txn.commit() + + with db.begin_txn() as txn: + assert txn.get(cf, b"key1") == b"value1" + assert txn.get(cf, b"key2") == b"value2" + + +class TestIterators: + """Tests for iterator operations.""" + + def test_forward_iteration(self, db, cf): """Test forward iteration.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Insert test data - test_data = { - b"key1": b"value1", - b"key2": b"value2", - b"key3": b"value3", - b"key4": b"value4", - b"key5": b"value5", - } - - with db.begin_txn() as txn: - for key, value in test_data.items(): - txn.put("test_cf", key, value) - txn.commit() - - # Iterate forward - with db.begin_read_txn() as txn: - with txn.new_iterator("test_cf") as it: - it.seek_to_first() - - count = 0 - while it.valid(): - key = it.key() - value = it.value() - - self.assertIn(key, test_data) - self.assertEqual(value, test_data[key]) - - count += 1 - it.next() - - self.assertEqual(count, len(test_data)) - - def test_iterator_backward(self): + with db.begin_txn() as txn: + txn.put(cf, b"a", b"1") + txn.put(cf, b"b", b"2") + txn.put(cf, b"c", b"3") + txn.commit() + + with db.begin_txn() as txn: + with txn.new_iterator(cf) as it: + it.seek_to_first() + items = list(it) + assert len(items) == 3 + assert items[0] == (b"a", b"1") + assert items[1] == (b"b", b"2") + assert items[2] == (b"c", b"3") + + def test_backward_iteration(self, db, cf): """Test backward iteration.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Insert test data - test_data = { - b"key1": b"value1", - b"key2": b"value2", - b"key3": b"value3", - } - - with db.begin_txn() as txn: - for key, value in test_data.items(): - txn.put("test_cf", key, value) - txn.commit() - - # Iterate backward - with db.begin_read_txn() as txn: - with txn.new_iterator("test_cf") as it: - it.seek_to_last() - - count = 0 - while it.valid(): - key = it.key() - value = it.value() - - self.assertIn(key, test_data) - self.assertEqual(value, test_data[key]) - - count += 1 - it.prev() - - self.assertEqual(count, len(test_data)) - - def test_iterator_as_python_iterator(self): - """Test iterator as Python iterator.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Insert test data - test_data = { - b"key1": b"value1", - b"key2": b"value2", - b"key3": b"value3", - } - - with db.begin_txn() as txn: - for key, value in test_data.items(): - txn.put("test_cf", key, value) - txn.commit() - - # Use as Python iterator - with db.begin_read_txn() as txn: - with txn.new_iterator("test_cf") as it: - it.seek_to_first() - - results = list(it) - self.assertEqual(len(results), len(test_data)) - - for key, value in results: - self.assertIn(key, test_data) - self.assertEqual(value, test_data[key]) - - def test_get_column_family_stats(self): + with db.begin_txn() as txn: + txn.put(cf, b"a", b"1") + txn.put(cf, b"b", b"2") + txn.put(cf, b"c", b"3") + txn.commit() + + with db.begin_txn() as txn: + with txn.new_iterator(cf) as it: + it.seek_to_last() + items = [] + while it.valid(): + items.append((it.key(), it.value())) + it.prev() + assert len(items) == 3 + assert items[0] == (b"c", b"3") + assert items[1] == (b"b", b"2") + assert items[2] == (b"a", b"1") + + def test_seek(self, db, cf): + """Test seek operations.""" + with db.begin_txn() as txn: + txn.put(cf, b"a", b"1") + txn.put(cf, b"c", b"3") + txn.put(cf, b"e", b"5") + txn.commit() + + with db.begin_txn() as txn: + with txn.new_iterator(cf) as it: + it.seek(b"b") + assert it.valid() + assert it.key() == b"c" + + it.seek_for_prev(b"d") + assert it.valid() + assert it.key() == b"c" + + +class TestTTL: + """Tests for TTL functionality.""" + + def test_ttl_expiration(self, db, cf): + """Test that keys with expired TTL are eventually not returned.""" + expired_ttl = int(time.time()) - 1 + + with db.begin_txn() as txn: + txn.put(cf, b"expired_key", b"value", ttl=expired_ttl) + txn.commit() + + cf.flush_memtable() + time.sleep(0.5) + + with db.begin_txn() as txn: + try: + txn.get(cf, b"expired_key") + except tidesdb.TidesDBError: + pass + + def test_no_ttl(self, db, cf): + """Test that keys without TTL persist.""" + with db.begin_txn() as txn: + txn.put(cf, b"permanent_key", b"value", ttl=-1) + txn.commit() + + with db.begin_txn() as txn: + value = txn.get(cf, b"permanent_key") + assert value == b"value" + + +class TestStats: + """Tests for statistics operations.""" + + def test_column_family_stats(self, db, cf): """Test getting column family statistics.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig( - memtable_flush_size=2 * 1024 * 1024, # 2MB - max_level=12, - compressed=True, - compress_algo=CompressionAlgo.SNAPPY, - bloom_filter_fp_rate=0.01 - ) - - db.create_column_family("test_cf", config) - - # Add some data - with db.begin_txn() as txn: - for i in range(10): - key = f"key{i}".encode() - value = f"value{i}".encode() - txn.put("test_cf", key, value) - txn.commit() - - # Get statistics - stats = db.get_column_family_stats("test_cf") - - self.assertEqual(stats.name, "test_cf") - self.assertEqual(stats.config.memtable_flush_size, 2 * 1024 * 1024) - self.assertEqual(stats.config.max_level, 12) - self.assertTrue(stats.config.compressed) - self.assertEqual(stats.config.compress_algo, CompressionAlgo.SNAPPY) - # Data may be in memtable or flushed to SSTables - self.assertGreaterEqual(stats.memtable_entries, 0) - self.assertGreaterEqual(stats.memtable_size, 0) - - def test_compaction(self): + with db.begin_txn() as txn: + txn.put(cf, b"key1", b"value1") + txn.commit() + + stats = cf.get_stats() + assert stats.num_levels >= 0 + assert stats.memtable_size >= 0 + + def test_cache_stats(self, db): + """Test getting cache statistics.""" + stats = db.get_cache_stats() + assert isinstance(stats.enabled, bool) + assert stats.hits >= 0 + assert stats.misses >= 0 + + +class TestMaintenance: + """Tests for maintenance operations.""" + + def test_flush_memtable(self, db, cf): + """Test manual memtable flush.""" + with db.begin_txn() as txn: + txn.put(cf, b"key1", b"value1") + txn.commit() + + cf.flush_memtable() + time.sleep(0.5) + + def test_compact(self, db, cf): """Test manual compaction.""" - with TidesDB(self.test_db_path) as db: - # Create CF with small flush threshold to force SSTables - config = ColumnFamilyConfig( - memtable_flush_size=1024, # 1KB - enable_background_compaction=False, - compaction_threads=2 - ) - - db.create_column_family("test_cf", config) - - # Add data to create multiple SSTables - for batch in range(5): - with db.begin_txn() as txn: - for i in range(20): - key = f"key{batch}_{i}".encode() - value = b"x" * 512 # 512 bytes - txn.put("test_cf", key, value) - txn.commit() - - # Get column family for compaction - cf = db.get_column_family("test_cf") - - # Check stats before compaction - stats_before = db.get_column_family_stats("test_cf") - - # Perform compaction if we have enough SSTables - if stats_before.num_sstables >= 2: - cf.compact() - - stats_after = db.get_column_family_stats("test_cf") - # Note: compaction may or may not reduce SSTable count - # depending on timing, but it should complete without error - self.assertIsNotNone(stats_after) - - def test_sync_modes(self): - """Test different sync modes.""" - sync_modes = [ - (SyncMode.NONE, "none"), - (SyncMode.BACKGROUND, "background"), - (SyncMode.FULL, "full"), - ] - - with TidesDB(self.test_db_path) as db: - for mode, name in sync_modes: - cf_name = f"cf_{name}" - - config = ColumnFamilyConfig( - sync_mode=mode, - sync_interval=1000 if mode == SyncMode.BACKGROUND else 0 - ) - - db.create_column_family(cf_name, config) - - # Verify sync mode - stats = db.get_column_family_stats(cf_name) - self.assertEqual(stats.config.sync_mode, mode) - - def test_compression_algorithms(self): - """Test different compression algorithms.""" - # Test with compression disabled - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(compressed=False) - db.create_column_family("cf_none", config) - stats = db.get_column_family_stats("cf_none") - self.assertFalse(stats.config.compressed) - - # Test with different compression algorithms - algorithms = [ - (CompressionAlgo.SNAPPY, "snappy"), - (CompressionAlgo.LZ4, "lz4"), - (CompressionAlgo.ZSTD, "zstd"), - ] - - with TidesDB(self.test_db_path) as db: - for algo, name in algorithms: - cf_name = f"cf_{name}" - - config = ColumnFamilyConfig( - compressed=True, - compress_algo=algo - ) - - db.create_column_family(cf_name, config) - - # Verify compression - stats = db.get_column_family_stats(cf_name) - self.assertTrue(stats.config.compressed) - self.assertEqual(stats.config.compress_algo, algo) - - def test_pickle_support(self): - """Test storing Python objects with pickle.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Store complex Python object - test_obj = { - "name": "John Doe", - "age": 30, - "scores": [95, 87, 92], - "metadata": {"city": "NYC", "country": "USA"} - } - - key = b"user:123" - value = pickle.dumps(test_obj) - - with db.begin_txn() as txn: - txn.put("test_cf", key, value) - txn.commit() - - # Retrieve and deserialize - with db.begin_read_txn() as txn: - stored_value = txn.get("test_cf", key) - retrieved_obj = pickle.loads(stored_value) - - self.assertEqual(retrieved_obj, test_obj) - - def test_error_handling(self): - """Test error handling.""" - with TidesDB(self.test_db_path) as db: - # Try to get stats for non-existent CF - with self.assertRaises(TidesDBException) as ctx: - db.get_column_family_stats("nonexistent_cf") - self.assertIsInstance(ctx.exception.code, int) - - # Try to drop non-existent CF - with self.assertRaises(TidesDBException): - db.drop_column_family("nonexistent_cf") - - # Try to get from non-existent CF - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - with db.begin_read_txn() as txn: - with self.assertRaises(TidesDBException): - txn.get("nonexistent_cf", b"key") - - def test_large_values(self): - """Test storing large values.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - # Store 1MB value - large_value = b"x" * (1024 * 1024) - - with db.begin_txn() as txn: - txn.put("test_cf", b"large_key", large_value) - txn.commit() - - # Retrieve it - with db.begin_read_txn() as txn: - retrieved = txn.get("test_cf", b"large_key") - self.assertEqual(len(retrieved), len(large_value)) - self.assertEqual(retrieved, large_value) - - def test_many_keys(self): - """Test storing many keys.""" - with TidesDB(self.test_db_path) as db: - config = ColumnFamilyConfig(enable_background_compaction=False) - db.create_column_family("test_cf", config) - - num_keys = 1000 - - # Insert many keys - with db.begin_txn() as txn: - for i in range(num_keys): - key = f"key_{i:06d}".encode() - value = f"value_{i}".encode() - txn.put("test_cf", key, value) - txn.commit() - - # Verify count with iterator - with db.begin_read_txn() as txn: - with txn.new_iterator("test_cf") as it: - it.seek_to_first() - count = sum(1 for _ in it) - self.assertEqual(count, num_keys) - - -def run_tests(): - """Run all tests.""" - unittest.main() - - -if __name__ == '__main__': - run_tests() \ No newline at end of file + with db.begin_txn() as txn: + for i in range(100): + txn.put(cf, f"key{i}".encode(), f"value{i}".encode()) + txn.commit() + + cf.flush_memtable() + time.sleep(0.5) + try: + cf.compact() + except tidesdb.TidesDBError: + pass + time.sleep(0.5) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tidesdb/__init__.py b/tidesdb/__init__.py deleted file mode 100644 index 16c6ab6..0000000 --- a/tidesdb/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -""" -TidesDB Python Bindings - -Official Python bindings for TidesDB v1+. -""" - -from .tidesdb import ( - TidesDB, - Transaction, - Iterator, - ColumnFamily, - ColumnFamilyConfig, - ColumnFamilyStat, - CompressionAlgo, - SyncMode, - ErrorCode, - TidesDBException, -) - -__version__ = "1.0.0" -__author__ = "TidesDB Authors" -__license__ = "MPL-2.0" - -__all__ = [ - 'TidesDB', - 'Transaction', - 'Iterator', - 'ColumnFamily', - 'ColumnFamilyConfig', - 'ColumnFamilyStat', - 'CompressionAlgo', - 'SyncMode', - 'ErrorCode', - 'TidesDBException', -] \ No newline at end of file diff --git a/tidesdb/tidesdb.py b/tidesdb/tidesdb.py deleted file mode 100644 index c10cfde..0000000 --- a/tidesdb/tidesdb.py +++ /dev/null @@ -1,951 +0,0 @@ -""" -TidesDB Python Bindings v1 - -Copyright (C) TidesDB -Original Author: Alex Gaetano Padula - -Licensed under the Mozilla Public License, v. 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - https://www.mozilla.org/en-US/MPL/2.0/ - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import ctypes -from ctypes import c_void_p, c_char_p, c_int, c_size_t, c_int64, c_float, c_double, POINTER, Structure -from typing import Optional, List, Tuple, Dict -from enum import IntEnum -import os - - -# Load the TidesDB shared library -def _load_library(): - """Load the TidesDB shared library.""" - lib_names = ['libtidesdb.so', 'libtidesdb.dylib', 'tidesdb.dll'] - - for lib_name in lib_names: - try: - return ctypes.CDLL(lib_name) - except OSError: - continue - - # Try system paths - try: - return ctypes.CDLL('tidesdb') - except OSError: - raise RuntimeError( - "Could not load TidesDB library. " - "Please ensure libtidesdb is installed and in your library path." - ) - - -_lib = _load_library() - - -def _get_libc(): - """Get the C standard library for memory management operations.""" - import sys - from ctypes.util import find_library - - if sys.platform == 'win32': - return ctypes.cdll.msvcrt - else: - # Use find_library to locate the correct libc - libc_name = find_library('c') - if libc_name: - return ctypes.CDLL(libc_name) - # Fallback to platform-specific names - elif sys.platform == 'darwin': - return ctypes.CDLL('libc.dylib') - else: - return ctypes.CDLL('libc.so.6') - - -_libc = _get_libc() - - -# Error codes -class ErrorCode(IntEnum): - """TidesDB error codes.""" - TDB_SUCCESS = 0 - TDB_ERROR = -1 - TDB_ERR_MEMORY = -2 - TDB_ERR_INVALID_ARGS = -3 - TDB_ERR_IO = -4 - TDB_ERR_NOT_FOUND = -5 - TDB_ERR_EXISTS = -6 - TDB_ERR_CORRUPT = -7 - TDB_ERR_LOCK = -8 - TDB_ERR_TXN_COMMITTED = -9 - TDB_ERR_TXN_ABORTED = -10 - TDB_ERR_READONLY = -11 - TDB_ERR_FULL = -12 - TDB_ERR_INVALID_NAME = -13 - TDB_ERR_COMPARATOR_NOT_FOUND = -14 - TDB_ERR_MAX_COMPARATORS = -15 - TDB_ERR_INVALID_CF = -16 - TDB_ERR_THREAD = -17 - TDB_ERR_CHECKSUM = -18 - TDB_ERR_KEY_DELETED = -19 - TDB_ERR_KEY_EXPIRED = -20 - - -class CompressionAlgo(IntEnum): - """Compression algorithm types (matches compress_type enum).""" - SNAPPY = 0 # COMPRESS_SNAPPY - LZ4 = 1 # COMPRESS_LZ4 - ZSTD = 2 # COMPRESS_ZSTD - - -class SyncMode(IntEnum): - """Sync modes for durability.""" - NONE = 0 # Fastest, least durable (OS handles flushing) - BACKGROUND = 1 # Balanced (fsync every N milliseconds) - FULL = 2 # Most durable (fsync on every write) - - -class TidesDBException(Exception): - """Base exception for TidesDB errors.""" - - def __init__(self, message: str, code: int = ErrorCode.TDB_ERROR): - super().__init__(message) - self.code = code - - @classmethod - def from_code(cls, code: int, context: str = "") -> 'TidesDBException': - """Create exception from error code.""" - error_messages = { - ErrorCode.TDB_ERR_MEMORY: "memory allocation failed", - ErrorCode.TDB_ERR_INVALID_ARGS: "invalid arguments", - ErrorCode.TDB_ERR_IO: "I/O error", - ErrorCode.TDB_ERR_NOT_FOUND: "not found", - ErrorCode.TDB_ERR_EXISTS: "already exists", - ErrorCode.TDB_ERR_CORRUPT: "data corruption", - ErrorCode.TDB_ERR_LOCK: "lock acquisition failed", - ErrorCode.TDB_ERR_TXN_COMMITTED: "transaction already committed", - ErrorCode.TDB_ERR_TXN_ABORTED: "transaction aborted", - ErrorCode.TDB_ERR_READONLY: "read-only transaction", - ErrorCode.TDB_ERR_FULL: "database full", - ErrorCode.TDB_ERR_INVALID_NAME: "invalid name", - ErrorCode.TDB_ERR_COMPARATOR_NOT_FOUND: "comparator not found", - ErrorCode.TDB_ERR_MAX_COMPARATORS: "max comparators reached", - ErrorCode.TDB_ERR_INVALID_CF: "invalid column family", - ErrorCode.TDB_ERR_THREAD: "thread operation failed", - ErrorCode.TDB_ERR_CHECKSUM: "checksum verification failed", - ErrorCode.TDB_ERR_KEY_DELETED: "key is deleted (tombstone)", - ErrorCode.TDB_ERR_KEY_EXPIRED: "key has expired (TTL)", - } - - msg = error_messages.get(code, "unknown error") - if context: - msg = f"{context}: {msg} (code: {code})" - else: - msg = f"{msg} (code: {code})" - - return cls(msg, code) - - -# C structures -class CConfig(Structure): - """C structure for tidesdb_config_t.""" - _fields_ = [ - ("db_path", ctypes.c_char * 1024), # TDB_MAX_PATH_LENGTH = 1024 - ("enable_debug_logging", c_int), - ("max_open_file_handles", c_int), - ] - - -class CColumnFamilyConfig(Structure): - """C structure for tidesdb_column_family_config_t.""" - _fields_ = [ - ("memtable_flush_size", c_size_t), - ("max_sstables_before_compaction", c_int), - ("compaction_threads", c_int), - ("max_level", c_int), - ("probability", c_float), - ("compressed", c_int), - ("compress_algo", c_int), - ("bloom_filter_fp_rate", c_double), - ("enable_background_compaction", c_int), - ("background_compaction_interval", c_int), - ("use_sbha", c_int), - ("sync_mode", c_int), - ("sync_interval", c_int), - ("comparator_name", c_char_p), - ] - - -class CColumnFamilyStat(Structure): - """C structure for tidesdb_column_family_stat_t.""" - _fields_ = [ - ("name", ctypes.c_char * 256), # TDB_MAX_CF_NAME_LENGTH - ("comparator_name", ctypes.c_char * 64), # TDB_MAX_COMPARATOR_NAME - ("num_sstables", c_int), - ("total_sstable_size", c_size_t), - ("memtable_size", c_size_t), - ("memtable_entries", c_int), - ("config", CColumnFamilyConfig), - ] - - -# Function signatures -_lib.tidesdb_open.argtypes = [POINTER(CConfig), POINTER(c_void_p)] -_lib.tidesdb_open.restype = c_int - -_lib.tidesdb_close.argtypes = [c_void_p] -_lib.tidesdb_close.restype = c_int - -_lib.tidesdb_default_column_family_config.argtypes = [] -_lib.tidesdb_default_column_family_config.restype = CColumnFamilyConfig - -_lib.tidesdb_create_column_family.argtypes = [c_void_p, c_char_p, POINTER(CColumnFamilyConfig)] -_lib.tidesdb_create_column_family.restype = c_int - -_lib.tidesdb_drop_column_family.argtypes = [c_void_p, c_char_p] -_lib.tidesdb_drop_column_family.restype = c_int - -_lib.tidesdb_get_column_family.argtypes = [c_void_p, c_char_p] -_lib.tidesdb_get_column_family.restype = c_void_p - -_lib.tidesdb_list_column_families.argtypes = [c_void_p, POINTER(POINTER(c_char_p)), POINTER(c_int)] -_lib.tidesdb_list_column_families.restype = c_int - -_lib.tidesdb_get_column_family_stats.argtypes = [c_void_p, c_char_p, POINTER(POINTER(CColumnFamilyStat))] -_lib.tidesdb_get_column_family_stats.restype = c_int - -_lib.tidesdb_compact.argtypes = [c_void_p] -_lib.tidesdb_compact.restype = c_int - -_lib.tidesdb_txn_begin.argtypes = [c_void_p, POINTER(c_void_p)] -_lib.tidesdb_txn_begin.restype = c_int - -_lib.tidesdb_txn_begin_read.argtypes = [c_void_p, POINTER(c_void_p)] -_lib.tidesdb_txn_begin_read.restype = c_int - -_lib.tidesdb_txn_put.argtypes = [c_void_p, c_char_p, POINTER(ctypes.c_uint8), c_size_t, - POINTER(ctypes.c_uint8), c_size_t, c_int64] -_lib.tidesdb_txn_put.restype = c_int - -_lib.tidesdb_txn_get.argtypes = [c_void_p, c_char_p, POINTER(ctypes.c_uint8), c_size_t, - POINTER(POINTER(ctypes.c_uint8)), POINTER(c_size_t)] -_lib.tidesdb_txn_get.restype = c_int - -_lib.tidesdb_txn_delete.argtypes = [c_void_p, c_char_p, POINTER(ctypes.c_uint8), c_size_t] -_lib.tidesdb_txn_delete.restype = c_int - -_lib.tidesdb_txn_commit.argtypes = [c_void_p] -_lib.tidesdb_txn_commit.restype = c_int - -_lib.tidesdb_txn_rollback.argtypes = [c_void_p] -_lib.tidesdb_txn_rollback.restype = c_int - -_lib.tidesdb_txn_free.argtypes = [c_void_p] -_lib.tidesdb_txn_free.restype = None - -_lib.tidesdb_iter_new.argtypes = [c_void_p, c_char_p, POINTER(c_void_p)] -_lib.tidesdb_iter_new.restype = c_int - -_lib.tidesdb_iter_seek_to_first.argtypes = [c_void_p] -_lib.tidesdb_iter_seek_to_first.restype = c_int - -_lib.tidesdb_iter_seek_to_last.argtypes = [c_void_p] -_lib.tidesdb_iter_seek_to_last.restype = c_int - -_lib.tidesdb_iter_valid.argtypes = [c_void_p] -_lib.tidesdb_iter_valid.restype = c_int - -_lib.tidesdb_iter_next.argtypes = [c_void_p] -_lib.tidesdb_iter_next.restype = c_int - -_lib.tidesdb_iter_prev.argtypes = [c_void_p] -_lib.tidesdb_iter_prev.restype = c_int - -_lib.tidesdb_iter_key.argtypes = [c_void_p, POINTER(POINTER(ctypes.c_uint8)), POINTER(c_size_t)] -_lib.tidesdb_iter_key.restype = c_int - -_lib.tidesdb_iter_value.argtypes = [c_void_p, POINTER(POINTER(ctypes.c_uint8)), POINTER(c_size_t)] -_lib.tidesdb_iter_value.restype = c_int - -_lib.tidesdb_iter_free.argtypes = [c_void_p] -_lib.tidesdb_iter_free.restype = None - - -class ColumnFamilyConfig: - """Configuration for a column family.""" - - def __init__( - self, - memtable_flush_size: int = 67108864, # 64MB - max_sstables_before_compaction: int = 128, - compaction_threads: int = 4, - max_level: int = 12, - probability: float = 0.25, - compressed: bool = True, - compress_algo: CompressionAlgo = CompressionAlgo.SNAPPY, - bloom_filter_fp_rate: float = 0.01, - enable_background_compaction: bool = True, - background_compaction_interval: int = 1000000, # 1 second in microseconds - use_sbha: bool = True, - sync_mode: SyncMode = SyncMode.BACKGROUND, - sync_interval: int = 1000, - comparator_name: Optional[str] = None - ): - """ - Initialize column family configuration. - - Args: - memtable_flush_size: Size threshold for memtable flush (default 64MB) - max_sstables_before_compaction: Trigger compaction at this many SSTables (default 128) - compaction_threads: Number of threads for parallel compaction (default 4, 0=single-threaded) - max_level: Skip list max level (default 12) - probability: Skip list probability (default 0.25) - compressed: Enable compression (default True) - compress_algo: Compression algorithm (default SNAPPY) - bloom_filter_fp_rate: Bloom filter false positive rate (default 0.01) - enable_background_compaction: Enable automatic background compaction (default True) - background_compaction_interval: Interval in microseconds between compaction checks (default 1000000 = 1 second) - use_sbha: Use sorted binary hash array for fast lookups (default True) - sync_mode: Durability sync mode (default BACKGROUND) - sync_interval: Sync interval in milliseconds for BACKGROUND mode (default 1000) - comparator_name: Name of custom comparator or None for default "memcmp" - """ - self.memtable_flush_size = memtable_flush_size - self.max_sstables_before_compaction = max_sstables_before_compaction - self.compaction_threads = compaction_threads - self.max_level = max_level - self.probability = probability - self.compressed = compressed - self.compress_algo = compress_algo - self.bloom_filter_fp_rate = bloom_filter_fp_rate - self.enable_background_compaction = enable_background_compaction - self.background_compaction_interval = background_compaction_interval - self.use_sbha = use_sbha - self.sync_mode = sync_mode - self.sync_interval = sync_interval - self.comparator_name = comparator_name - - @classmethod - def default(cls) -> 'ColumnFamilyConfig': - """Get default column family configuration.""" - c_config = _lib.tidesdb_default_column_family_config() - return cls( - memtable_flush_size=c_config.memtable_flush_size, - max_sstables_before_compaction=c_config.max_sstables_before_compaction, - compaction_threads=c_config.compaction_threads, - max_level=c_config.max_level, - probability=c_config.probability, - compressed=bool(c_config.compressed), - compress_algo=CompressionAlgo(c_config.compress_algo), - bloom_filter_fp_rate=c_config.bloom_filter_fp_rate, - enable_background_compaction=bool(c_config.enable_background_compaction), - background_compaction_interval=c_config.background_compaction_interval, - use_sbha=bool(c_config.use_sbha), - sync_mode=SyncMode(c_config.sync_mode), - sync_interval=c_config.sync_interval, - comparator_name=None - ) - - def _to_c_struct(self) -> CColumnFamilyConfig: - """Convert to C structure.""" - c_config = CColumnFamilyConfig() - c_config.memtable_flush_size = self.memtable_flush_size - c_config.max_sstables_before_compaction = self.max_sstables_before_compaction - c_config.compaction_threads = self.compaction_threads - c_config.max_level = self.max_level - c_config.probability = self.probability - c_config.compressed = 1 if self.compressed else 0 - c_config.compress_algo = int(self.compress_algo) - c_config.bloom_filter_fp_rate = self.bloom_filter_fp_rate - c_config.enable_background_compaction = 1 if self.enable_background_compaction else 0 - c_config.background_compaction_interval = self.background_compaction_interval - c_config.use_sbha = 1 if self.use_sbha else 0 - c_config.sync_mode = int(self.sync_mode) - c_config.sync_interval = self.sync_interval - c_config.comparator_name = self.comparator_name.encode() if self.comparator_name else None - return c_config - - -class ColumnFamilyStat: - """Statistics for a column family.""" - - def __init__( - self, - name: str, - comparator_name: str, - num_sstables: int, - total_sstable_size: int, - memtable_size: int, - memtable_entries: int, - config: ColumnFamilyConfig - ): - self.name = name - self.comparator_name = comparator_name - self.num_sstables = num_sstables - self.total_sstable_size = total_sstable_size - self.memtable_size = memtable_size - self.memtable_entries = memtable_entries - self.config = config - - def __repr__(self) -> str: - return ( - f"ColumnFamilyStat(name={self.name!r}, comparator={self.comparator_name!r}, " - f"sstables={self.num_sstables}, memtable_entries={self.memtable_entries})" - ) - - -class Iterator: - """Iterator for traversing key-value pairs in a column family.""" - - def __init__(self, iter_ptr: c_void_p): - """Initialize iterator with C pointer.""" - self._iter = iter_ptr - self._closed = False - - def seek_to_first(self) -> None: - """Position iterator at the first key.""" - if self._closed: - raise TidesDBException("Iterator is closed") - _lib.tidesdb_iter_seek_to_first(self._iter) - - def seek_to_last(self) -> None: - """Position iterator at the last key.""" - if self._closed: - raise TidesDBException("Iterator is closed") - _lib.tidesdb_iter_seek_to_last(self._iter) - - def valid(self) -> bool: - """Check if iterator is positioned at a valid entry.""" - if self._closed: - return False - return bool(_lib.tidesdb_iter_valid(self._iter)) - - def next(self) -> None: - """Move iterator to the next entry.""" - if self._closed: - raise TidesDBException("Iterator is closed") - _lib.tidesdb_iter_next(self._iter) - - def prev(self) -> None: - """Move iterator to the previous entry.""" - if self._closed: - raise TidesDBException("Iterator is closed") - _lib.tidesdb_iter_prev(self._iter) - - def key(self) -> bytes: - """Get the current key.""" - if self._closed: - raise TidesDBException("Iterator is closed") - - key_ptr = POINTER(ctypes.c_uint8)() - key_size = c_size_t() - - result = _lib.tidesdb_iter_key(self._iter, ctypes.byref(key_ptr), ctypes.byref(key_size)) - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to get key") - - # key_ptr points to internal iterator memory, do NOT free it - return ctypes.string_at(key_ptr, key_size.value) - - def value(self) -> bytes: - """Get the current value.""" - if self._closed: - raise TidesDBException("Iterator is closed") - - value_ptr = POINTER(ctypes.c_uint8)() - value_size = c_size_t() - - result = _lib.tidesdb_iter_value(self._iter, ctypes.byref(value_ptr), ctypes.byref(value_size)) - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to get value") - - # value_ptr points to internal iterator memory, do NOT free it - return ctypes.string_at(value_ptr, value_size.value) - - def items(self) -> List[Tuple[bytes, bytes]]: - """Get all remaining items as a list of (key, value) tuples.""" - results = [] - while self.valid(): - results.append((self.key(), self.value())) - self.next() - return results - - def close(self) -> None: - """Free iterator resources.""" - if not self._closed: - _lib.tidesdb_iter_free(self._iter) - self._closed = True - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - def __iter__(self): - return self - - def __next__(self) -> Tuple[bytes, bytes]: - if not self.valid(): - raise StopIteration - key = self.key() - value = self.value() - self.next() - return key, value - - def __del__(self): - self.close() - - -class Transaction: - """Transaction for atomic operations.""" - - def __init__(self, txn_ptr: c_void_p): - """Initialize transaction with C pointer.""" - self._txn = txn_ptr - self._closed = False - self._committed = False - - def put(self, column_family: str, key: bytes, value: bytes, ttl: int = -1) -> None: - """ - Put a key-value pair in the transaction. - - Args: - column_family: Name of the column family - key: Key as bytes - value: Value as bytes - ttl: Time-to-live as Unix timestamp, or -1 for no expiration - """ - if self._closed: - raise TidesDBException("Transaction is closed") - if self._committed: - raise TidesDBException("Transaction already committed") - - cf_name = column_family.encode() - key_buf = (ctypes.c_uint8 * len(key)).from_buffer_copy(key) - value_buf = (ctypes.c_uint8 * len(value)).from_buffer_copy(value) - - result = _lib.tidesdb_txn_put( - self._txn, cf_name, - key_buf, len(key), - value_buf, len(value), - ttl - ) - - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to put key-value pair") - - def get(self, column_family: str, key: bytes) -> bytes: - """ - Get a value from the transaction. - - Args: - column_family: Name of the column family - key: Key as bytes - - Returns: - Value as bytes - """ - if self._closed: - raise TidesDBException("Transaction is closed") - - cf_name = column_family.encode() - key_buf = (ctypes.c_uint8 * len(key)).from_buffer_copy(key) - value_ptr = POINTER(ctypes.c_uint8)() - value_size = c_size_t() - - result = _lib.tidesdb_txn_get( - self._txn, cf_name, - key_buf, len(key), - ctypes.byref(value_ptr), ctypes.byref(value_size) - ) - - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to get value") - - value = ctypes.string_at(value_ptr, value_size.value) - - # Free the malloc'd value (C API allocates with malloc) - _libc.free(ctypes.cast(value_ptr, ctypes.c_void_p)) - return value - - def delete(self, column_family: str, key: bytes) -> None: - """ - Delete a key-value pair in the transaction. - - Args: - column_family: Name of the column family - key: Key as bytes - """ - if self._closed: - raise TidesDBException("Transaction is closed") - if self._committed: - raise TidesDBException("Transaction already committed") - - cf_name = column_family.encode() - key_buf = (ctypes.c_uint8 * len(key)).from_buffer_copy(key) - - result = _lib.tidesdb_txn_delete(self._txn, cf_name, key_buf, len(key)) - - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to delete key") - - def commit(self) -> None: - """Commit the transaction.""" - if self._closed: - raise TidesDBException("Transaction is closed") - if self._committed: - raise TidesDBException("Transaction already committed") - - result = _lib.tidesdb_txn_commit(self._txn) - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to commit transaction") - - self._committed = True - - def rollback(self) -> None: - """Rollback the transaction.""" - if self._closed: - raise TidesDBException("Transaction is closed") - if self._committed: - raise TidesDBException("Transaction already committed") - - result = _lib.tidesdb_txn_rollback(self._txn) - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to rollback transaction") - - def new_iterator(self, column_family: str) -> Iterator: - """ - Create a new iterator for the column family. - - Args: - column_family: Name of the column family - - Returns: - Iterator instance - """ - if self._closed: - raise TidesDBException("Transaction is closed") - - cf_name = column_family.encode() - iter_ptr = c_void_p() - - result = _lib.tidesdb_iter_new(self._txn, cf_name, ctypes.byref(iter_ptr)) - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to create iterator") - - return Iterator(iter_ptr) - - def close(self) -> None: - """Free transaction resources.""" - if not self._closed: - _lib.tidesdb_txn_free(self._txn) - self._closed = True - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if exc_type is not None and not self._committed: - try: - self.rollback() - except: - pass - self.close() - return False - - def __del__(self): - self.close() - - -class ColumnFamily: - """Column family handle.""" - - def __init__(self, cf_ptr: c_void_p, name: str): - """Initialize column family with C pointer.""" - self._cf = cf_ptr - self.name = name - - def compact(self) -> None: - """ - Manually trigger compaction for this column family. - Requires minimum 2 SSTables to merge. - Uses parallel compaction if compaction_threads > 0. - """ - result = _lib.tidesdb_compact(self._cf) - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to compact column family") - - -class TidesDB: - """TidesDB database instance.""" - - def __init__(self, path: str, enable_debug_logging: bool = False, max_open_file_handles: int = 0): - """Open a TidesDB database. - - Args: - path: Path to the database directory - enable_debug_logging: Enable debug logging to stderr - max_open_file_handles: Maximum number of open file handles to cache (0 = unlimited) - """ - import os - - # Initialize state first - self._db = None - self._closed = False - - # Create directory if it doesn't exist - os.makedirs(path, exist_ok=True) - - # Convert to absolute path - abs_path = os.path.abspath(path) - - # Encode path and ensure it fits in TDB_MAX_PATH_LENGTH (1024 bytes) - path_bytes = abs_path.encode('utf-8') - if len(path_bytes) >= 1024: - raise ValueError(f"Database path too long (max 1023 bytes): {abs_path}") - - # Create config with fixed-size char array - self._config = CConfig( - enable_debug_logging=1 if enable_debug_logging else 0, - max_open_file_handles=max_open_file_handles - ) - # Copy path into the fixed-size array - self._config.db_path = path_bytes - - db_ptr = c_void_p() - result = _lib.tidesdb_open(ctypes.byref(self._config), ctypes.byref(db_ptr)) - - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to open database") - - self._db = db_ptr - - def close(self) -> None: - """Close the database.""" - if not self._closed and self._db: - result = _lib.tidesdb_close(self._db) - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to close database") - self._closed = True - - def create_column_family(self, name: str, config: Optional[ColumnFamilyConfig] = None) -> None: - """ - Create a new column family. - - Args: - name: Name of the column family - config: Configuration for the column family, or None for defaults - """ - if self._closed: - raise TidesDBException("Database is closed") - - if config is None: - config = ColumnFamilyConfig.default() - - cf_name = name.encode() - c_config = config._to_c_struct() - - result = _lib.tidesdb_create_column_family(self._db, cf_name, ctypes.byref(c_config)) - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to create column family") - - def drop_column_family(self, name: str) -> None: - """ - Drop a column family and all its data. - - Args: - name: Name of the column family - """ - if self._closed: - raise TidesDBException("Database is closed") - - cf_name = name.encode() - result = _lib.tidesdb_drop_column_family(self._db, cf_name) - - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to drop column family") - - def get_column_family(self, name: str) -> ColumnFamily: - """ - Get a column family handle. - - Args: - name: Name of the column family - - Returns: - ColumnFamily instance - """ - if self._closed: - raise TidesDBException("Database is closed") - - cf_name = name.encode() - cf_ptr = _lib.tidesdb_get_column_family(self._db, cf_name) - - if not cf_ptr: - raise TidesDBException(f"Column family not found: {name}", ErrorCode.TDB_ERR_NOT_FOUND) - - return ColumnFamily(cf_ptr, name) - - def list_column_families(self) -> List[str]: - """ - List all column families in the database. - - Returns: - List of column family names - """ - if self._closed: - raise TidesDBException("Database is closed") - - names_array_ptr = ctypes.POINTER(c_char_p)() - count = c_int() - - result = _lib.tidesdb_list_column_families(self._db, ctypes.byref(names_array_ptr), ctypes.byref(count)) - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to list column families") - - if count.value == 0: - return [] - - names = [] - - # Copy all strings first before freeing anything - for i in range(count.value): - # names_array_ptr[i] automatically dereferences to get the char* value - name_bytes = names_array_ptr[i] - if name_bytes: - names.append(name_bytes.decode('utf-8')) - - # Now free each string pointer - # We need to reinterpret the array as void pointers to free them - void_ptr_array = ctypes.cast(names_array_ptr, ctypes.POINTER(ctypes.c_void_p)) - for i in range(count.value): - ptr = void_ptr_array[i] - if ptr: - _libc.free(ptr) - - # Free the array itself - _libc.free(ctypes.cast(names_array_ptr, ctypes.c_void_p)) - - return names - - def get_column_family_stats(self, name: str) -> ColumnFamilyStat: - """ - Get statistics for a column family. - - Args: - name: Name of the column family - - Returns: - ColumnFamilyStat instance - """ - if self._closed: - raise TidesDBException("Database is closed") - - cf_name = name.encode() - stats_ptr = POINTER(CColumnFamilyStat)() - - result = _lib.tidesdb_get_column_family_stats(self._db, cf_name, ctypes.byref(stats_ptr)) - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to get column family stats") - - c_stats = stats_ptr.contents - - config = ColumnFamilyConfig( - memtable_flush_size=c_stats.config.memtable_flush_size, - max_sstables_before_compaction=c_stats.config.max_sstables_before_compaction, - compaction_threads=c_stats.config.compaction_threads, - max_level=c_stats.config.max_level, - probability=c_stats.config.probability, - compressed=bool(c_stats.config.compressed), - compress_algo=CompressionAlgo(c_stats.config.compress_algo), - bloom_filter_fp_rate=c_stats.config.bloom_filter_fp_rate, - enable_background_compaction=bool(c_stats.config.enable_background_compaction), - background_compaction_interval=c_stats.config.background_compaction_interval, - use_sbha=bool(c_stats.config.use_sbha), - sync_mode=SyncMode(c_stats.config.sync_mode), - sync_interval=c_stats.config.sync_interval, - ) - - stats = ColumnFamilyStat( - name=c_stats.name.decode('utf-8').rstrip('\x00'), - comparator_name=c_stats.comparator_name.decode('utf-8').rstrip('\x00'), - num_sstables=c_stats.num_sstables, - total_sstable_size=c_stats.total_sstable_size, - memtable_size=c_stats.memtable_size, - memtable_entries=c_stats.memtable_entries, - config=config - ) - - # Free the malloc'd stats structure (C API requires caller to free) - _libc.free(ctypes.cast(stats_ptr, ctypes.c_void_p)) - return stats - - def begin_txn(self) -> Transaction: - """ - Begin a new write transaction. - - Returns: - Transaction instance - """ - if self._closed: - raise TidesDBException("Database is closed") - - txn_ptr = c_void_p() - result = _lib.tidesdb_txn_begin(self._db, ctypes.byref(txn_ptr)) - - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to begin transaction") - - return Transaction(txn_ptr) - - def begin_read_txn(self) -> Transaction: - """ - Begin a new read-only transaction. - - Returns: - Transaction instance - """ - if self._closed: - raise TidesDBException("Database is closed") - - txn_ptr = c_void_p() - result = _lib.tidesdb_txn_begin_read(self._db, ctypes.byref(txn_ptr)) - - if result != ErrorCode.TDB_SUCCESS: - raise TidesDBException.from_code(result, "failed to begin read transaction") - - return Transaction(txn_ptr) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - return False - - def __del__(self): - if not self._closed: - try: - self.close() - except: - pass - - -__all__ = [ - 'TidesDB', - 'Transaction', - 'Iterator', - 'ColumnFamily', - 'ColumnFamilyConfig', - 'ColumnFamilyStat', - 'CompressionAlgo', - 'SyncMode', - 'ErrorCode', - 'TidesDBException', -] \ No newline at end of file diff --git a/venv/bin/Activate.ps1 b/venv/bin/Activate.ps1 deleted file mode 100644 index b49d77b..0000000 --- a/venv/bin/Activate.ps1 +++ /dev/null @@ -1,247 +0,0 @@ -<# -.Synopsis -Activate a Python virtual environment for the current PowerShell session. - -.Description -Pushes the python executable for a virtual environment to the front of the -$Env:PATH environment variable and sets the prompt to signify that you are -in a Python virtual environment. Makes use of the command line switches as -well as the `pyvenv.cfg` file values present in the virtual environment. - -.Parameter VenvDir -Path to the directory that contains the virtual environment to activate. The -default value for this is the parent of the directory that the Activate.ps1 -script is located within. - -.Parameter Prompt -The prompt prefix to display when this virtual environment is activated. By -default, this prompt is the name of the virtual environment folder (VenvDir) -surrounded by parentheses and followed by a single space (ie. '(.venv) '). - -.Example -Activate.ps1 -Activates the Python virtual environment that contains the Activate.ps1 script. - -.Example -Activate.ps1 -Verbose -Activates the Python virtual environment that contains the Activate.ps1 script, -and shows extra information about the activation as it executes. - -.Example -Activate.ps1 -VenvDir C:\Users\MyUser\Common\.venv -Activates the Python virtual environment located in the specified location. - -.Example -Activate.ps1 -Prompt "MyPython" -Activates the Python virtual environment that contains the Activate.ps1 script, -and prefixes the current prompt with the specified string (surrounded in -parentheses) while the virtual environment is active. - -.Notes -On Windows, it may be required to enable this Activate.ps1 script by setting the -execution policy for the user. You can do this by issuing the following PowerShell -command: - -PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - -For more information on Execution Policies: -https://go.microsoft.com/fwlink/?LinkID=135170 - -#> -Param( - [Parameter(Mandatory = $false)] - [String] - $VenvDir, - [Parameter(Mandatory = $false)] - [String] - $Prompt -) - -<# Function declarations --------------------------------------------------- #> - -<# -.Synopsis -Remove all shell session elements added by the Activate script, including the -addition of the virtual environment's Python executable from the beginning of -the PATH variable. - -.Parameter NonDestructive -If present, do not remove this function from the global namespace for the -session. - -#> -function global:deactivate ([switch]$NonDestructive) { - # Revert to original values - - # The prior prompt: - if (Test-Path -Path Function:_OLD_VIRTUAL_PROMPT) { - Copy-Item -Path Function:_OLD_VIRTUAL_PROMPT -Destination Function:prompt - Remove-Item -Path Function:_OLD_VIRTUAL_PROMPT - } - - # The prior PYTHONHOME: - if (Test-Path -Path Env:_OLD_VIRTUAL_PYTHONHOME) { - Copy-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME -Destination Env:PYTHONHOME - Remove-Item -Path Env:_OLD_VIRTUAL_PYTHONHOME - } - - # The prior PATH: - if (Test-Path -Path Env:_OLD_VIRTUAL_PATH) { - Copy-Item -Path Env:_OLD_VIRTUAL_PATH -Destination Env:PATH - Remove-Item -Path Env:_OLD_VIRTUAL_PATH - } - - # Just remove the VIRTUAL_ENV altogether: - if (Test-Path -Path Env:VIRTUAL_ENV) { - Remove-Item -Path env:VIRTUAL_ENV - } - - # Just remove VIRTUAL_ENV_PROMPT altogether. - if (Test-Path -Path Env:VIRTUAL_ENV_PROMPT) { - Remove-Item -Path env:VIRTUAL_ENV_PROMPT - } - - # Just remove the _PYTHON_VENV_PROMPT_PREFIX altogether: - if (Get-Variable -Name "_PYTHON_VENV_PROMPT_PREFIX" -ErrorAction SilentlyContinue) { - Remove-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Scope Global -Force - } - - # Leave deactivate function in the global namespace if requested: - if (-not $NonDestructive) { - Remove-Item -Path function:deactivate - } -} - -<# -.Description -Get-PyVenvConfig parses the values from the pyvenv.cfg file located in the -given folder, and returns them in a map. - -For each line in the pyvenv.cfg file, if that line can be parsed into exactly -two strings separated by `=` (with any amount of whitespace surrounding the =) -then it is considered a `key = value` line. The left hand string is the key, -the right hand is the value. - -If the value starts with a `'` or a `"` then the first and last character is -stripped from the value before being captured. - -.Parameter ConfigDir -Path to the directory that contains the `pyvenv.cfg` file. -#> -function Get-PyVenvConfig( - [String] - $ConfigDir -) { - Write-Verbose "Given ConfigDir=$ConfigDir, obtain values in pyvenv.cfg" - - # Ensure the file exists, and issue a warning if it doesn't (but still allow the function to continue). - $pyvenvConfigPath = Join-Path -Resolve -Path $ConfigDir -ChildPath 'pyvenv.cfg' -ErrorAction Continue - - # An empty map will be returned if no config file is found. - $pyvenvConfig = @{ } - - if ($pyvenvConfigPath) { - - Write-Verbose "File exists, parse `key = value` lines" - $pyvenvConfigContent = Get-Content -Path $pyvenvConfigPath - - $pyvenvConfigContent | ForEach-Object { - $keyval = $PSItem -split "\s*=\s*", 2 - if ($keyval[0] -and $keyval[1]) { - $val = $keyval[1] - - # Remove extraneous quotations around a string value. - if ("'""".Contains($val.Substring(0, 1))) { - $val = $val.Substring(1, $val.Length - 2) - } - - $pyvenvConfig[$keyval[0]] = $val - Write-Verbose "Adding Key: '$($keyval[0])'='$val'" - } - } - } - return $pyvenvConfig -} - - -<# Begin Activate script --------------------------------------------------- #> - -# Determine the containing directory of this script -$VenvExecPath = Split-Path -Parent $MyInvocation.MyCommand.Definition -$VenvExecDir = Get-Item -Path $VenvExecPath - -Write-Verbose "Activation script is located in path: '$VenvExecPath'" -Write-Verbose "VenvExecDir Fullname: '$($VenvExecDir.FullName)" -Write-Verbose "VenvExecDir Name: '$($VenvExecDir.Name)" - -# Set values required in priority: CmdLine, ConfigFile, Default -# First, get the location of the virtual environment, it might not be -# VenvExecDir if specified on the command line. -if ($VenvDir) { - Write-Verbose "VenvDir given as parameter, using '$VenvDir' to determine values" -} -else { - Write-Verbose "VenvDir not given as a parameter, using parent directory name as VenvDir." - $VenvDir = $VenvExecDir.Parent.FullName.TrimEnd("\\/") - Write-Verbose "VenvDir=$VenvDir" -} - -# Next, read the `pyvenv.cfg` file to determine any required value such -# as `prompt`. -$pyvenvCfg = Get-PyVenvConfig -ConfigDir $VenvDir - -# Next, set the prompt from the command line, or the config file, or -# just use the name of the virtual environment folder. -if ($Prompt) { - Write-Verbose "Prompt specified as argument, using '$Prompt'" -} -else { - Write-Verbose "Prompt not specified as argument to script, checking pyvenv.cfg value" - if ($pyvenvCfg -and $pyvenvCfg['prompt']) { - Write-Verbose " Setting based on value in pyvenv.cfg='$($pyvenvCfg['prompt'])'" - $Prompt = $pyvenvCfg['prompt']; - } - else { - Write-Verbose " Setting prompt based on parent's directory's name. (Is the directory name passed to venv module when creating the virtual environment)" - Write-Verbose " Got leaf-name of $VenvDir='$(Split-Path -Path $venvDir -Leaf)'" - $Prompt = Split-Path -Path $venvDir -Leaf - } -} - -Write-Verbose "Prompt = '$Prompt'" -Write-Verbose "VenvDir='$VenvDir'" - -# Deactivate any currently active virtual environment, but leave the -# deactivate function in place. -deactivate -nondestructive - -# Now set the environment variable VIRTUAL_ENV, used by many tools to determine -# that there is an activated venv. -$env:VIRTUAL_ENV = $VenvDir - -if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { - - Write-Verbose "Setting prompt to '$Prompt'" - - # Set the prompt to include the env name - # Make sure _OLD_VIRTUAL_PROMPT is global - function global:_OLD_VIRTUAL_PROMPT { "" } - Copy-Item -Path function:prompt -Destination function:_OLD_VIRTUAL_PROMPT - New-Variable -Name _PYTHON_VENV_PROMPT_PREFIX -Description "Python virtual environment prompt prefix" -Scope Global -Option ReadOnly -Visibility Public -Value $Prompt - - function global:prompt { - Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " - _OLD_VIRTUAL_PROMPT - } - $env:VIRTUAL_ENV_PROMPT = $Prompt -} - -# Clear PYTHONHOME -if (Test-Path -Path Env:PYTHONHOME) { - Copy-Item -Path Env:PYTHONHOME -Destination Env:_OLD_VIRTUAL_PYTHONHOME - Remove-Item -Path Env:PYTHONHOME -} - -# Add the venv to the PATH -Copy-Item -Path Env:PATH -Destination Env:_OLD_VIRTUAL_PATH -$Env:PATH = "$VenvExecDir$([System.IO.Path]::PathSeparator)$Env:PATH" diff --git a/venv/bin/activate b/venv/bin/activate deleted file mode 100644 index 3c12ee0..0000000 --- a/venv/bin/activate +++ /dev/null @@ -1,69 +0,0 @@ -# This file must be used with "source bin/activate" *from bash* -# you cannot run it directly - -deactivate () { - # reset old environment variables - if [ -n "${_OLD_VIRTUAL_PATH:-}" ] ; then - PATH="${_OLD_VIRTUAL_PATH:-}" - export PATH - unset _OLD_VIRTUAL_PATH - fi - if [ -n "${_OLD_VIRTUAL_PYTHONHOME:-}" ] ; then - PYTHONHOME="${_OLD_VIRTUAL_PYTHONHOME:-}" - export PYTHONHOME - unset _OLD_VIRTUAL_PYTHONHOME - fi - - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null - fi - - if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then - PS1="${_OLD_VIRTUAL_PS1:-}" - export PS1 - unset _OLD_VIRTUAL_PS1 - fi - - unset VIRTUAL_ENV - unset VIRTUAL_ENV_PROMPT - if [ ! "${1:-}" = "nondestructive" ] ; then - # Self destruct! - unset -f deactivate - fi -} - -# unset irrelevant variables -deactivate nondestructive - -VIRTUAL_ENV="/home/agpmastersystem/tidesdb-python/venv" -export VIRTUAL_ENV - -_OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/bin:$PATH" -export PATH - -# unset PYTHONHOME if set -# this will fail if PYTHONHOME is set to the empty string (which is bad anyway) -# could use `if (set -u; : $PYTHONHOME) ;` in bash -if [ -n "${PYTHONHOME:-}" ] ; then - _OLD_VIRTUAL_PYTHONHOME="${PYTHONHOME:-}" - unset PYTHONHOME -fi - -if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then - _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="(venv) ${PS1:-}" - export PS1 - VIRTUAL_ENV_PROMPT="(venv) " - export VIRTUAL_ENV_PROMPT -fi - -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting -# past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null -fi diff --git a/venv/bin/activate.csh b/venv/bin/activate.csh deleted file mode 100644 index e2ff4d2..0000000 --- a/venv/bin/activate.csh +++ /dev/null @@ -1,26 +0,0 @@ -# This file must be used with "source bin/activate.csh" *from csh*. -# You cannot run it directly. -# Created by Davide Di Blasi . -# Ported to Python 3.3 venv by Andrew Svetlov - -alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PATH" && unset _OLD_VIRTUAL_PATH; rehash; test $?_OLD_VIRTUAL_PROMPT != 0 && set prompt="$_OLD_VIRTUAL_PROMPT" && unset _OLD_VIRTUAL_PROMPT; unsetenv VIRTUAL_ENV; unsetenv VIRTUAL_ENV_PROMPT; test "\!:*" != "nondestructive" && unalias deactivate' - -# Unset irrelevant variables. -deactivate nondestructive - -setenv VIRTUAL_ENV "/home/agpmastersystem/tidesdb-python/venv" - -set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/bin:$PATH" - - -set _OLD_VIRTUAL_PROMPT="$prompt" - -if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "(venv) $prompt" - setenv VIRTUAL_ENV_PROMPT "(venv) " -endif - -alias pydoc python -m pydoc - -rehash diff --git a/venv/bin/activate.fish b/venv/bin/activate.fish deleted file mode 100644 index 7f4762c..0000000 --- a/venv/bin/activate.fish +++ /dev/null @@ -1,69 +0,0 @@ -# This file must be used with "source /bin/activate.fish" *from fish* -# (https://fishshell.com/); you cannot run it directly. - -function deactivate -d "Exit virtual environment and return to normal shell environment" - # reset old environment variables - if test -n "$_OLD_VIRTUAL_PATH" - set -gx PATH $_OLD_VIRTUAL_PATH - set -e _OLD_VIRTUAL_PATH - end - if test -n "$_OLD_VIRTUAL_PYTHONHOME" - set -gx PYTHONHOME $_OLD_VIRTUAL_PYTHONHOME - set -e _OLD_VIRTUAL_PYTHONHOME - end - - if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - set -e _OLD_FISH_PROMPT_OVERRIDE - # prevents error when using nested fish instances (Issue #93858) - if functions -q _old_fish_prompt - functions -e fish_prompt - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt - end - end - - set -e VIRTUAL_ENV - set -e VIRTUAL_ENV_PROMPT - if test "$argv[1]" != "nondestructive" - # Self-destruct! - functions -e deactivate - end -end - -# Unset irrelevant variables. -deactivate nondestructive - -set -gx VIRTUAL_ENV "/home/agpmastersystem/tidesdb-python/venv" - -set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/bin" $PATH - -# Unset PYTHONHOME if set. -if set -q PYTHONHOME - set -gx _OLD_VIRTUAL_PYTHONHOME $PYTHONHOME - set -e PYTHONHOME -end - -if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" - # fish uses a function instead of an env var to generate the prompt. - - # Save the current fish_prompt function as the function _old_fish_prompt. - functions -c fish_prompt _old_fish_prompt - - # With the original prompt function renamed, we can override with our own. - function fish_prompt - # Save the return status of the last command. - set -l old_status $status - - # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "(venv) " (set_color normal) - - # Restore the return status of the previous command. - echo "exit $old_status" | . - # Output the original/"old" prompt. - _old_fish_prompt - end - - set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" - set -gx VIRTUAL_ENV_PROMPT "(venv) " -end diff --git a/venv/bin/coverage b/venv/bin/coverage deleted file mode 100755 index 9f4bed8..0000000 --- a/venv/bin/coverage +++ /dev/null @@ -1,8 +0,0 @@ -#!/home/agpmastersystem/tidesdb-python/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from coverage.cmdline import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/venv/bin/coverage-3.11 b/venv/bin/coverage-3.11 deleted file mode 100755 index 9f4bed8..0000000 --- a/venv/bin/coverage-3.11 +++ /dev/null @@ -1,8 +0,0 @@ -#!/home/agpmastersystem/tidesdb-python/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from coverage.cmdline import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/venv/bin/coverage3 b/venv/bin/coverage3 deleted file mode 100755 index 9f4bed8..0000000 --- a/venv/bin/coverage3 +++ /dev/null @@ -1,8 +0,0 @@ -#!/home/agpmastersystem/tidesdb-python/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from coverage.cmdline import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/venv/bin/pip b/venv/bin/pip deleted file mode 100755 index 6cba1c9..0000000 --- a/venv/bin/pip +++ /dev/null @@ -1,8 +0,0 @@ -#!/home/agpmastersystem/tidesdb-python/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/venv/bin/pip3 b/venv/bin/pip3 deleted file mode 100755 index 6cba1c9..0000000 --- a/venv/bin/pip3 +++ /dev/null @@ -1,8 +0,0 @@ -#!/home/agpmastersystem/tidesdb-python/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/venv/bin/pip3.11 b/venv/bin/pip3.11 deleted file mode 100755 index 6cba1c9..0000000 --- a/venv/bin/pip3.11 +++ /dev/null @@ -1,8 +0,0 @@ -#!/home/agpmastersystem/tidesdb-python/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from pip._internal.cli.main import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/venv/bin/py.test b/venv/bin/py.test deleted file mode 100755 index a1787d6..0000000 --- a/venv/bin/py.test +++ /dev/null @@ -1,8 +0,0 @@ -#!/home/agpmastersystem/tidesdb-python/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from pytest import console_main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(console_main()) diff --git a/venv/bin/pygmentize b/venv/bin/pygmentize deleted file mode 100755 index 3f12b32..0000000 --- a/venv/bin/pygmentize +++ /dev/null @@ -1,8 +0,0 @@ -#!/home/agpmastersystem/tidesdb-python/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from pygments.cmdline import main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(main()) diff --git a/venv/bin/pytest b/venv/bin/pytest deleted file mode 100755 index a1787d6..0000000 --- a/venv/bin/pytest +++ /dev/null @@ -1,8 +0,0 @@ -#!/home/agpmastersystem/tidesdb-python/venv/bin/python3 -# -*- coding: utf-8 -*- -import re -import sys -from pytest import console_main -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) - sys.exit(console_main()) diff --git a/venv/bin/python b/venv/bin/python deleted file mode 120000 index b8a0adb..0000000 --- a/venv/bin/python +++ /dev/null @@ -1 +0,0 @@ -python3 \ No newline at end of file diff --git a/venv/bin/python3 b/venv/bin/python3 deleted file mode 120000 index ae65fda..0000000 --- a/venv/bin/python3 +++ /dev/null @@ -1 +0,0 @@ -/usr/bin/python3 \ No newline at end of file diff --git a/venv/bin/python3.11 b/venv/bin/python3.11 deleted file mode 120000 index b8a0adb..0000000 --- a/venv/bin/python3.11 +++ /dev/null @@ -1 +0,0 @@ -python3 \ No newline at end of file diff --git a/venv/lib64 b/venv/lib64 deleted file mode 120000 index 7951405..0000000 --- a/venv/lib64 +++ /dev/null @@ -1 +0,0 @@ -lib \ No newline at end of file diff --git a/venv/pyvenv.cfg b/venv/pyvenv.cfg deleted file mode 100644 index 34c62e6..0000000 --- a/venv/pyvenv.cfg +++ /dev/null @@ -1,5 +0,0 @@ -home = /usr/bin -include-system-site-packages = false -version = 3.11.4 -executable = /usr/bin/python3.11 -command = /usr/bin/python3 -m venv /home/agpmastersystem/tidesdb-python/venv