Event indexing: on-chain topics, new indexer service, optimized storage, filtering API, archiving, and query performance monitoring

Cargo.toml

@@ -24,6 +24,7 @@ members = [
     "contracts/third-party",
     "contracts/staking",
     "contracts/governance",
+    "indexer",
 ]
 resolver = "2"
 

Dockerfile.indexer

@@ -0,0 +1,17 @@
+FROM rust:1.76 as builder
+WORKDIR /app
+COPY Cargo.toml ./
+COPY indexer/Cargo.toml indexer/Cargo.toml
+RUN mkdir -p contracts && mkdir -p security-audit && mkdir -p contracts/lib && mkdir -p contracts/traits && mkdir -p contracts/proxy && mkdir -p contracts/escrow && mkdir -p contracts/ipfs-metadata && mkdir -p contracts/oracle && mkdir -p contracts/bridge && mkdir -p contracts/property-token && mkdir -p contracts/insurance && mkdir -p contracts/analytics && mkdir -p contracts/fees && mkdir -p contracts/compliance_registry && mkdir -p contracts/fractional && mkdir -p contracts/prediction-market && mkdir -p contracts/metadata && mkdir -p contracts/database && mkdir -p contracts/third-party && mkdir -p contracts/staking && mkdir -p contracts/governance
+# Create empty Cargo.toml for workspace members to allow cargo to resolve workspace (avoid building them)
+RUN bash -lc 'for d in contracts/* security-audit; do echo -e "[package]\nname=\"dummy-${d//\//-}\"\nversion=\"0.0.0\"\nedition=\"2021\"\n[lib]\npath=\"lib.rs\"\n" > $d/Cargo.toml && echo "" > $d/lib.rs; done'
+COPY indexer /app/indexer
+RUN cargo build -p propchain-indexer --release --features ingest
+
+FROM gcr.io/distroless/cc-debian12
+WORKDIR /app
+COPY --from=builder /app/target/release/propchain-indexer /usr/local/bin/propchain-indexer
+ENV RUST_LOG=info
+EXPOSE 8088
+ENTRYPOINT ["/usr/local/bin/propchain-indexer"]
+

cobertura.xml

@@ -0,0 +1 @@
+<?xml version="1.0"?><coverage lines-covered="145" lines-valid="262" line-rate="0.5534351145038168" branches-covered="0" branches-valid="0" branch-rate="0" complexity="0" version="1.9" timestamp="1774716234"><sources><source>/home/simze/web3-project/PropChain-contract</source></sources><packages><package name="contracts/third-party/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/bridge/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/analytics/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/database/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/metadata/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/ai-valuation/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/prediction-market/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/fractional/src" line-rate="0" branch-rate="0" complexity="0"><classes><class name="lib" filename="contracts/fractional/src/lib.rs" line-rate="0" branch-rate="0" complexity="0"><methods/><lines><line number="62" hits="0"/><line number="77" hits="0"/><line number="82" hits="0"/><line number="87" hits="0"/><line number="107" hits="0"/></lines></class></classes></package><package name="contracts/fees/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/escrow/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/staking/src" line-rate="0.88" branch-rate="0" complexity="0"><classes><class name="lib" filename="contracts/staking/src/lib.rs" line-rate="0.88" branch-rate="0" complexity="0"><methods/><lines><line number="112" hits="4"/><line number="122" hits="1"/><line number="231" hits="0"/><line number="233" hits="0"/><line number="257" hits="0"/><line number="263" hits="0"/><line number="264" hits="3"/><line number="269" hits="0"/><line number="270" hits="1"/><line number="275" hits="0"/><line number="276" hits="2"/><line number="281" hits="2"/><line number="282" hits="1"/><line number="283" hits="1"/><line number="291" hits="2"/><line number="297" hits="0"/><line number="298" hits="1"/><line number="305" hits="5"/><line number="308" hits="1"/><line number="311" hits="1"/><line number="314" hits="3"/><line number="318" hits="2"/><line number="327" hits="4"/><line number="331" hits="4"/><line number="332" hits="4"/><line number="333" hits="4"/><line number="336" hits="4"/><line number="337" hits="4"/><line number="338" hits="4"/><line number="352" hits="2"/><line number="354" hits="4"/><line number="356" hits="1"/><line number="357" hits="1"/><line number="364" hits="1"/><line number="366" hits="1"/><line number="367" hits="2"/><line number="370" hits="1"/><line number="384" hits="2"/><line number="386" hits="4"/><line number="388" hits="1"/><line number="389" hits="1"/><line number="390" hits="0"/><line number="392" hits="1"/><line number="393" hits="0"/><line number="396" hits="1"/><line number="397" hits="1"/><line number="398" hits="1"/><line number="405" hits="1"/><line number="410" hits="2"/><line number="412" hits="4"/><line number="414" hits="3"/><line number="419" hits="1"/><line number="422" hits="1"/><line number="423" hits="1"/><line number="428" hits="2"/><line number="430" hits="1"/><line number="440" hits="2"/><line number="441" hits="1"/><line number="443" hits="1"/><line number="447" hits="1"/><line number="459" hits="2"/><line number="464" hits="2"/><line number="466" hits="1"/><line number="470" hits="1"/><line number="471" hits="1"/><line number="490" hits="1"/><line number="491" hits="1"/><line number="493" hits="1"/><line number="499" hits="2"/><line number="501" hits="1"/><line number="507" hits="1"/><line number="510" hits="1"/><line number="512" hits="1"/><line number="513" hits="2"/><line number="514" hits="2"/></lines></class></classes></package><package name="contracts/traits/src" line-rate="0" branch-rate="0" complexity="0"><classes><class name="errors" filename="contracts/traits/src/errors.rs" line-rate="0" branch-rate="0" complexity="0"><methods/><lines><line number="30" hits="0"/><line number="31" hits="0"/><line number="32" hits="0"/><line number="33" hits="0"/><line number="34" hits="0"/><line number="35" hits="0"/><line number="36" hits="0"/><line number="37" hits="0"/><line number="38" hits="0"/><line number="39" hits="0"/><line number="40" hits="0"/><line number="41" hits="0"/></lines></class></classes></package><package name="tests" line-rate="0" branch-rate="0" complexity="0"><classes><class name="test_utils" filename="tests/test_utils.rs" line-rate="0" branch-rate="0" complexity="0"><methods/><lines><line number="169" hits="0"/><line number="170" hits="0"/><line number="171" hits="0"/><line number="172" hits="0"/><line number="174" hits="0"/><line number="180" hits="0"/><line number="181" hits="0"/><line number="236" hits="0"/><line number="237" hits="0"/><line number="238" hits="0"/><line number="239" hits="0"/><line number="247" hits="0"/><line number="248" hits="0"/><line number="249" hits="0"/><line number="250" hits="0"/></lines></class></classes></package><package name="contracts/lib/src" line-rate="0" branch-rate="0" complexity="0"><classes><class name="error_handling" filename="contracts/lib/src/error_handling.rs" line-rate="0" branch-rate="0" complexity="0"><methods/><lines><line number="76" hits="0"/><line number="77" hits="0"/><line number="80" hits="0"/><line number="81" hits="0"/><line number="87" hits="0"/><line number="88" hits="0"/><line number="89" hits="0"/><line number="99" hits="0"/><line number="100" hits="0"/><line number="101" hits="0"/><line number="299" hits="0"/><line number="300" hits="0"/><line number="301" hits="0"/><line number="302" hits="0"/><line number="304" hits="0"/><line number="308" hits="0"/><line number="310" hits="0"/><line number="311" hits="0"/><line number="312" hits="0"/><line number="313" hits="0"/><line number="325" hits="0"/><line number="326" hits="0"/><line number="327" hits="0"/><line number="328" hits="0"/><line number="330" hits="0"/><line number="340" hits="0"/><line number="341" hits="0"/><line number="342" hits="0"/><line number="343" hits="0"/><line number="345" hits="0"/><line number="356" hits="0"/><line number="357" hits="0"/><line number="358" hits="0"/><line number="359" hits="0"/><line number="361" hits="0"/><line number="374" hits="0"/><line number="375" hits="0"/><line number="376" hits="0"/><line number="377" hits="0"/><line number="379" hits="0"/><line number="380" hits="0"/><line number="392" hits="0"/><line number="393" hits="0"/><line number="394" hits="0"/><line number="395" hits="0"/><line number="396" hits="0"/><line number="397" hits="0"/><line number="399" hits="0"/><line number="400" hits="0"/><line number="412" hits="0"/><line number="413" hits="0"/><line number="414" hits="0"/><line number="415" hits="0"/><line number="416" hits="0"/><line number="417" hits="0"/><line number="419" hits="0"/><line number="420" hits="0"/><line number="422" hits="0"/><line number="423" hits="0"/><line number="426" hits="0"/><line number="437" hits="0"/><line number="438" hits="0"/><line number="439" hits="0"/><line number="440" hits="0"/><line number="441" hits="0"/><line number="442" hits="0"/></lines></class></classes></package><package name="contracts/zk-compliance" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/governance/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/insurance/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/compliance_registry" line-rate="0.8876404494382022" branch-rate="0" complexity="0"><classes><class name="lib" filename="contracts/compliance_registry/lib.rs" line-rate="0.8876404494382022" branch-rate="0" complexity="0"><methods/><lines><line number="489" hits="6"/><line number="492" hits="3"/><line number="512" hits="1"/><line number="593" hits="3"/><line number="603" hits="2"/><line number="605" hits="1"/><line number="610" hits="2"/><line number="618" hits="1"/><line number="623" hits="2"/><line number="624" hits="1"/><line number="625" hits="2"/><line number="656" hits="1"/><line number="677" hits="1"/><line number="687" hits="2"/><line number="691" hits="1"/><line number="692" hits="0"/><line number="693" hits="0"/><line number="697" hits="1"/><line number="698" hits="1"/><line number="699" hits="1"/><line number="708" hits="2"/><line number="709" hits="4"/><line number="710" hits="5"/><line number="712" hits="5"/><line number="715" hits="5"/><line number="718" hits="2"/><line number="726" hits="3"/><line number="727" hits="1"/><line number="731" hits="1"/><line number="734" hits="1"/><line number="748" hits="4"/><line number="754" hits="2"/><line number="756" hits="2"/><line number="757" hits="2"/><line number="758" hits="2"/><line number="766" hits="2"/><line number="767" hits="1"/><line number="769" hits="2"/><line number="771" hits="2"/><line number="778" hits="2"/><line number="788" hits="5"/><line number="794" hits="2"/><line number="796" hits="2"/><line number="797" hits="3"/><line number="798" hits="3"/><line number="799" hits="3"/><line number="800" hits="0"/><line number="806" hits="3"/><line number="837" hits="6"/><line number="840" hits="6"/><line number="844" hits="3"/><line number="845" hits="3"/><line number="846" hits="3"/><line number="856" hits="3"/><line number="971" hits="2"/><line number="980" hits="1"/><line number="981" hits="0"/><line number="983" hits="0"/><line number="988" hits="1"/><line number="989" hits="1"/><line number="1002" hits="1"/><line number="1004" hits="1"/><line number="1011" hits="1"/><line number="1194" hits="2"/><line number="1199" hits="1"/><line number="1202" hits="1"/><line number="1204" hits="1"/><line number="1206" hits="1"/><line number="1208" hits="1"/><line number="1212" hits="1"/><line number="1218" hits="3"/><line number="1229" hits="1"/><line number="1242" hits="2"/><line number="1243" hits="1"/><line number="1244" hits="1"/><line number="1245" hits="1"/><line number="1247" hits="1"/><line number="1248" hits="0"/><line number="1253" hits="2"/><line number="1270" hits="2"/><line number="1271" hits="1"/><line number="1283" hits="0"/><line number="1303" hits="0"/><line number="1304" hits="0"/><line number="1338" hits="2"/><line number="1339" hits="1"/><line number="1346" hits="2"/><line number="1347" hits="1"/><line number="1401" hits="0"/></lines></class></classes></package><package name="indexer/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/property-token/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/proxy/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="security-audit/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/oracle/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/property-management/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package><package name="contracts/ipfs-metadata/src" line-rate="0" branch-rate="0" complexity="0"><classes></classes></package></packages></coverage>

contracts/compliance_registry/lib.rs

@@ -536,6 +536,12 @@ mod compliance_registry {
         pub lists_checked: Vec<u8>,
     }
 
+    impl Default for ComplianceRegistry {
+        fn default() -> Self {
+            Self::new()
+        }
+    }
+
     impl ComplianceRegistry {
         /// Constructor
         #[ink(constructor)]

contracts/database/src/lib.rs

@@ -275,6 +275,7 @@ mod propchain_database {
 
         /// Records an analytics snapshot on-chain for later verification
         #[ink(message)]
+        #[allow(clippy::too_many_arguments)]
         pub fn record_analytics_snapshot(
             &mut self,
             total_properties: u64,
@@ -573,7 +574,6 @@ mod propchain_database {
     // UNIT TESTS
     // ========================================================================
 
-
     // Unit tests extracted to tests.rs (Issue #101)
     include!("tests.rs");
 }

contracts/fractional/src/lib.rs

@@ -64,7 +64,15 @@ mod fractional {
                 last_prices: Mapping::default(),
             }
         }
+    }
 
+    impl Default for Fractional {
+        fn default() -> Self {
+            Self::new()
+        }
+    }
+
+    impl Fractional {
         #[ink(message)]
         pub fn set_last_price(&mut self, token_id: u64, price_per_share: u128) {
             self.last_prices.insert(token_id, &price_per_share);

contracts/property-token/src/lib.rs

@@ -328,6 +328,12 @@ mod property_token {
         pub token_id: TokenId,
     }
 
+    impl Default for PropertyToken {
+        fn default() -> Self {
+            Self::new()
+        }
+    }
+
     impl PropertyToken {
         /// Creates a new PropertyToken contract
         #[ink(constructor)]

contracts/proxy/src/lib.rs

@@ -784,7 +784,6 @@ mod propchain_proxy {
         }
     }
 
-
     // Unit tests extracted to tests.rs (Issue #101)
     include!("tests.rs");
 }

contracts/staking/src/lib.rs

@@ -54,7 +54,9 @@ mod staking {
 
     #[ink(event)]
     pub struct StakingConfigUpdated {
+        #[ink(topic)]
         pub min_stake: u128,
+        #[ink(topic)]
         pub reward_rate_bps: u128,
     }
 

contracts/third-party/src/lib.rs

@@ -527,7 +527,6 @@ mod propchain_third_party {
     // UNIT TESTS
     // ========================================================================
 
-
     // Unit tests extracted to tests.rs (Issue #101)
     include!("tests.rs");
 }

contracts/traits/src/constants.rs

@@ -1,8 +1,8 @@
-// Centralized configuration constants for PropChain contracts.
-//
-// All magic numbers are extracted here with documentation explaining
-// their purpose and valid ranges. Contracts import from this module
-// instead of using inline literals.
+//! Centralized configuration constants for PropChain contracts.
+//!
+//! All magic numbers are extracted here with documentation explaining
+//! their purpose and valid ranges. Contracts import from this module
+//! instead of using inline literals.
 
 // ── Oracle Constants ─────────────────────────────────────────────────────────
 

docker-compose.yml

@@ -58,6 +58,24 @@ services:
     networks:
       - propchain-network
 
+  # Event indexer and API
+  indexer:
+    build:
+      context: .
+      dockerfile: Dockerfile.indexer
+    container_name: propchain-indexer
+    environment:
+      DATABASE_URL: postgres://propchain:propchain123@postgres:5432/propchain
+      SUBSTRATE_WS: ws://substrate-node:9944
+      BIND_ADDR: 0.0.0.0:8088
+    depends_on:
+      - postgres
+      - substrate-node
+    ports:
+      - "8088:8088"
+    networks:
+      - propchain-network
+
 volumes:
   substrate_data:
   ipfs_data:

indexer/Cargo.toml

@@ -0,0 +1,31 @@
+[package]
+name = "propchain-indexer"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+default = []
+ingest = ["subxt"]
+
+[dependencies]
+anyhow = "1.0"
+axum = { version = "0.7", features = ["macros", "json"] }
+axum-prometheus = "0.6"
+chrono = { version = "0.4", features = ["serde"] }
+clap = { version = "4.5", features = ["derive", "env"] }
+futures = "0.3"
+hex = "0.4"
+once_cell = "1.19"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+sqlx = { version = "0.7", features = ["postgres", "runtime-tokio-rustls", "chrono", "uuid"] }
+thiserror = "1.0"
+tokio = { version = "1.37", features = ["rt-multi-thread", "macros", "signal"] }
+tower = "0.4"
+tower-http = { version = "0.5", features = ["trace", "cors"] }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+uuid = { version = "1.8", features = ["v4", "serde"] }
+subxt = { version = "0.33", optional = true }
+url = "2.5"
+

indexer/README.md

@@ -0,0 +1,45 @@
+### PropChain Indexer and Event API
+
+This service ingests on-chain ink! contract events (via `Contracts::ContractEmitted`) and stores them in PostgreSQL with efficient indexes for querying. It also exposes a REST API for filtering and retrieving events, plus Prometheus metrics for performance monitoring.
+
+Setup
+
+- Environment:
+  - `DATABASE_URL` (e.g., postgres://propchain:propchain123@localhost:5432/propchain)
+  - `SUBSTRATE_WS` (e.g., ws://127.0.0.1:9944)
+  - `BIND_ADDR` (default: 0.0.0.0:8088)
+
+Run
+
+```bash
+cargo run -p propchain-indexer
+```
+
+API
+
+- GET /health
+- GET /events
+  - Query params: `contract`, `event_type`, `topic`, `from_ts`, `to_ts`, `from_block`, `to_block`, `limit`, `offset`
+  - `from_ts`/`to_ts` use RFC3339 timestamps
+- GET /metrics (Prometheus)
+
+Storage layout
+
+- Narrow append-only `contract_events` table:
+  - Core columns: `block_number`, `block_hash`, `block_timestamp`, `contract`, `payload_hex`
+  - Optional columns for decoded data: `event_type`, `topics[]`
+  - Composite/time-based indexes for efficient filtering
+
+Archiving strategy
+
+- Primary table sized for near-term queries (e.g., 90 days).
+- Archive older rows to cold storage (separate `events_archive` table or object store) via `scripts/archive-events.sh`.
+- Suggested enhancements:
+  - Postgres monthly partitioning by `block_timestamp` with retention policy
+  - Parquet export to S3 for long-term analytics
+
+Monitoring
+
+- Request metrics exposed at `/metrics`
+- Recommended Grafana dashboard: track p95/p99 query latency, insert throughput, errors
+