Monorepo consolidation: workspace, shared types, transport plans, docker/swam assets
This commit is contained in:
30
projection/.gitignore
vendored
Normal file
30
projection/.gitignore
vendored
Normal file
@@ -0,0 +1,30 @@
|
||||
/target/
|
||||
/target-*/
|
||||
**/target/
|
||||
/data/
|
||||
*.mdbx
|
||||
*.mdbx-lock
|
||||
*.dat
|
||||
*.lck
|
||||
|
||||
.DS_Store
|
||||
|
||||
.env
|
||||
.env.*
|
||||
.envrc
|
||||
.direnv/
|
||||
|
||||
.idea/
|
||||
.vscode/
|
||||
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
docker-compose.override.yml
|
||||
|
||||
/tmp/
|
||||
/coverage/
|
||||
lcov.info
|
||||
*.profraw
|
||||
*.profdata
|
||||
44
projection/COMPATIBILITY_PLAN.md
Normal file
44
projection/COMPATIBILITY_PLAN.md
Normal file
@@ -0,0 +1,44 @@
|
||||
## Goal
|
||||
Make `projection` and `aggregate` compatible to run behind the same gateway and against the same NATS JetStream cluster, with consistent shared types and operational conventions.
|
||||
|
||||
## Constraints
|
||||
- Keep both services independently deployable.
|
||||
- Preserve existing stream/subject conventions unless they are unsafe for multi-instance use.
|
||||
- Prefer backward-compatible changes in message formats (optional fields, tolerant decoding).
|
||||
- Keep changes minimal and verified by fmt/test/clippy.
|
||||
|
||||
## Compatibility Checklist
|
||||
### JetStream / NATS
|
||||
- Stream name: `AGGREGATE_EVENTS` in both services.
|
||||
- Subject pattern: `tenant.*.aggregate.*.*` in both services.
|
||||
- Consumer durability:
|
||||
- Projection: stable durable name (or per-view derived durable).
|
||||
- Aggregate: avoid fixed durable names for ad-hoc fetch operations to prevent collisions across instances.
|
||||
|
||||
### Common Types
|
||||
- `TenantId`: identical newtype behavior across both codebases.
|
||||
- Event payload shape:
|
||||
- Aggregate publishes `types::Event` with `event_id`, `command_id`, `version`, etc.
|
||||
- Projection consumes an `EventEnvelope`; keep required fields stable and add optional fields to mirror aggregate where useful.
|
||||
|
||||
## Plan (Implementation Order)
|
||||
1. Align projection’s event envelope schema to accept and (optionally) emit aggregate-compatible identifiers:
|
||||
- Add optional `event_id`, `command_id`, `version` fields.
|
||||
- Keep decoding tolerant with `#[serde(default)]`.
|
||||
- Avoid changing serialized output unless those fields are present.
|
||||
2. Make aggregate’s JetStream fetch consumer safe for shared clusters:
|
||||
- Generate unique consumer names per fetch call (tenant + aggregate + uuid).
|
||||
- Use explicit ack policy.
|
||||
- Bound fetch loops by timeout/idle to avoid hanging.
|
||||
- Best-effort delete the consumer when done.
|
||||
3. Enforce consistent “strict clippy” hygiene:
|
||||
- Remove trivial `assert!(true)` and other clippy warnings in aggregate so both crates can run `cargo clippy -- -D warnings`.
|
||||
4. Verification:
|
||||
- `cargo fmt --check`
|
||||
- `cargo test`
|
||||
- `cargo clippy --all-targets -- -D warnings`
|
||||
|
||||
## Expected Outcomes
|
||||
- Both binaries can connect to the same JetStream cluster without consumer name collisions.
|
||||
- Projection can decode aggregate-published events and has access to event identifiers/versions when present.
|
||||
- Both repositories share the same strictness level for formatting and linting.
|
||||
36
projection/Cargo.toml
Normal file
36
projection/Cargo.toml
Normal file
@@ -0,0 +1,36 @@
|
||||
[package]
|
||||
name = "projection"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
runtime-v8 = ["v8"]
|
||||
runtime-wasm = []
|
||||
|
||||
[dependencies]
|
||||
shared = { path = "../shared" }
|
||||
edge_storage = { version = "0.1", registry = "madapes" }
|
||||
runtime-function = { version = "0.2", registry = "madapes" }
|
||||
edge-logger-client = { version = "0.1", registry = "madapes" }
|
||||
query_engine = { version = "0.1", registry = "madapes" }
|
||||
async-nats = "0.39"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
serde_yaml = "0.9"
|
||||
toml = "0.8"
|
||||
libmdbx = "0.6"
|
||||
thiserror = "2"
|
||||
anyhow = "1"
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] }
|
||||
uuid = { version = "1", features = ["v7", "serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
futures = "0.3"
|
||||
axum = "0.7"
|
||||
v8 = { version = "0.106", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
tower = "0.5"
|
||||
406
projection/DEVELOPMENT_PLAN.md
Normal file
406
projection/DEVELOPMENT_PLAN.md
Normal file
@@ -0,0 +1,406 @@
|
||||
# Development Plan: Projection Node
|
||||
|
||||
## Overview
|
||||
|
||||
This plan breaks down the Projection node implementation into milestones ordered by dependency. Each milestone includes:
|
||||
- **Tasks** with clear deliverables
|
||||
- **Test Requirements** (unit tests + tautological tests + integration tests where applicable)
|
||||
- **Dependencies** on previous milestones
|
||||
|
||||
**Development Approach:**
|
||||
1. Complete one milestone at a time
|
||||
2. Write tests before implementation (TDD where applicable)
|
||||
3. All tests must pass before moving to the next milestone
|
||||
4. Mark tasks complete with `[x]` as you progress
|
||||
|
||||
---
|
||||
|
||||
## Milestone 1: Project Foundation
|
||||
|
||||
**Goal:** Set up the Rust project with proper structure, dependencies, and basic tooling.
|
||||
|
||||
### Tasks
|
||||
- [x] **1.1** Initialize Cargo project
|
||||
- Create `src/lib.rs` and `src/main.rs`
|
||||
- Configure `Cargo.toml` with madapes registry
|
||||
- [x] **1.2** Configure dependencies
|
||||
- `edge-storage` (KvStore)
|
||||
- `runtime-function` (DAG program execution for `project`)
|
||||
- `query-engine` (UQF query support)
|
||||
- `edge-logger-client` (structured logs client)
|
||||
- `async-nats` (JetStream consumption)
|
||||
- `tokio`, `serde`, `serde_json`, `thiserror`, `anyhow`, `tracing`
|
||||
- [x] **1.3** Establish initial module layout
|
||||
```
|
||||
src/
|
||||
├── lib.rs
|
||||
├── main.rs
|
||||
├── types/
|
||||
│ ├── mod.rs
|
||||
│ ├── id.rs
|
||||
│ ├── event.rs
|
||||
│ ├── view.rs
|
||||
│ ├── checkpoint.rs
|
||||
│ └── error.rs
|
||||
├── config/
|
||||
│ ├── mod.rs
|
||||
│ └── settings.rs
|
||||
├── storage/
|
||||
│ ├── mod.rs
|
||||
│ └── kv.rs
|
||||
├── stream/
|
||||
│ ├── mod.rs
|
||||
│ └── jetstream.rs
|
||||
├── project/
|
||||
│ ├── mod.rs
|
||||
│ ├── runtime.rs
|
||||
│ └── manifest.rs
|
||||
├── query/
|
||||
│ ├── mod.rs
|
||||
│ └── uqf.rs
|
||||
└── observability/
|
||||
└── mod.rs
|
||||
```
|
||||
- [x] **1.4** Configure clippy and rustfmt
|
||||
|
||||
### Tests
|
||||
- [x] **T1.1** Project compiles successfully
|
||||
- [x] **T1.2** Dependencies resolve from madapes registry
|
||||
- [x] **T1.3** Clippy passes with no warnings
|
||||
|
||||
---
|
||||
|
||||
## Milestone 2: Core Types (Envelopes, View Keys, Checkpoints)
|
||||
|
||||
**Goal:** Define all core types required for event consumption, view persistence, and idempotency.
|
||||
|
||||
### Dependencies
|
||||
- Milestone 1 (project foundation)
|
||||
|
||||
### Tasks
|
||||
- [x] **2.1** Implement `TenantId` type
|
||||
- Optional with default empty string for non-multi-tenant setups
|
||||
- Display, FromStr, Serialize, Deserialize
|
||||
- [x] **2.2** Implement `ViewType` and `ViewId` types
|
||||
- String wrappers (consistent with `../aggregate` style: no validation in the type wrapper)
|
||||
- Display, FromStr, Serialize, Deserialize
|
||||
- [x] **2.3** Implement `ViewKey` composition
|
||||
- `view:{tenant_id}:{view_type}:{view_id}`
|
||||
- Centralize formatting/parsing in one place
|
||||
- [x] **2.4** Implement `CheckpointKey` composition
|
||||
- `checkpoint:{tenant_id}:{view_type}`
|
||||
- [x] **2.5** Define event envelope type consumed from JetStream
|
||||
- `tenant_id`, `aggregate_id`, `aggregate_type`, `event_type`, `payload`, `timestamp`
|
||||
- Support forward-compatible decoding (unknown fields ignored)
|
||||
- [x] **2.6** Define checkpoint representation
|
||||
- Persisted value holds JetStream stream sequence (u64) and optional metadata
|
||||
- [x] **2.7** Implement Projection error model
|
||||
- Storage errors, stream errors, decode errors, project errors, tenant access errors
|
||||
- [x] **2.8** Implement `ProjectionManifest` type
|
||||
- Defines projections (`view_type`) and the `project` program reference for each
|
||||
- Validates referenced programs exist
|
||||
- [ ] **2.9** Add correlation/trace context fields to the event envelope (forward compatible)
|
||||
- Support optional `correlation_id` and `traceparent` (or `trace_id`) so logs/traces can be stitched back to Gateway flows
|
||||
- Preserve unknown fields for forward compatibility where practical
|
||||
|
||||
### Tests
|
||||
- [x] **T2.1** `TenantId` round-trips serialization and defaults to empty
|
||||
- [x] **T2.2** `ViewType`, `ViewId` and key composition produce stable strings
|
||||
- [x] **T2.3** Checkpoint encoding/decoding round-trips
|
||||
- [x] **T2.4** Envelope decoding handles unknown fields
|
||||
- [x] **T2.5** Tautological test: core types are Send + Sync
|
||||
|
||||
---
|
||||
|
||||
## Milestone 3: Configuration
|
||||
|
||||
**Goal:** Implement configuration loading and validation for the Projection node.
|
||||
|
||||
### Dependencies
|
||||
- Milestone 2 (core types)
|
||||
|
||||
### Tasks
|
||||
- [x] **3.1** Define `Settings` struct
|
||||
- NATS URL, stream name, subject filter(s)
|
||||
- Durable consumer name strategy (per tenant/view)
|
||||
- Storage path
|
||||
- Multi-tenancy enabled flag + default tenant behavior
|
||||
- Backpressure configuration (max in-flight, batching, ack timeout)
|
||||
- Manifest path (projection definitions and project program refs)
|
||||
- [x] **3.2** Implement config loading from environment
|
||||
- [x] **3.3** Implement config loading from file (YAML/TOML/JSON)
|
||||
- Environment overrides file
|
||||
- [x] **3.4** Implement config validation
|
||||
- Required fields present
|
||||
- Manifest loads and validates at startup (not inside `Settings::validate`)
|
||||
|
||||
### Tests
|
||||
- [x] **T3.1** Settings loads from environment variables
|
||||
- [x] **T3.2** Settings validation catches missing/invalid values
|
||||
- [x] **T3.3** Tautological test: Settings is Clone + Debug
|
||||
|
||||
---
|
||||
|
||||
## Milestone 4: Storage Layer (KvStore Views + Atomic Checkpoints)
|
||||
|
||||
**Goal:** Integrate `edge-storage` `KvStore` with transactionally correct view + checkpoint updates.
|
||||
|
||||
### Dependencies
|
||||
- Milestone 2 (core types)
|
||||
- Milestone 3 (configuration)
|
||||
|
||||
### Tasks
|
||||
- [x] **4.1** Create `KvClient` wrapper
|
||||
- Opens MDBX-backed KvStore at configured path
|
||||
- Tenant-aware key composition helpers
|
||||
- [x] **4.2** Implement view CRUD primitives
|
||||
- `get_view(view_key) -> Option<Value>`
|
||||
- `put_view(view_key, value)`
|
||||
- `delete_view_prefix(tenant_id, view_type)` for rebuilds
|
||||
- [x] **4.3** Implement checkpoint primitives
|
||||
- `get_checkpoint(checkpoint_key) -> Option<Sequence>`
|
||||
- `put_checkpoint(checkpoint_key, sequence)`
|
||||
- [x] **4.4** Implement atomic commit primitive
|
||||
- `txn { put_view(s); put_checkpoint }` as one MDBX transaction
|
||||
- Expose API that makes it hard to update one without the other
|
||||
- [x] **4.5** Optional: storage circuit breaker
|
||||
- Protects node from tight retry loops when storage is degraded
|
||||
|
||||
### Tests
|
||||
- [x] **T4.1** View round-trip: put/get returns identical JSON
|
||||
- [x] **T4.2** Checkpoint round-trip: put/get returns identical sequence
|
||||
- [x] **T4.3** Atomicity: if transaction fails, neither view nor checkpoint is committed
|
||||
- [x] **T4.4** Prefix delete removes all keys for tenant/view_type
|
||||
|
||||
---
|
||||
|
||||
## Milestone 5: JetStream Consumption (Durable Consumer + Idempotency)
|
||||
|
||||
**Goal:** Consume events from NATS JetStream with correct delivery semantics and checkpoint-based idempotency.
|
||||
|
||||
### Dependencies
|
||||
- Milestone 4 (storage layer)
|
||||
|
||||
### Tasks
|
||||
- [x] **5.1** Implement JetStream client wrapper
|
||||
- Connect to NATS and bind to configured stream
|
||||
- Create/bind durable consumer (single filter subject for now)
|
||||
- [x] **5.2** Decode messages into event envelopes
|
||||
- Extract JetStream stream sequence from message metadata
|
||||
- Decode payload into the envelope type
|
||||
- [x] **5.3** Implement idempotency gate
|
||||
- Load checkpoint for `(tenant_id, view_type)`
|
||||
- Skip/ack messages with sequence `<= checkpoint`
|
||||
- [x] **5.4** Implement ack discipline
|
||||
- Ack only after the MDBX transaction commits
|
||||
- Define behavior for transient errors (no ack, allow redelivery)
|
||||
- [x] **5.5** Implement poison-message policy
|
||||
- Enforced via consumer max-deliver + TERM ack on excessive deliveries and KV quarantine record
|
||||
|
||||
### Tests
|
||||
- [x] **T5.1** Unit test: checkpoint gate skips sequences `<= checkpoint`
|
||||
- [x] **T5.2** Unit test: ack is not called when storage commit fails
|
||||
- [x] **T5.3** Integration test: JetStream redelivery re-processes unacked message and is made idempotent by checkpoint (ignored by default; run with `PROJECTION_TEST_NATS_URL=... cargo test -- --ignored`)
|
||||
|
||||
---
|
||||
|
||||
## Milestone 6: Projection Execution (runtime-function `project` Program)
|
||||
|
||||
**Goal:** Apply deterministic projection logic to turn `(current_view, event)` into `new_view`.
|
||||
|
||||
### Dependencies
|
||||
- Milestone 5 (stream consumption)
|
||||
|
||||
### Tasks
|
||||
- [x] **6.1** Implement `project` execution wrapper
|
||||
- Loads program referenced by manifest
|
||||
- Executes deterministically with gas limits + timeouts via `runtime-function`
|
||||
- [x] **6.2** Define projection invocation contract
|
||||
- Input: `{ current_view, event }`
|
||||
- Output: `{ new_view, view_id }` (or equivalent)
|
||||
- [x] **6.3** Implement per-message processing pipeline
|
||||
- Load current view
|
||||
- Execute project program
|
||||
- Atomically write new view + checkpoint
|
||||
- Ack message
|
||||
- [x] **6.4** Enforce per-entity ordering where required
|
||||
- Serialize updates per `(tenant_id, view_type, view_id)` if correctness depends on ordering
|
||||
|
||||
### Tests
|
||||
- [x] **T6.1** Unit test: project program transforms input deterministically (same input → same output)
|
||||
- [x] **T6.2** Unit test: pipeline writes checkpoint only when view write succeeds
|
||||
- [ ] **T6.3** Integration test: concurrent processing does not violate per-key ordering when enabled
|
||||
|
||||
---
|
||||
|
||||
## Milestone 7: Query Support (query-engine UQF)
|
||||
|
||||
**Goal:** Provide query access to stored views using `query-engine` UQF over `KvStore::query()` prefix scans.
|
||||
|
||||
### Dependencies
|
||||
- Milestone 4 (storage layer)
|
||||
|
||||
### Tasks
|
||||
- [x] **7.1** Implement query wrapper around `KvStore` prefix scans
|
||||
- Tenant-scoped prefix scanning
|
||||
- UQF filtering/sorting support
|
||||
- [x] **7.2** Define query interface (library API and/or service endpoint)
|
||||
- Ensure tenant isolation for all query paths
|
||||
- [x] **7.3** Add query-time safeguards
|
||||
- Limits on result size and scan cost
|
||||
- Stable pagination strategy (if required)
|
||||
|
||||
### Tests
|
||||
- [x] **T7.1** Unit test: tenant-scoped queries never return other-tenant keys
|
||||
- [x] **T7.2** Unit test: UQF filter works on a small in-memory fixture dataset
|
||||
|
||||
---
|
||||
|
||||
## Milestone 8: Replay, Rebuild, and Hot Provisioning
|
||||
|
||||
**Goal:** Implement operational workflows: rebuilds, backfills, rolling upgrades, and safety checks.
|
||||
|
||||
### Dependencies
|
||||
- Milestone 5 (JetStream consumption)
|
||||
- Milestone 6 (projection execution)
|
||||
|
||||
### Tasks
|
||||
- [x] **8.1** Implement catch-up mode
|
||||
- When no checkpoint exists, start from sequence 1 and process until tail
|
||||
- [x] **8.2** Implement rebuild workflow
|
||||
- Delete view prefix + checkpoint for `(tenant_id, view_type)`
|
||||
- Re-consume from sequence 1 (or chosen seed sequence)
|
||||
- [x] **8.3** Implement hot upgrade workflow (versioned views)
|
||||
- New `view_type` (or suffix) + independent checkpoint
|
||||
- Backfill then cutover routing, then retire old
|
||||
- [x] **8.4** Add health/readiness signals
|
||||
- Storage reachable
|
||||
- NATS reachable
|
||||
- Consumer lag below threshold (optional)
|
||||
|
||||
### Tests
|
||||
- [x] **T8.1** Integration test: rebuild from scratch produces identical view as uninterrupted consumption (ignored by default; run with `PROJECTION_TEST_NATS_URL=... cargo test -- --ignored`)
|
||||
- [x] **T8.2** Integration test: rolling restart resumes from checkpoint without duplicating results (ignored by default; run with `PROJECTION_TEST_NATS_URL=... cargo test -- --ignored`)
|
||||
|
||||
---
|
||||
|
||||
## Milestone 9: Container & Deployment
|
||||
|
||||
**Goal:** Package as a container and enable predictable local and production deployment.
|
||||
|
||||
### Dependencies
|
||||
- Milestone 8 (replay/rebuild + health/readiness)
|
||||
|
||||
### Tasks
|
||||
- [x] **9.1** Create `docker/Dockerfile.rust`
|
||||
- Multi-stage build
|
||||
- Minimal runtime image
|
||||
- Health check integration
|
||||
- [x] **9.2** Create `docker-compose.yml` for local dev
|
||||
- Projection container
|
||||
- NATS server with JetStream enabled
|
||||
- Optional: Grafana, Victoria Metrics, Loki
|
||||
- [x] **9.3** Create container entrypoint behavior
|
||||
- Config loading
|
||||
- Graceful shutdown on SIGTERM
|
||||
- Stop pulling new JetStream messages
|
||||
- Complete or safely abandon in-flight processing without acking early
|
||||
- [x] **9.4** Define environment variables and defaults
|
||||
- NATS URL, stream name, subject filters
|
||||
- Storage path
|
||||
- Consumer naming strategy (durable)
|
||||
- Multi-tenancy enabled flag
|
||||
- [x] **9.5** Create release build optimization
|
||||
- LTO, strip, single codegen unit
|
||||
|
||||
### Tests
|
||||
- [x] **T9.1** Container builds successfully
|
||||
```bash
|
||||
docker build -f docker/Dockerfile.rust --build-arg PACKAGE=projection --build-arg BIN=projection -t cloudlysis/projection:local .
|
||||
docker run cloudlysis/projection:local --help
|
||||
```
|
||||
- [x] **T9.2** Container starts with valid config
|
||||
```bash
|
||||
docker run -e PROJECTION_NATS_URL=nats://nats:4222 cloudlysis/projection:local
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Milestone 10: Provisioning, Scalability, and Docker Swarm Deployment
|
||||
|
||||
**Goal:** Support horizontal scaling and safe rollouts in Docker Swarm with clear provisioning semantics for JetStream consumers.
|
||||
|
||||
### Dependencies
|
||||
- Milestone 9 (container & deployment)
|
||||
|
||||
### Tasks
|
||||
- [x] **10.1** Define the scaling model for JetStream consumption
|
||||
- Use a durable consumer per `(view_type, shard)` so multiple replicas can share the same consumer workload
|
||||
- Define the subject filter(s) used by each consumer (tenant wildcard vs tenant-range sharded)
|
||||
- Document consumer configuration requirements (ack policy, max in-flight, replay policy)
|
||||
- [x] **10.2** Implement replica-safe consumption
|
||||
- Multiple replicas pulling from the same durable consumer distribute work
|
||||
- Enforce per-key serialization if required for correctness
|
||||
- [x] **10.3** Add tenant-aware provisioning option (sharding)
|
||||
- Optional tenant-range sharding by subject filters (e.g., `tenant.<range>.*`)
|
||||
- Placement constraints for Swarm nodes (e.g., `node.labels.tenant_range==<range>`)
|
||||
- Strategy for adding/removing shards
|
||||
- [x] **10.4** Create Swarm stack definition (`swarm/stacks/platform.yml`)
|
||||
- Service definition
|
||||
- Replicas configuration
|
||||
- Resource limits (CPU, memory)
|
||||
- Health check integration
|
||||
- Storage volume mapping for `edge-storage` data directory
|
||||
- [x] **10.5** Define rollout strategy
|
||||
- Rolling update parameters
|
||||
- Backfill/cutover strategy for versioned `view_type` upgrades
|
||||
- Safe rollback story (old view still present + routing switch back)
|
||||
|
||||
### Tests
|
||||
- [x] **T10.1** Stack file valid
|
||||
```bash
|
||||
docker stack config -c swarm/stacks/platform.yml
|
||||
```
|
||||
- [x] **T10.2** Scale-out does not duplicate work (ignored by default; run with `PROJECTION_TEST_NATS_URL=... cargo test -- --ignored`)
|
||||
- Start 2+ replicas pulling from the same durable consumer
|
||||
- Verify the checkpoint is monotonic and view updates are not applied twice for the same sequence
|
||||
- [x] **T10.3** Rolling restart preserves correctness (ignored by default; run with `PROJECTION_TEST_NATS_URL=... cargo test -- --ignored`)
|
||||
- Restart replicas during active consumption
|
||||
- Verify idempotency holds (no view corruption, checkpoint monotonic)
|
||||
|
||||
---
|
||||
|
||||
## Milestone 11: Operational Endpoints & Observability
|
||||
|
||||
**Goal:** Provide the minimum operational surface required to provision, scale, and monitor Projection nodes in production.
|
||||
|
||||
### Dependencies
|
||||
- Milestone 9 (container & deployment)
|
||||
- Milestone 10 (provisioning & scaling semantics)
|
||||
|
||||
### Tasks
|
||||
- [x] **11.1** Implement `/health` endpoint
|
||||
- Process is up
|
||||
- Storage opened successfully
|
||||
- [x] **11.2** Implement `/ready` endpoint
|
||||
- NATS connection established
|
||||
- JetStream consumer bound
|
||||
- Storage writable
|
||||
- [x] **11.3** Implement `/metrics` endpoint (Prometheus)
|
||||
- Consumer lag (stream sequence - checkpoint)
|
||||
- Processing throughput and latency
|
||||
- Redelivery count / ack failures
|
||||
- Storage commit failures
|
||||
- [x] **11.4** Add build/runtime identity
|
||||
- Version, commit hash (if available), configured `view_type` set
|
||||
- [x] **11.5** Add graceful drain behavior for rollouts
|
||||
- Report “not ready” before shutdown
|
||||
- Stop pulling, wait for in-flight work up to a timeout
|
||||
- [ ] **11.6** Correlation-aware logging for investigations
|
||||
- When processing messages, include `tenant_id`, `correlation_id`, and `trace_id` in structured logs/spans when present in the envelope/headers
|
||||
|
||||
### Tests
|
||||
- [x] **T11.1** Readiness fails when NATS is unavailable
|
||||
- [x] **T11.2** Metrics include lag gauge and counters
|
||||
- [x] **T11.3** Drain transitions ready → not ready before exit
|
||||
- [ ] **T11.4** Unit test: envelope decoding accepts optional correlation/trace fields and exposes them for logging
|
||||
28
projection/docs/scaling.md
Normal file
28
projection/docs/scaling.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# Scaling & Provisioning
|
||||
|
||||
## Consumer Modes
|
||||
|
||||
The projection supports two JetStream consumption modes (configured via `PROJECTION_CONSUMER_MODE`):
|
||||
|
||||
- `single`: one durable consumer per projection process. Each message is processed once and the projection updates all `view_type`s defined in the manifest for that message.
|
||||
- `per_view`: one durable consumer per `view_type`. Each consumer processes all events, but only updates its own `view_type` (checkpoint isolation and independent scaling).
|
||||
|
||||
In `per_view` mode, durable names are derived as:
|
||||
|
||||
`{PROJECTION_DURABLE_NAME}_{view_type}` (with non `[A-Za-z0-9_-]` replaced by `_`).
|
||||
|
||||
## Replica Scaling
|
||||
|
||||
To scale replicas, run multiple instances with the same durable name(s):
|
||||
|
||||
- `single` mode: all replicas share one durable consumer and work-steal messages from that consumer.
|
||||
- `per_view` mode: all replicas share a durable consumer per `view_type`.
|
||||
|
||||
## Sharding
|
||||
|
||||
JetStream subject filtering does not support “hash/range shard by aggregate_id” on the consumer side.
|
||||
|
||||
If strict per-entity ordering is required across replicas, sharding must be encoded in the published subjects (for example `shard.<n>.tenant.<id>.aggregate.<type>.<id>`), and each shard must run with a matching `PROJECTION_SUBJECT_FILTERS` value.
|
||||
|
||||
Without subject-based sharding, multiple replicas can process events for the same `view_id` concurrently, which can break projections that depend on ordered read-modify-write updates.
|
||||
|
||||
192
projection/external_prd.md
Normal file
192
projection/external_prd.md
Normal file
@@ -0,0 +1,192 @@
|
||||
### External PRD: Changes Required in Aggregate, Projection, Runner
|
||||
|
||||
This document captures the work needed outside the Gateway to support:
|
||||
- Tenant-aware routing via `x-tenant-id`
|
||||
- Independent horizontal scalability of Aggregate, Projection, Runner
|
||||
- A safe mechanism for tenant rebalancing per service kind
|
||||
|
||||
---
|
||||
|
||||
## **Target State**
|
||||
|
||||
### Independent Placements
|
||||
|
||||
Each service kind has its own placement map:
|
||||
- `aggregate_placement[tenant_id] -> aggregate_shard_id`
|
||||
- `projection_placement[tenant_id] -> projection_shard_id`
|
||||
- `runner_placement[tenant_id] -> runner_shard_id`
|
||||
|
||||
Each shard is a replica set that can scale independently.
|
||||
|
||||
### Rebalancing Contract (Per Service Kind)
|
||||
|
||||
All nodes MUST support:
|
||||
- Dynamic placement updates (watch NATS KV or reload config)
|
||||
- A drain mechanism that can target a specific tenant (stop acquiring new work for that tenant, finish in-flight, report status)
|
||||
- Clear readiness semantics that reflect whether the node will accept work for a tenant
|
||||
|
||||
Additionally, all nodes SHOULD converge on the same operational contract:
|
||||
- A per-tenant “accepting” gate (can this shard accept new work/queries/commands for tenant X?)
|
||||
- A per-tenant “drained” signal (no in-flight work remains for tenant X)
|
||||
- A per-tenant warmup/catchup signal where relevant (projection lag, aggregate snapshot availability)
|
||||
|
||||
---
|
||||
|
||||
## **Aggregate: Required Changes**
|
||||
|
||||
### 1) Expose a Real Command API (Gateway Upstream)
|
||||
|
||||
Today, Aggregate has internal command handling types (e.g., `CommandServer`) but its running HTTP server only exposes health/metrics/admin endpoints ([aggregate/http_server.rs](file:///Users/vlad/Developer/cloudlysis/aggregate/src/http_server.rs#L15-L82), [aggregate/server/mod.rs](file:///Users/vlad/Developer/cloudlysis/aggregate/src/server/mod.rs#L81-L213)).
|
||||
|
||||
Aggregate MUST expose one of the following upstream APIs for the Gateway to call:
|
||||
- **Option A (Recommended)**: gRPC server implementing `aggregate.gateway.v1.CommandService/SubmitCommand` compatible with [aggregate.proto](file:///Users/vlad/Developer/cloudlysis/aggregate/proto/aggregate.proto#L1-L31).
|
||||
- **Option B**: HTTP endpoint for command submission (REST), with a stable request/response shape that the Gateway can proxy.
|
||||
|
||||
### 2) Tenant Placement Enforcement
|
||||
|
||||
Aggregate MUST enforce “hosted tenants” so independent scaling is safe:
|
||||
- If an Aggregate shard/node is not assigned a tenant, it MUST reject commands for that tenant (e.g., `403` or `503` with retriable hint depending on whether the issue is authorization vs placement).
|
||||
- Aggregate SHOULD maintain an in-memory allowlist of hosted tenants that is driven by:
|
||||
- NATS KV placement watcher (preferred), or
|
||||
- Hot-reloaded config pushed via `/admin/reload`
|
||||
|
||||
Aggregate already has admin hooks for drain/reload, but they are currently generic and/or illustrative ([aggregate/http_server.rs](file:///Users/vlad/Developer/cloudlysis/aggregate/src/http_server.rs#L15-L72), [aggregate/server/mod.rs](file:///Users/vlad/Developer/cloudlysis/aggregate/src/server/mod.rs#L402-L442)). These need to become placement-aware.
|
||||
|
||||
### 3) Tenant Drain (Per Tenant)
|
||||
|
||||
Aggregate MUST provide a per-tenant drain mechanism to support rebalancing:
|
||||
- Stop accepting new commands for the tenant.
|
||||
- Allow in-flight commands to finish (bounded wait), then report drained.
|
||||
- Expose drain status per tenant (admin endpoint).
|
||||
|
||||
### 4) Rebalancing State Strategy
|
||||
|
||||
Aggregate persists snapshots locally (MDBX) and uses JetStream for events. To move a tenant:
|
||||
- **Approach 1 (Snapshot migration)**: copy tenant snapshot DB/state to the target shard, then switch placement.
|
||||
- **Approach 2 (Cold rehydrate)**: switch placement and let the target shard rebuild state by replaying events from JetStream; expect higher latency during warmup.
|
||||
|
||||
The system should support both, with the rebalancer selecting the strategy based on tenant size/SLO.
|
||||
|
||||
### 5) Metrics for Placement Decisions
|
||||
|
||||
Aggregate SHOULD expose:
|
||||
- Per-tenant command rate, error rate
|
||||
- In-flight commands by tenant
|
||||
- Rehydrate time / snapshot hit ratio
|
||||
- Storage size per tenant (if feasible)
|
||||
|
||||
---
|
||||
|
||||
## **Projection: Required Changes**
|
||||
|
||||
### 1) Expose Query API Upstream for Gateway
|
||||
|
||||
Projection has a working `QueryService` with tenant-scoped prefix scans ([uqf.rs](file:///Users/vlad/Developer/cloudlysis/projection/src/query/uqf.rs#L121-L162)) but it is not exposed via HTTP/gRPC (current HTTP routes are health/ready/metrics/info only: [projection/http/mod.rs](file:///Users/vlad/Developer/cloudlysis/projection/src/http/mod.rs#L102-L109)).
|
||||
|
||||
Projection MUST add one upstream API the Gateway can route to:
|
||||
- `POST /query/{view_type}` (HTTP) accepting `x-tenant-id` and a UQF payload, returning `QueryResponse`.
|
||||
- Or a gRPC query service (new proto) if gRPC is preferred end-to-end.
|
||||
|
||||
### 2) Tenant Placement Filtering (Independent Scaling)
|
||||
|
||||
Projection MUST support running in one of these modes:
|
||||
- **Multi-tenant shard**: consumes all tenants (simple, less isolated).
|
||||
- **Tenant-filtered shard (required for rebalancing)**:
|
||||
- only consumes/serves queries for the tenants assigned to that shard
|
||||
- rejects queries for unassigned tenants (consistent error semantics)
|
||||
|
||||
Implementation direction:
|
||||
- Add a placement watcher similar to Runner’s tenant filter ([runner/tenant_placement.rs](file:///Users/vlad/Developer/cloudlysis/runner/src/tenant_placement.rs#L8-L100)).
|
||||
- Apply tenant filter to:
|
||||
- event consumption subject filters (preferred), and
|
||||
- query serving validation (always).
|
||||
|
||||
### 3) Drain + Warmup Endpoints
|
||||
|
||||
Projection SHOULD add:
|
||||
- `/admin/drain?tenant_id=...` (stop consuming new events for that tenant, finish in-flight, flush checkpoints)
|
||||
- `/admin/reload` (apply latest placement/config)
|
||||
- Optional warmup status: whether the shard has caught up to JetStream tail for that tenant/view_types
|
||||
|
||||
### 4) Rebalancing Strategy for Projection
|
||||
|
||||
Projection can rebalance safely with “warm then cut over”:
|
||||
- Assign tenant to the new projection shard while old shard still serves.
|
||||
- New shard catches up (replay from JetStream, build view KV).
|
||||
- Switch Gateway placement for query routing to new shard.
|
||||
- Drain old shard for that tenant and optionally delete old tenant KV keys.
|
||||
|
||||
### 5) Metrics for Placement Decisions
|
||||
|
||||
Projection SHOULD expose:
|
||||
- JetStream lag per tenant/view_type (tail minus checkpoint)
|
||||
- Query latency and scan counts
|
||||
- Storage size per tenant (if feasible)
|
||||
|
||||
---
|
||||
|
||||
## **Runner: Required Changes**
|
||||
|
||||
Runner already has:
|
||||
- A tenant placement watcher capable of producing an allowlist ([tenant_placement.rs](file:///Users/vlad/Developer/cloudlysis/runner/src/tenant_placement.rs#L8-L100))
|
||||
- Admin endpoints including drain/reload/config ([runner/http/mod.rs](file:///Users/vlad/Developer/cloudlysis/runner/src/http/mod.rs#L69-L86))
|
||||
- Gateway client integration for aggregate command submission ([runner/gateway/mod.rs](file:///Users/vlad/Developer/cloudlysis/runner/src/gateway/mod.rs#L1-L47))
|
||||
|
||||
To support independent scalability + rebalancing, Runner needs the following.
|
||||
|
||||
### 1) Per-Tenant Drain (Not Only Global)
|
||||
|
||||
Runner’s current drain is global (`/admin/drain` toggles a single draining flag). Runner MUST support draining a specific tenant:
|
||||
- Stop acquiring new saga/effect work for the tenant.
|
||||
- Allow in-flight work for the tenant to finish (bounded).
|
||||
- Flush outbox for the tenant (or guarantee idempotency on handoff).
|
||||
- Persist final checkpoints so another shard can continue without duplication beyond at-least-once bounds.
|
||||
|
||||
### 2) Placement-Enforced Work Acquisition
|
||||
|
||||
Runner MUST validate tenant assignment at the boundary where it:
|
||||
- consumes JetStream messages (saga triggers, effect commands), and
|
||||
- dispatches outbox work.
|
||||
|
||||
If a tenant is not assigned to the shard, Runner must not process its work.
|
||||
|
||||
### 3) Handoff Safety Rules for Rebalancing
|
||||
|
||||
Runner rebalancing should follow:
|
||||
- New shard begins processing only after it is assigned the tenant.
|
||||
- Old shard stops acquiring new work for that tenant, then drains.
|
||||
- Idempotency remains correct across handoff using checkpoints and dedupe markers.
|
||||
|
||||
### 4) Metrics for Placement Decisions
|
||||
|
||||
Runner SHOULD expose:
|
||||
- Outbox depth by tenant
|
||||
- Work processing latency and retries by tenant/effect
|
||||
- Schedule due items by tenant
|
||||
- Consumer lag by tenant (if the consumption model supports per-tenant lag)
|
||||
|
||||
### 5) Auth Delivery Side Effects (Email/SMS/Push)
|
||||
|
||||
If the platform’s AuthN flows require out-of-band delivery (password reset links, email verification, MFA codes), the Runner SHOULD be the standard place to execute those side effects:
|
||||
- Define a stable effect interface for sending transactional emails (reset links, verification links, security alerts).
|
||||
- Optionally add SMS/push providers later under the same effect contract.
|
||||
|
||||
This keeps the Gateway free of long-lived provider credentials and aligns with the existing “effects are executed by workers” pattern.
|
||||
|
||||
---
|
||||
|
||||
## **Gateway Integration Notes**
|
||||
|
||||
Once the above changes exist:
|
||||
- Gateway routes per `(tenant_id, service_kind)` using independent placement maps.
|
||||
- Gateway can implement “warm then cut over” rebalancing for Projection and Runner by switching only query/workflow routing after readiness conditions are met.
|
||||
- Gateway can enforce consistent tenant validation, authn/authz, and error semantics at the edge even as placements move.
|
||||
|
||||
---
|
||||
|
||||
## **Gaps / Opportunities**
|
||||
|
||||
- **KV schema + ownership**: define the exact NATS KV bucket layout, key naming, revisioning rules, and who is allowed to write placement updates.
|
||||
- **Rebalancer API**: define operator workflows (plan/apply/rollback), status reporting, and audit log requirements for placement changes.
|
||||
- **Shard discovery**: define how shard endpoints are registered (static config vs KV directory entries) and how health is represented.
|
||||
- **Consistency boundaries**: define rebalancing guarantees per service kind (projection can be warm-cutover; runner requires checkpoint handoff; aggregate requires single-writer and state availability).
|
||||
109
projection/prd.md
Normal file
109
projection/prd.md
Normal file
@@ -0,0 +1,109 @@
|
||||
The **Projection** is the "Read Side" of your CQRS (Command Query Responsibility Segregation) architecture. While Aggregates focus on **writing** valid data, Projections focus on **reading** and **formatting** that data for the end-user or application.
|
||||
|
||||
In your framework, Projections are **event-driven views** that transform the stream of facts from **NATS JetStream** into highly optimized, queryable state in `edge-storage` `KvStore`, queryable via the embedded `query-engine` (UQF).
|
||||
|
||||
---
|
||||
|
||||
### 🧱 Component: Projection (Read Model)
|
||||
**Definition:**
|
||||
A Projection is a standalone Rust-based container that consumes Events from **NATS JetStream** and incrementally updates one or more "Read Models" in `edge-storage`. Its sole purpose is to provide a high-performance, pre-computed view of the system state that is optimized for specific queries, bypassing the need to rehydrate Aggregate state or replay event streams at query time.
|
||||
|
||||
**Multi-Tenancy:**
|
||||
The Projection supports optional multi-tenancy via `tenant_id`. When enabled:
|
||||
- **Subject Naming:** JetStream subjects include `tenant_id` (e.g., `tenant.<tenant_id>.aggregate.<aggregate_type>.<aggregate_id>`)
|
||||
- **Storage Namespacing:** Views and checkpoints are namespaced by `tenant_id` to prevent cross-tenant reads
|
||||
- **Query Isolation:** Queries are tenant-scoped (e.g., `x-tenant-id` header) and only scan tenant-prefixed keys
|
||||
- **Backward Compatibility:** Deployments without multi-tenancy use a default/empty `tenant_id`
|
||||
|
||||
**Dependencies:**
|
||||
* Core crates pulled from the custom Cargo registry:
|
||||
```toml
|
||||
[registries.madapes]
|
||||
index = "sparse+https://git.madapes.com/api/packages/madapes/cargo/"
|
||||
```
|
||||
|
||||
| Crate | Purpose |
|
||||
|-------|---------|
|
||||
| `edge-storage` | libmdbx-backed `KvStore` for durable view storage |
|
||||
| `runtime-function` | Deterministic DAG execution for `project` programs |
|
||||
| `edge-logger` | High-performance logging (UDS + Protobuf, Loki sink) |
|
||||
| `query-engine` | UQF query support for filtering/querying view state |
|
||||
| `async-nats` | NATS JetStream client for event consumption |
|
||||
|
||||
* Source code available at `../../madapes/`
|
||||
* **Note:** This is a standalone container — it does not use `event-bus` or gRPC `Consume`/`FetchBatch` APIs
|
||||
|
||||
#### 1. Core Responsibilities
|
||||
* **Event Consumption:** Subscribes to one or more JetStream subjects (typically Aggregate event subjects) using a durable consumer, filtering with subject wildcards.
|
||||
* **State Transformation:** Uses a `project` program (`runtime-function` DAG) to map an incoming event to a state change (e.g., `IncrementCounter`, `UpdateUserEmail`, `AddToList`).
|
||||
* **Read Model Persistence:** Stores the resulting "View" in `edge-storage` `KvStore` as a JSON document, keyed by `view:{tenant_id}:{view_type}:{view_id}` (e.g., `view:tenant_a:UserDashboard:user_123`).
|
||||
* **Query Serving:** Provides read access via `query-engine` UQF queries. The existing `KvStore::query()` integration performs prefix scans and applies UQF filters/sorts.
|
||||
* **Checkpointing:** Tracks its stream position (JetStream stream sequence) in `edge-storage` `KvStore` (key: `checkpoint:{tenant_id}:{view_type}`) to resume correctly after a restart.
|
||||
* **Safe Acknowledgement:** Acks JetStream messages only after the view update and checkpoint are durably committed.
|
||||
|
||||
#### 2. The Lifecycle of a Projection Update
|
||||
1. **Ingestion:** The Projection receives a JetStream message whose payload is a `FrameworkEnvelope` (or equivalent event envelope). It extracts the message metadata (at minimum, the JetStream **stream sequence**) used for idempotency.
|
||||
2. **Context Loading:**
|
||||
* The Projection fetches the current "View" from `edge-storage` `KvStore` (e.g., `kv.get("view:tenant_a:UserDashboard:user_123")`).
|
||||
3. **Transformation (`runtime-function`):**
|
||||
* It executes the `project` DAG program: `(current_view_state, incoming_event) → new_view_state`.
|
||||
* Alternatively, it can use `KvStore::query()` (with `query-engine` UQF) to perform cross-projection lookups to build the new state.
|
||||
4. **Atomic Update:**
|
||||
* The Projection saves the `new_view_state` back to `edge-storage` `KvStore`.
|
||||
* **Critical:** It must save the **checkpoint** (JetStream stream sequence) as part of the same MDBX transaction (e.g., `kv.put_sync("checkpoint:tenant_a:UserDashboard", stream_sequence)`). This ensures crash-recovery correctness.
|
||||
5. **Acknowledge:** After the transaction commits, the Projection acks the JetStream message so it will not be redelivered.
|
||||
6. **Query Availability:** The updated state is immediately available for applications to query via `query-engine` UQF queries.
|
||||
|
||||
#### 3. Technical Constraints & Guarantees
|
||||
* **Eventual Consistency:** Projections are inherently "behind" the Aggregate. There is a sub-second (usually) delay between an event being committed and the Projection reflecting that change.
|
||||
* **Idempotency:** Since JetStream provides **at-least-once** delivery, the Projection must use its stored **Checkpoint** (stream sequence) to ignore events it has already processed.
|
||||
* **Disposable & Rebuildable:** Because JetStream is a durable log, Projections are "disposable." If a business requirement changes, you can delete a Projection's KV entries, create a new `runtime-function` program, and **replay** the entire history from JetStream (starting from sequence 1) to build a new view from scratch.
|
||||
* **Read-Only:** Projections never produce events or commands. They are strictly "sinks" for data.
|
||||
|
||||
#### 4. Replay & Recovery Model
|
||||
* **Catch-up Mode:** When a new Projection is deployed (no checkpoint exists), it starts from the beginning of the JetStream stream (sequence 1) and consumes as fast as possible until it reaches the tail.
|
||||
* **Live Mode:** Once caught up, it continues consuming in real time using the same durable consumer, relying on JetStream acks/redelivery for reliability.
|
||||
|
||||
#### 5. Snapshots (Relationship to Aggregates)
|
||||
The Projection does not require Aggregate snapshots to function, because its source of truth for changes is the JetStream event stream. However, snapshots are still relevant in two ways:
|
||||
|
||||
* **Aggregate Snapshots (Write Side):** Aggregates persist versioned snapshots in `edge-storage` `AggregateStore` to speed up Aggregate rehydration. These snapshots are not a read API for projections and should not be treated as a substitute for consuming events.
|
||||
* **Projection State (Read Side):** A Projection’s stored View in `edge-storage` `KvStore` is effectively its own “snapshot” of the read model at a specific checkpoint (stream sequence).
|
||||
* **Fast Recovery:** On restart, the Projection loads `checkpoint:{tenant_id}:{view_type}`, resumes JetStream consumption from the next sequence, and continues updating existing View records in place. No replay is required unless the checkpoint is missing or the view schema/logic has changed.
|
||||
* **Optional Seeding:** For very large histories, a Projection may optionally seed an initial View state from a recent Aggregate snapshot or an external export, then set its checkpoint to a known JetStream stream sequence and continue consuming events forward from that point. This preserves incremental correctness while reducing rebuild time.
|
||||
|
||||
#### 6. Hot Provisioning (Rolling Scale + Rolling Upgrades)
|
||||
Projections are designed to be provisioned and updated without downtime.
|
||||
|
||||
* **Hot Scale-Out:** Multiple Projection replicas can run concurrently per `tenant_id` and `view_type`. JetStream consumer configuration is used to ensure each event is processed by exactly one replica within a replica set.
|
||||
* **Hot Restart:** A restarted instance resumes from the persisted checkpoint and continues consumption; recovery time is proportional to the gap between the checkpoint and the stream tail.
|
||||
* **Hot Upgrade (Projection Logic):** To change a `project` program safely:
|
||||
* Deploy a new Projection version under a new `view_type` (or `view_type` + version suffix) with its own checkpoint.
|
||||
* Backfill by consuming from sequence 1 (or from a chosen seed sequence) until caught up.
|
||||
* Switch query routing from the old view keys to the new view keys.
|
||||
* Retire old view data and checkpoint after the cutover.
|
||||
* **In-Place Migration:** If the schema change is backward compatible, a Projection may evolve the stored View shape incrementally while processing events, but this requires strict versioning in the View payload.
|
||||
|
||||
#### 7. Caveats & Operational Notes
|
||||
* **Ordering Guarantees:** JetStream preserves ordering per stream, but if the Projection processes messages concurrently it can violate per-entity ordering. If ordering matters for a `view_id`, enforce per-key serialization in the Projection.
|
||||
* **At-Least-Once Reality:** Redeliveries can happen (network splits, ack timeouts, restarts). The Projection must be idempotent via checkpoint checks and/or per-event dedupe keyed by stream sequence.
|
||||
* **Ack Discipline:** Never ack before the MDBX transaction commits. Treat “view update + checkpoint update + ack” as one logical commit.
|
||||
* **Poison Messages:** A single malformed event or incompatible schema can stall a durable consumer. Define a policy for retries, quarantine, and alerting (including whether to skip and record the failure).
|
||||
* **Schema Evolution:** Projection logic must be able to handle old event versions or explicitly version the stream/subjects. Projection View schemas also need versioning if you support in-place migrations.
|
||||
* **Backpressure & Lag:** Catch-up replays can saturate storage and CPU. Monitor consumer lag, redeliveries, and processing latency; apply limits (max in-flight, batching) to protect the node.
|
||||
* **Rebuild Semantics:** Rebuilds must delete both View keys and checkpoints for the target `tenant_id`/`view_type`. Partial deletes can create “mixed era” views.
|
||||
* **Cross-View Lookups:** Using `KvStore::query()` to join across projections is convenient but can amplify read load and introduce consistency anomalies between views. Prefer event-local computation when possible.
|
||||
|
||||
#### 8. Data Structure (The View Envelope)
|
||||
* `view_id`: The unique key for the record (e.g., `user_id`). Used in KvStore key: `view:{tenant_id}:{view_type}:{view_id}`.
|
||||
* `view_type`: The name of the projection (e.g., `active_users_list`).
|
||||
* `last_event_sequence`: The checkpoint (JetStream stream sequence) of the last event processed. Stored separately in `checkpoint:{tenant_id}:{view_type}`.
|
||||
* `data`: The actual payload (JSON) optimized for the UI or API, stored as the KvStore value.
|
||||
|
||||
---
|
||||
|
||||
### 💡 Key Distinction for your PRD:
|
||||
In your framework, the **Projection** is where the "Distributed" part of the system becomes visible to the user.
|
||||
|
||||
* **Aggregates** are for **Consistency** (The Truth).
|
||||
* **Projections** are for **Performance** (The Speed).
|
||||
2
projection/rustfmt.toml
Normal file
2
projection/rustfmt.toml
Normal file
@@ -0,0 +1,2 @@
|
||||
edition = "2021"
|
||||
newline_style = "Unix"
|
||||
3
projection/src/config/mod.rs
Normal file
3
projection/src/config/mod.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
mod settings;
|
||||
|
||||
pub use settings::{ConsumerMode, Settings, SettingsLoadError};
|
||||
325
projection/src/config/settings.rs
Normal file
325
projection/src/config/settings.rs
Normal file
@@ -0,0 +1,325 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct Settings {
|
||||
pub nats_url: String,
|
||||
pub stream_name: String,
|
||||
pub subject_filters: Vec<String>,
|
||||
pub durable_name: String,
|
||||
pub storage_path: String,
|
||||
pub manifest_path: String,
|
||||
pub multi_tenant_enabled: bool,
|
||||
pub default_tenant_id: Option<String>,
|
||||
pub tenant_placement_path: Option<String>,
|
||||
pub max_in_flight: usize,
|
||||
pub ack_timeout_ms: u64,
|
||||
pub max_deliver: i64,
|
||||
pub consumer_mode: ConsumerMode,
|
||||
pub http_addr: String,
|
||||
pub storage_backoff_ms: u64,
|
||||
pub storage_backoff_max_ms: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ConsumerMode {
|
||||
#[default]
|
||||
Single,
|
||||
PerView,
|
||||
}
|
||||
|
||||
impl Default for Settings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
nats_url: "nats://localhost:4222".to_string(),
|
||||
stream_name: "AGGREGATE_EVENTS".to_string(),
|
||||
subject_filters: vec!["tenant.*.aggregate.*.*".to_string()],
|
||||
durable_name: "projection".to_string(),
|
||||
storage_path: "./data".to_string(),
|
||||
manifest_path: "./projection-manifest.yaml".to_string(),
|
||||
multi_tenant_enabled: true,
|
||||
default_tenant_id: None,
|
||||
tenant_placement_path: None,
|
||||
max_in_flight: 128,
|
||||
ack_timeout_ms: 30_000,
|
||||
max_deliver: 10,
|
||||
consumer_mode: ConsumerMode::Single,
|
||||
http_addr: "0.0.0.0:8080".to_string(),
|
||||
storage_backoff_ms: 50,
|
||||
storage_backoff_max_ms: 2_000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
pub fn from_env() -> Result<Self, std::env::VarError> {
|
||||
let mut settings = Self::default();
|
||||
settings.apply_env_overrides();
|
||||
Ok(settings)
|
||||
}
|
||||
|
||||
pub fn from_yaml(yaml: &str) -> Result<Self, serde_yaml::Error> {
|
||||
serde_yaml::from_str(yaml)
|
||||
}
|
||||
|
||||
pub fn from_toml(toml_str: &str) -> Result<Self, toml::de::Error> {
|
||||
toml::from_str(toml_str)
|
||||
}
|
||||
|
||||
pub fn from_json(json: &str) -> Result<Self, serde_json::Error> {
|
||||
serde_json::from_str(json)
|
||||
}
|
||||
|
||||
pub fn from_file(path: impl AsRef<Path>) -> Result<Self, SettingsLoadError> {
|
||||
let path = path.as_ref();
|
||||
let raw = std::fs::read_to_string(path)?;
|
||||
let ext = path.extension().and_then(|e| e.to_str()).unwrap_or("");
|
||||
|
||||
match ext {
|
||||
"yaml" | "yml" => Ok(Self::from_yaml(&raw)?),
|
||||
"toml" => Ok(Self::from_toml(&raw)?),
|
||||
"json" => Ok(Self::from_json(&raw)?),
|
||||
_ => Err(SettingsLoadError::UnsupportedFormat {
|
||||
path: path.display().to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn load_from_file_with_env_overrides(
|
||||
path: impl AsRef<Path>,
|
||||
) -> Result<Self, SettingsLoadError> {
|
||||
let mut settings = Self::from_file(path)?;
|
||||
settings.apply_env_overrides();
|
||||
Ok(settings)
|
||||
}
|
||||
|
||||
fn apply_env_overrides(&mut self) {
|
||||
if let Ok(url) = std::env::var("PROJECTION_NATS_URL") {
|
||||
self.nats_url = url;
|
||||
}
|
||||
|
||||
if let Ok(stream) = std::env::var("PROJECTION_STREAM_NAME") {
|
||||
self.stream_name = stream;
|
||||
}
|
||||
|
||||
if let Ok(filters) = std::env::var("PROJECTION_SUBJECT_FILTERS") {
|
||||
let values = filters
|
||||
.split(',')
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect::<Vec<_>>();
|
||||
if !values.is_empty() {
|
||||
self.subject_filters = values;
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(durable) = std::env::var("PROJECTION_DURABLE_NAME") {
|
||||
self.durable_name = durable;
|
||||
}
|
||||
|
||||
if let Ok(path) = std::env::var("PROJECTION_STORAGE_PATH") {
|
||||
self.storage_path = path;
|
||||
}
|
||||
|
||||
if let Ok(path) = std::env::var("PROJECTION_MANIFEST_PATH") {
|
||||
self.manifest_path = path;
|
||||
}
|
||||
|
||||
if let Ok(enabled) = std::env::var("PROJECTION_MULTI_TENANT") {
|
||||
if let Ok(value) = enabled.parse() {
|
||||
self.multi_tenant_enabled = value;
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(default_tenant_id) = std::env::var("PROJECTION_DEFAULT_TENANT_ID") {
|
||||
if default_tenant_id.is_empty() {
|
||||
self.default_tenant_id = None;
|
||||
} else {
|
||||
self.default_tenant_id = Some(default_tenant_id);
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(path) = std::env::var("PROJECTION_TENANT_PLACEMENT_PATH") {
|
||||
if path.trim().is_empty() {
|
||||
self.tenant_placement_path = None;
|
||||
} else {
|
||||
self.tenant_placement_path = Some(path);
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(max_in_flight) = std::env::var("PROJECTION_MAX_IN_FLIGHT") {
|
||||
if let Ok(value) = max_in_flight.parse() {
|
||||
self.max_in_flight = value;
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(ms) = std::env::var("PROJECTION_ACK_TIMEOUT_MS") {
|
||||
if let Ok(value) = ms.parse() {
|
||||
self.ack_timeout_ms = value;
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(max_deliver) = std::env::var("PROJECTION_MAX_DELIVER") {
|
||||
if let Ok(value) = max_deliver.parse() {
|
||||
self.max_deliver = value;
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(mode) = std::env::var("PROJECTION_CONSUMER_MODE") {
|
||||
self.consumer_mode = match mode.trim().to_ascii_lowercase().as_str() {
|
||||
"single" => ConsumerMode::Single,
|
||||
"per_view" | "per-view" | "perview" => ConsumerMode::PerView,
|
||||
_ => self.consumer_mode,
|
||||
};
|
||||
}
|
||||
|
||||
if let Ok(addr) = std::env::var("PROJECTION_HTTP_ADDR") {
|
||||
if !addr.trim().is_empty() {
|
||||
self.http_addr = addr;
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(ms) = std::env::var("PROJECTION_STORAGE_BACKOFF_MS") {
|
||||
if let Ok(value) = ms.parse() {
|
||||
self.storage_backoff_ms = value;
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(ms) = std::env::var("PROJECTION_STORAGE_BACKOFF_MAX_MS") {
|
||||
if let Ok(value) = ms.parse() {
|
||||
self.storage_backoff_max_ms = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if self.nats_url.is_empty() {
|
||||
return Err("NATS URL is required".to_string());
|
||||
}
|
||||
if self.stream_name.is_empty() {
|
||||
return Err("Stream name is required".to_string());
|
||||
}
|
||||
if self.storage_path.is_empty() {
|
||||
return Err("Storage path is required".to_string());
|
||||
}
|
||||
if self.subject_filters.is_empty() {
|
||||
return Err("At least one subject filter is required".to_string());
|
||||
}
|
||||
if self.durable_name.is_empty() {
|
||||
return Err("Durable name is required".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum SettingsLoadError {
|
||||
#[error("Failed to read config file: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error("Failed to parse YAML config: {0}")]
|
||||
Yaml(#[from] serde_yaml::Error),
|
||||
#[error("Failed to parse TOML config: {0}")]
|
||||
Toml(#[from] toml::de::Error),
|
||||
#[error("Failed to parse JSON config: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
#[error("Unsupported config format: {path}")]
|
||||
UnsupportedFormat { path: String },
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
fn env_lock() -> std::sync::MutexGuard<'static, ()> {
|
||||
static LOCK: std::sync::OnceLock<std::sync::Mutex<()>> = std::sync::OnceLock::new();
|
||||
LOCK.get_or_init(|| std::sync::Mutex::new(()))
|
||||
.lock()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_from_env() {
|
||||
let _guard = env_lock();
|
||||
std::env::set_var("PROJECTION_NATS_URL", "nats://localhost:4222");
|
||||
let settings = Settings::from_env().unwrap();
|
||||
assert_eq!(settings.nats_url, "nats://localhost:4222");
|
||||
std::env::remove_var("PROJECTION_NATS_URL");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn consumer_mode_from_env() {
|
||||
let _guard = env_lock();
|
||||
std::env::set_var("PROJECTION_CONSUMER_MODE", "per_view");
|
||||
let settings = Settings::from_env().unwrap();
|
||||
assert_eq!(settings.consumer_mode, ConsumerMode::PerView);
|
||||
std::env::remove_var("PROJECTION_CONSUMER_MODE");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_addr_from_env() {
|
||||
let _guard = env_lock();
|
||||
std::env::set_var("PROJECTION_HTTP_ADDR", "127.0.0.1:12345");
|
||||
let settings = Settings::from_env().unwrap();
|
||||
assert_eq!(settings.http_addr, "127.0.0.1:12345");
|
||||
std::env::remove_var("PROJECTION_HTTP_ADDR");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn storage_backoff_from_env() {
|
||||
let _guard = env_lock();
|
||||
std::env::set_var("PROJECTION_STORAGE_BACKOFF_MS", "10");
|
||||
std::env::set_var("PROJECTION_STORAGE_BACKOFF_MAX_MS", "20");
|
||||
let settings = Settings::from_env().unwrap();
|
||||
assert_eq!(settings.storage_backoff_ms, 10);
|
||||
assert_eq!(settings.storage_backoff_max_ms, 20);
|
||||
std::env::remove_var("PROJECTION_STORAGE_BACKOFF_MS");
|
||||
std::env::remove_var("PROJECTION_STORAGE_BACKOFF_MAX_MS");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_validation() {
|
||||
let settings = Settings {
|
||||
nats_url: "".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(settings.validate().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_from_yaml_file_and_env_override() {
|
||||
let _guard = env_lock();
|
||||
let dir = tempdir().unwrap();
|
||||
let file_path = dir.path().join("projection.yaml");
|
||||
std::fs::write(
|
||||
&file_path,
|
||||
r#"
|
||||
nats_url: "nats://from-file:4222"
|
||||
stream_name: "AGGREGATE_EVENTS"
|
||||
subject_filters:
|
||||
- "tenant.*.aggregate.*.*"
|
||||
storage_path: "/tmp/proj"
|
||||
durable_name: "proj"
|
||||
multi_tenant_enabled: false
|
||||
"#,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
std::env::set_var("PROJECTION_NATS_URL", "nats://from-env:4222");
|
||||
let settings = Settings::load_from_file_with_env_overrides(&file_path).unwrap();
|
||||
assert_eq!(settings.nats_url, "nats://from-env:4222");
|
||||
assert_eq!(settings.storage_path, "/tmp/proj");
|
||||
assert_eq!(settings.durable_name, "proj");
|
||||
assert!(!settings.multi_tenant_enabled);
|
||||
std::env::remove_var("PROJECTION_NATS_URL");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settings_is_clone() {
|
||||
let s = Settings::default();
|
||||
let _s2 = s.clone();
|
||||
let _ = format!("{:?}", s);
|
||||
}
|
||||
}
|
||||
714
projection/src/http/mod.rs
Normal file
714
projection/src/http/mod.rs
Normal file
@@ -0,0 +1,714 @@
|
||||
use crate::config::Settings;
|
||||
use crate::observability::Observability;
|
||||
use crate::project::ProjectionManifest;
|
||||
use crate::query::{QueryError, QueryRequest, QueryService};
|
||||
use crate::storage::KvClient;
|
||||
use crate::tenant_placement::TenantPlacement;
|
||||
use crate::types::{CheckpointKey, ProjectionError, TenantId, ViewType};
|
||||
use async_nats::jetstream;
|
||||
use axum::extract::{Path, State};
|
||||
use axum::http::{header, HeaderMap, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
use axum::routing::{get, post};
|
||||
use axum::{Json, Router};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct AppState {
|
||||
pub settings: Settings,
|
||||
pub ready: Arc<AtomicBool>,
|
||||
pub draining: Arc<AtomicBool>,
|
||||
pub observability: Observability,
|
||||
pub storage: KvClient,
|
||||
pub manifest: ProjectionManifest,
|
||||
pub jetstream: Option<jetstream::Context>,
|
||||
pub tenant_placement: TenantPlacement,
|
||||
pub query: QueryService,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for AppState {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("AppState")
|
||||
.field("settings", &self.settings)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn build_state(
|
||||
settings: Settings,
|
||||
ready: Arc<AtomicBool>,
|
||||
draining: Arc<AtomicBool>,
|
||||
observability: Observability,
|
||||
tenant_placement: TenantPlacement,
|
||||
) -> Result<AppState, ProjectionError> {
|
||||
let storage = KvClient::open(settings.storage_path.clone())
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
|
||||
let manifest_raw = std::fs::read_to_string(&settings.manifest_path)
|
||||
.map_err(|e| ProjectionError::ManifestError(e.to_string()))?;
|
||||
let ext = std::path::Path::new(&settings.manifest_path)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or("");
|
||||
let manifest = match ext {
|
||||
"yaml" | "yml" => ProjectionManifest::load_from_yaml(&manifest_raw)
|
||||
.map_err(|e| ProjectionError::ManifestError(e.to_string()))?,
|
||||
"json" => ProjectionManifest::load_from_json(&manifest_raw)
|
||||
.map_err(|e| ProjectionError::ManifestError(e.to_string()))?,
|
||||
_ => {
|
||||
return Err(ProjectionError::ManifestError(format!(
|
||||
"Unsupported manifest format: {}",
|
||||
settings.manifest_path
|
||||
)));
|
||||
}
|
||||
};
|
||||
manifest.validate()?;
|
||||
|
||||
let jetstream = match async_nats::connect(&settings.nats_url).await {
|
||||
Ok(client) => Some(jetstream::new(client)),
|
||||
Err(_) => None,
|
||||
};
|
||||
|
||||
let query = QueryService::new(storage.clone());
|
||||
|
||||
Ok(AppState {
|
||||
settings,
|
||||
ready,
|
||||
draining,
|
||||
observability,
|
||||
storage,
|
||||
manifest,
|
||||
jetstream,
|
||||
tenant_placement,
|
||||
query,
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn serve(
|
||||
state: AppState,
|
||||
shutdown: Arc<tokio::sync::Notify>,
|
||||
) -> Result<(), ProjectionError> {
|
||||
let addr = state.settings.http_addr.clone();
|
||||
let app = router(state);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(&addr)
|
||||
.await
|
||||
.map_err(|e| ProjectionError::StreamError(e.to_string()))?;
|
||||
|
||||
axum::serve(listener, app)
|
||||
.with_graceful_shutdown(async move {
|
||||
shutdown.notified().await;
|
||||
})
|
||||
.await
|
||||
.map_err(|e| ProjectionError::StreamError(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn router(state: AppState) -> Router {
|
||||
Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/ready", get(ready))
|
||||
.route("/metrics", get(metrics))
|
||||
.route("/info", get(info))
|
||||
.route("/query/:view_type", post(query))
|
||||
.route("/admin/status", get(admin_status))
|
||||
.route("/admin/drain", post(admin_drain))
|
||||
.route("/admin/reload", post(admin_reload))
|
||||
.route("/admin/warmup", get(admin_warmup))
|
||||
.with_state(state)
|
||||
}
|
||||
|
||||
async fn health(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let key = format!("health:{}", uuid::Uuid::now_v7());
|
||||
let value = json!({"ok": true});
|
||||
let writable = state
|
||||
.storage
|
||||
.put_json(&key, &value)
|
||||
.and_then(|_| state.storage.delete_key(key.as_bytes()));
|
||||
|
||||
match writable {
|
||||
Ok(()) => (StatusCode::OK, Json(json!({"status": "ok"}))).into_response(),
|
||||
Err(e) => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({"status": "error", "error": e.to_string()})),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn ready(State(state): State<AppState>) -> impl IntoResponse {
|
||||
if state.draining.load(Ordering::Relaxed) {
|
||||
return (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({"ready": false})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
if !state.ready.load(Ordering::Relaxed) {
|
||||
return (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({"ready": false})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
|
||||
let key = format!("ready:{}", uuid::Uuid::now_v7());
|
||||
let value = json!({"ok": true});
|
||||
if state
|
||||
.storage
|
||||
.put_json(&key, &value)
|
||||
.and_then(|_| state.storage.delete_key(key.as_bytes()))
|
||||
.is_err()
|
||||
{
|
||||
return (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({"ready": false})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
|
||||
let connect = async {
|
||||
let client = async_nats::connect(&state.settings.nats_url).await?;
|
||||
let js = jetstream::new(client);
|
||||
let stream = js.get_stream(&state.settings.stream_name).await?;
|
||||
let mut stream = stream;
|
||||
let _ = stream.info().await?;
|
||||
Ok::<(), async_nats::Error>(())
|
||||
};
|
||||
|
||||
match tokio::time::timeout(Duration::from_millis(800), connect).await {
|
||||
Ok(Ok(())) => (StatusCode::OK, Json(json!({"ready": true}))).into_response(),
|
||||
_ => (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({"ready": false})),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn info(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let view_types = state
|
||||
.manifest
|
||||
.all()
|
||||
.map(|d| d.view_type.as_str().to_string())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let build = json!({
|
||||
"name": env!("CARGO_PKG_NAME"),
|
||||
"version": env!("CARGO_PKG_VERSION"),
|
||||
"git_sha": option_env!("GIT_SHA").unwrap_or("unknown"),
|
||||
});
|
||||
|
||||
let payload = json!({
|
||||
"build": build,
|
||||
"stream_name": state.settings.stream_name,
|
||||
"durable_name": state.settings.durable_name,
|
||||
"subject_filters": state.settings.subject_filters,
|
||||
"consumer_mode": format!("{:?}", state.settings.consumer_mode).to_ascii_lowercase(),
|
||||
"view_types": view_types,
|
||||
"ready": state.ready.load(Ordering::Relaxed),
|
||||
"draining": state.draining.load(Ordering::Relaxed),
|
||||
});
|
||||
|
||||
(StatusCode::OK, Json(payload)).into_response()
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct QueryBody {
|
||||
uqf: String,
|
||||
}
|
||||
|
||||
async fn query(
|
||||
State(state): State<AppState>,
|
||||
Path(view_type): Path<String>,
|
||||
headers: HeaderMap,
|
||||
Json(body): Json<QueryBody>,
|
||||
) -> impl IntoResponse {
|
||||
let tenant_id = match tenant_from_headers(&state.settings, &headers) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
return (e.status, Json(json!({"error": e.message}))).into_response();
|
||||
}
|
||||
};
|
||||
|
||||
if state.tenant_placement.is_draining(&tenant_id) {
|
||||
return (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({"error": "tenant is draining", "tenant_id": tenant_id.as_str()})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
|
||||
if !state.tenant_placement.is_hosted(&tenant_id) {
|
||||
return (
|
||||
StatusCode::FORBIDDEN,
|
||||
Json(json!({"error": "tenant not hosted on this shard", "tenant_id": tenant_id.as_str()})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
|
||||
let view_type = ViewType::new(view_type);
|
||||
if state.manifest.get(&view_type).is_none() {
|
||||
return (
|
||||
StatusCode::NOT_FOUND,
|
||||
Json(json!({"error": "unknown view type", "view_type": view_type.as_str()})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
|
||||
let request = QueryRequest {
|
||||
tenant_id,
|
||||
view_type,
|
||||
uqf: body.uqf,
|
||||
};
|
||||
|
||||
match state.query.query(request) {
|
||||
Ok(resp) => (StatusCode::OK, Json(resp)).into_response(),
|
||||
Err(QueryError::InvalidQuery(e)) => {
|
||||
(StatusCode::BAD_REQUEST, Json(json!({"error": e}))).into_response()
|
||||
}
|
||||
Err(QueryError::Execution(e)) => {
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, Json(json!({"error": e}))).into_response()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct TenantHeaderError {
|
||||
status: StatusCode,
|
||||
message: &'static str,
|
||||
}
|
||||
|
||||
fn tenant_from_headers(
|
||||
settings: &Settings,
|
||||
headers: &HeaderMap,
|
||||
) -> Result<TenantId, TenantHeaderError> {
|
||||
let header_value = headers
|
||||
.get("x-tenant-id")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.map(|s| s.trim())
|
||||
.unwrap_or("");
|
||||
|
||||
if settings.multi_tenant_enabled {
|
||||
if header_value.is_empty() {
|
||||
if let Some(default) = &settings.default_tenant_id {
|
||||
return Ok(TenantId::new(default));
|
||||
}
|
||||
return Err(TenantHeaderError {
|
||||
status: StatusCode::BAD_REQUEST,
|
||||
message: "missing x-tenant-id",
|
||||
});
|
||||
}
|
||||
return Ok(TenantId::new(header_value));
|
||||
}
|
||||
|
||||
if let Some(default) = &settings.default_tenant_id {
|
||||
return Ok(TenantId::new(default));
|
||||
}
|
||||
|
||||
Ok(TenantId::new(header_value))
|
||||
}
|
||||
|
||||
async fn admin_status(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let snapshot = state.tenant_placement.snapshot();
|
||||
let payload = json!({
|
||||
"ready": state.ready.load(Ordering::Relaxed),
|
||||
"draining": state.draining.load(Ordering::Relaxed),
|
||||
"placement": snapshot,
|
||||
});
|
||||
(StatusCode::OK, Json(payload)).into_response()
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct DrainParams {
|
||||
tenant_id: String,
|
||||
#[serde(default)]
|
||||
draining: Option<bool>,
|
||||
}
|
||||
|
||||
async fn admin_drain(
|
||||
State(state): State<AppState>,
|
||||
axum::extract::Query(req): axum::extract::Query<DrainParams>,
|
||||
) -> impl IntoResponse {
|
||||
let tenant_id = TenantId::new(req.tenant_id);
|
||||
let draining = req.draining.unwrap_or(true);
|
||||
match state.tenant_placement.set_draining(tenant_id, draining) {
|
||||
Ok(()) => (
|
||||
StatusCode::OK,
|
||||
Json(json!({"ok": true, "placement": state.tenant_placement.snapshot()})),
|
||||
)
|
||||
.into_response(),
|
||||
Err(e) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({"ok": false, "error": e})),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn admin_reload(State(state): State<AppState>) -> impl IntoResponse {
|
||||
match state.tenant_placement.reload(&state.settings) {
|
||||
Ok(()) => (
|
||||
StatusCode::OK,
|
||||
Json(json!({"ok": true, "placement": state.tenant_placement.snapshot()})),
|
||||
)
|
||||
.into_response(),
|
||||
Err(e) => (
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
Json(json!({"ok": false, "error": e})),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct WarmupParams {
|
||||
tenant_id: Option<String>,
|
||||
}
|
||||
|
||||
async fn admin_warmup(
|
||||
State(state): State<AppState>,
|
||||
axum::extract::Query(params): axum::extract::Query<WarmupParams>,
|
||||
headers: HeaderMap,
|
||||
) -> impl IntoResponse {
|
||||
let tenant_id = if let Some(t) = params.tenant_id {
|
||||
TenantId::new(t)
|
||||
} else {
|
||||
match tenant_from_headers(&state.settings, &headers) {
|
||||
Ok(t) => t,
|
||||
Err(e) => return (e.status, Json(json!({"error": e.message}))).into_response(),
|
||||
}
|
||||
};
|
||||
|
||||
let Some(js) = &state.jetstream else {
|
||||
return (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({"error": "nats unavailable"})),
|
||||
)
|
||||
.into_response();
|
||||
};
|
||||
|
||||
let stream = match js.get_stream(&state.settings.stream_name).await {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
return (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({"error": e.to_string()})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
|
||||
let mut stream = stream;
|
||||
let info = match stream.info().await {
|
||||
Ok(i) => i,
|
||||
Err(e) => {
|
||||
return (
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
Json(json!({"error": e.to_string()})),
|
||||
)
|
||||
.into_response();
|
||||
}
|
||||
};
|
||||
let tail = info.state.last_sequence;
|
||||
|
||||
let mut views = Vec::new();
|
||||
for def in state.manifest.all() {
|
||||
let ck = CheckpointKey::new(&tenant_id, &def.view_type);
|
||||
let cp = state
|
||||
.storage
|
||||
.get_checkpoint(&ck)
|
||||
.unwrap_or(None)
|
||||
.unwrap_or(0);
|
||||
let lag = tail.saturating_sub(cp);
|
||||
views.push(json!({
|
||||
"view_type": def.view_type.as_str(),
|
||||
"checkpoint": cp,
|
||||
"lag": lag
|
||||
}));
|
||||
}
|
||||
|
||||
let payload = json!({
|
||||
"tenant_id": tenant_id.as_str(),
|
||||
"stream_name": state.settings.stream_name,
|
||||
"tail": tail,
|
||||
"views": views
|
||||
});
|
||||
|
||||
(StatusCode::OK, Json(payload)).into_response()
|
||||
}
|
||||
|
||||
async fn metrics(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let mut output = state.observability.export_metrics();
|
||||
output.push('\n');
|
||||
|
||||
let ready_value =
|
||||
if state.ready.load(Ordering::Relaxed) && !state.draining.load(Ordering::Relaxed) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
output.push_str("# HELP projection_ready Projection readiness (1=ready,0=not ready)\n");
|
||||
output.push_str("# TYPE projection_ready gauge\n");
|
||||
output.push_str(&format!("projection_ready {}\n", ready_value));
|
||||
|
||||
if let Some(js) = &state.jetstream {
|
||||
if let Ok(stream) = js.get_stream(&state.settings.stream_name).await {
|
||||
let mut stream = stream;
|
||||
if let Ok(info) = stream.info().await {
|
||||
let tail = info.state.last_sequence;
|
||||
output.push_str(
|
||||
"\n# HELP projection_stream_last_sequence JetStream stream tail sequence\n",
|
||||
);
|
||||
output.push_str("# TYPE projection_stream_last_sequence gauge\n");
|
||||
output.push_str(&format!("projection_stream_last_sequence {}\n", tail));
|
||||
|
||||
let tenant_id =
|
||||
TenantId::new(state.settings.default_tenant_id.clone().unwrap_or_default());
|
||||
|
||||
output.push_str(
|
||||
"\n# HELP projection_lag Stream tail sequence minus checkpoint per view_type\n",
|
||||
);
|
||||
output.push_str("# TYPE projection_lag gauge\n");
|
||||
for def in state.manifest.all() {
|
||||
let ck = CheckpointKey::new(&tenant_id, &def.view_type);
|
||||
let cp = state
|
||||
.storage
|
||||
.get_checkpoint(&ck)
|
||||
.unwrap_or(None)
|
||||
.unwrap_or(0);
|
||||
let lag = tail.saturating_sub(cp);
|
||||
output.push_str(&format!(
|
||||
"projection_lag{{tenant_id=\"{}\",view_type=\"{}\"}} {}\n",
|
||||
tenant_id.as_str(),
|
||||
def.view_type.as_str(),
|
||||
lag
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
(
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "text/plain; version=0.0.4")],
|
||||
output,
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::body::Body;
|
||||
use axum::http::Request;
|
||||
use query_engine::FilterNode;
|
||||
use query_engine::Query;
|
||||
use serde_json::json;
|
||||
use tower::ServiceExt;
|
||||
|
||||
fn test_state() -> AppState {
|
||||
test_state_with_nats_url("nats://127.0.0.1:65535")
|
||||
}
|
||||
|
||||
fn test_state_with_nats_url(nats_url: &str) -> AppState {
|
||||
let settings = Settings {
|
||||
nats_url: nats_url.to_string(),
|
||||
..Settings::default()
|
||||
};
|
||||
let ready = Arc::new(AtomicBool::new(false));
|
||||
let draining = Arc::new(AtomicBool::new(false));
|
||||
let observability = Observability::default();
|
||||
let storage = KvClient::in_memory();
|
||||
let tenant_placement = TenantPlacement::load(&settings).unwrap();
|
||||
let query = QueryService::new(storage.clone());
|
||||
let mut manifest = ProjectionManifest::new();
|
||||
manifest.register(crate::project::ProjectionDefinition {
|
||||
view_type: crate::types::ViewType::new("User"),
|
||||
project_program: "/tmp/prog".to_string(),
|
||||
});
|
||||
|
||||
AppState {
|
||||
settings,
|
||||
ready,
|
||||
draining,
|
||||
observability,
|
||||
storage,
|
||||
manifest,
|
||||
jetstream: None,
|
||||
tenant_placement,
|
||||
query,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_returns_503_when_not_ready() {
|
||||
let state = test_state();
|
||||
let app = router(state);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/ready")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_requires_tenant_header_in_multi_tenant_mode() {
|
||||
let state = test_state();
|
||||
let app = router(state);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/query/User")
|
||||
.header("content-type", "application/json")
|
||||
.body(Body::from(r#"{"uqf":"{}"}"#))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn query_returns_hits_for_hosted_tenant() {
|
||||
let mut state = test_state();
|
||||
state.settings.multi_tenant_enabled = true;
|
||||
state.tenant_placement = TenantPlacement::load(&state.settings).unwrap();
|
||||
|
||||
let tenant_id = TenantId::new("t1");
|
||||
let view_type = ViewType::new("User");
|
||||
let view_id = crate::types::ViewId::new("u1");
|
||||
let view_key = crate::types::ViewKey::new(&tenant_id, &view_type, &view_id);
|
||||
let cp_key = CheckpointKey::new(&tenant_id, &view_type);
|
||||
state
|
||||
.storage
|
||||
.commit_view_and_checkpoint(&view_key, &json!({"x": 1}), &cp_key, 1)
|
||||
.unwrap();
|
||||
|
||||
let uqf = Query::new(FilterNode::r#true()).to_json().unwrap();
|
||||
let app = router(state);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.method("POST")
|
||||
.uri("/query/User")
|
||||
.header("content-type", "application/json")
|
||||
.header("x-tenant-id", "t1")
|
||||
.body(Body::from(
|
||||
serde_json::to_vec(&json!({"uqf": uqf})).unwrap(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_returns_503_when_nats_unavailable() {
|
||||
let state = test_state_with_nats_url("nats://127.0.0.1:65535");
|
||||
state.ready.store(true, Ordering::Relaxed);
|
||||
let app = router(state);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/ready")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn ready_returns_200_when_nats_and_stream_available() {
|
||||
let Ok(nats_url) = std::env::var("PROJECTION_TEST_NATS_URL") else {
|
||||
return;
|
||||
};
|
||||
|
||||
let id = uuid::Uuid::now_v7().to_string();
|
||||
let stream_name = format!("projection_ready_test_{}", id);
|
||||
let subject = format!("tenant.t1.aggregate.Account.{}", id);
|
||||
|
||||
let client = async_nats::connect(&nats_url).await.unwrap();
|
||||
let js = async_nats::jetstream::new(client);
|
||||
let _stream = js
|
||||
.get_or_create_stream(async_nats::jetstream::stream::Config {
|
||||
name: stream_name.clone(),
|
||||
subjects: vec![subject],
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut state = test_state_with_nats_url(&nats_url);
|
||||
state.settings.stream_name = stream_name;
|
||||
state.ready.store(true, Ordering::Relaxed);
|
||||
|
||||
let app = router(state);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/ready")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_returns_503_when_draining() {
|
||||
let state = test_state();
|
||||
state.ready.store(true, Ordering::Relaxed);
|
||||
state.draining.store(true, Ordering::Relaxed);
|
||||
let app = router(state);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/ready")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metrics_includes_ready_gauge() {
|
||||
let state = test_state();
|
||||
state.ready.store(true, Ordering::Relaxed);
|
||||
let app = router(state);
|
||||
let response = app
|
||||
.oneshot(
|
||||
Request::builder()
|
||||
.uri("/metrics")
|
||||
.body(Body::empty())
|
||||
.unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
|
||||
let body = axum::body::to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.unwrap();
|
||||
let text = String::from_utf8(body.to_vec()).unwrap();
|
||||
assert!(text.contains("projection_ready 1"));
|
||||
}
|
||||
}
|
||||
17
projection/src/lib.rs
Normal file
17
projection/src/lib.rs
Normal file
@@ -0,0 +1,17 @@
|
||||
pub mod config;
|
||||
pub mod http;
|
||||
pub mod observability;
|
||||
pub mod project;
|
||||
pub mod query;
|
||||
pub mod storage;
|
||||
pub mod stream;
|
||||
pub mod tenant_placement;
|
||||
pub mod types;
|
||||
|
||||
pub use config::Settings;
|
||||
pub use observability::Observability;
|
||||
pub use project::{ProjectionManifest, ProjectionRuntime};
|
||||
pub use storage::KvClient;
|
||||
pub use stream::JetStreamClient;
|
||||
pub use tenant_placement::TenantPlacement;
|
||||
pub use types::*;
|
||||
253
projection/src/main.rs
Normal file
253
projection/src/main.rs
Normal file
@@ -0,0 +1,253 @@
|
||||
use projection::config::Settings;
|
||||
use projection::Observability;
|
||||
use projection::TenantPlacement;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
match std::env::args().nth(1).as_deref() {
|
||||
Some("-h") | Some("--help") => {
|
||||
print_help();
|
||||
return;
|
||||
}
|
||||
Some("serve") | None => serve().await,
|
||||
Some("rebuild") => rebuild().await,
|
||||
Some("backfill") => backfill().await,
|
||||
Some("health") => health().await,
|
||||
Some(other) => {
|
||||
eprintln!("Unknown command: {}", other);
|
||||
print_help();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn serve() {
|
||||
let settings = load_settings();
|
||||
let _ = settings.validate();
|
||||
init_logging(&settings);
|
||||
|
||||
tracing::info!(settings = ?settings, "Projection starting");
|
||||
|
||||
let shutdown = Arc::new(tokio::sync::Notify::new());
|
||||
let ready = Arc::new(AtomicBool::new(false));
|
||||
let draining = Arc::new(AtomicBool::new(false));
|
||||
let observability = Observability::default();
|
||||
let tenant_placement = TenantPlacement::load(&settings).unwrap_or_else(|e| {
|
||||
tracing::error!(error = %e, "Failed to load tenant placement, defaulting to allow-all");
|
||||
TenantPlacement::default()
|
||||
});
|
||||
|
||||
let http_state = match projection::http::build_state(
|
||||
settings.clone(),
|
||||
ready.clone(),
|
||||
draining.clone(),
|
||||
observability.clone(),
|
||||
tenant_placement.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(state) => state,
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "Failed to initialize HTTP state");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
|
||||
let http_shutdown = shutdown.clone();
|
||||
let http_task =
|
||||
tokio::spawn(async move { projection::http::serve(http_state, http_shutdown).await });
|
||||
|
||||
let signal_shutdown = shutdown.clone();
|
||||
let signal_ready = ready.clone();
|
||||
let signal_draining = draining.clone();
|
||||
tokio::spawn(async move {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use tokio::signal::unix::{signal, SignalKind};
|
||||
let mut sigterm = signal(SignalKind::terminate()).ok();
|
||||
let mut sigint = signal(SignalKind::interrupt()).ok();
|
||||
tokio::select! {
|
||||
_ = tokio::signal::ctrl_c() => {},
|
||||
_ = async { if let Some(s) = &mut sigterm { let _ = s.recv().await; } } => {},
|
||||
_ = async { if let Some(s) = &mut sigint { let _ = s.recv().await; } } => {},
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = tokio::signal::ctrl_c().await;
|
||||
}
|
||||
|
||||
signal_draining.store(true, Ordering::Relaxed);
|
||||
signal_ready.store(false, Ordering::Relaxed);
|
||||
signal_shutdown.notify_waiters();
|
||||
});
|
||||
|
||||
let worker_shutdown = shutdown.clone();
|
||||
let worker_ready = ready.clone();
|
||||
let worker_obs = observability.clone();
|
||||
let worker_tenant_placement = tenant_placement.clone();
|
||||
let worker_task = tokio::spawn(async move {
|
||||
projection::stream::run_projection_with_signals(
|
||||
settings,
|
||||
worker_shutdown,
|
||||
worker_ready,
|
||||
worker_obs,
|
||||
worker_tenant_placement,
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
let worker_result = worker_task.await;
|
||||
shutdown.notify_waiters();
|
||||
|
||||
let _ = http_task.await;
|
||||
|
||||
match worker_result {
|
||||
Ok(Ok(())) => {}
|
||||
Ok(Err(e)) => {
|
||||
tracing::error!(error = %e, "Projection terminated with error");
|
||||
std::process::exit(1);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "Projection task join error");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn rebuild() {
|
||||
let mut settings = load_settings();
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
|
||||
let tenant_id = flag_value(&args, "--tenant").unwrap_or_default();
|
||||
let view_type = match flag_value(&args, "--view-type") {
|
||||
Some(v) => v,
|
||||
None => {
|
||||
eprintln!("Missing --view-type");
|
||||
print_help();
|
||||
std::process::exit(2);
|
||||
}
|
||||
};
|
||||
let from_seq = flag_value(&args, "--from-seq")
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(1);
|
||||
if let Some(path) = flag_value(&args, "--manifest") {
|
||||
settings.manifest_path = path;
|
||||
}
|
||||
|
||||
let _ = settings.validate();
|
||||
init_logging(&settings);
|
||||
|
||||
let tenant = projection::types::TenantId::new(tenant_id);
|
||||
let view_type = projection::types::ViewType::new(view_type);
|
||||
|
||||
if let Err(e) = projection::stream::rebuild_view(settings, tenant, view_type, from_seq).await {
|
||||
tracing::error!(error = %e, "Rebuild failed");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async fn backfill() {
|
||||
let mut settings = load_settings();
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
|
||||
let tenant_id = flag_value(&args, "--tenant").unwrap_or_default();
|
||||
let from_seq = flag_value(&args, "--from-seq")
|
||||
.and_then(|v| v.parse::<u64>().ok())
|
||||
.unwrap_or(1);
|
||||
if let Some(path) = flag_value(&args, "--manifest") {
|
||||
settings.manifest_path = path;
|
||||
}
|
||||
|
||||
let _ = settings.validate();
|
||||
init_logging(&settings);
|
||||
|
||||
let tenant = projection::types::TenantId::new(tenant_id);
|
||||
|
||||
if let Err(e) = projection::stream::backfill_to_tail(settings, tenant, from_seq).await {
|
||||
tracing::error!(error = %e, "Backfill failed");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async fn health() {
|
||||
let mut settings = load_settings();
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
|
||||
let tenant_id = flag_value(&args, "--tenant").unwrap_or_default();
|
||||
if let Some(path) = flag_value(&args, "--manifest") {
|
||||
settings.manifest_path = path;
|
||||
}
|
||||
|
||||
let _ = settings.validate();
|
||||
init_logging(&settings);
|
||||
|
||||
let tenant = projection::types::TenantId::new(tenant_id);
|
||||
match projection::stream::health_report(settings, tenant).await {
|
||||
Ok(report) => {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&report_to_json(report)).unwrap()
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "Health check failed");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn print_help() {
|
||||
println!(
|
||||
"projection\n\nUSAGE:\n projection [COMMAND]\n\nCOMMANDS:\n serve Start the projection worker (default)\n rebuild Delete view+checkpoint for a tenant/view_type and backfill from a sequence\n backfill Backfill current manifest to tail for a tenant (for hot upgrades)\n health Print storage/NATS health and per-view lag\n\nOPTIONS:\n -h, --help Print help\n --manifest <path> Manifest path override\n --tenant <tenant_id> Tenant (empty string for default)\n --view-type <view_type> View type (rebuild only)\n --from-seq <u64> Start sequence (default 1)\n"
|
||||
);
|
||||
}
|
||||
|
||||
fn load_settings() -> Settings {
|
||||
if let Ok(path) = std::env::var("PROJECTION_CONFIG_PATH") {
|
||||
if let Ok(settings) = Settings::load_from_file_with_env_overrides(path) {
|
||||
return settings;
|
||||
}
|
||||
}
|
||||
|
||||
Settings::from_env().unwrap_or_default()
|
||||
}
|
||||
|
||||
fn init_logging(settings: &Settings) {
|
||||
let _ = settings;
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
|
||||
)
|
||||
.json()
|
||||
.init();
|
||||
}
|
||||
|
||||
fn flag_value(args: &[String], name: &str) -> Option<String> {
|
||||
args.iter()
|
||||
.position(|a| a == name)
|
||||
.and_then(|idx| args.get(idx + 1))
|
||||
.map(|v| v.to_string())
|
||||
}
|
||||
|
||||
fn report_to_json(report: projection::stream::HealthReport) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"storage_ok": report.storage_ok,
|
||||
"nats_ok": report.nats_ok,
|
||||
"stream_last_sequence": report.stream_last_sequence,
|
||||
"lags": report.lags.into_iter().map(|(view_type, lag)| {
|
||||
serde_json::json!({"view_type": view_type, "lag": lag})
|
||||
}).collect::<Vec<_>>()
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
#[test]
|
||||
fn binary_exists() {
|
||||
assert!(std::env::current_exe().is_ok());
|
||||
}
|
||||
}
|
||||
203
projection/src/observability/metrics.rs
Normal file
203
projection/src/observability/metrics.rs
Normal file
@@ -0,0 +1,203 @@
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::RwLock;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct AtomicHistogram {
|
||||
count: AtomicU64,
|
||||
sum: AtomicU64,
|
||||
buckets: Vec<(f64, AtomicU64)>,
|
||||
}
|
||||
|
||||
impl AtomicHistogram {
|
||||
fn new() -> Self {
|
||||
let buckets: Vec<(f64, AtomicU64)> = vec![
|
||||
0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
|
||||
]
|
||||
.into_iter()
|
||||
.map(|v| (v, AtomicU64::new(0)))
|
||||
.collect();
|
||||
|
||||
Self {
|
||||
count: AtomicU64::new(0),
|
||||
sum: AtomicU64::new(0),
|
||||
buckets,
|
||||
}
|
||||
}
|
||||
|
||||
fn observe(&self, duration: Duration) {
|
||||
let value_ms = duration.as_secs_f64() * 1000.0;
|
||||
self.count.fetch_add(1, Ordering::Relaxed);
|
||||
self.sum
|
||||
.fetch_add((value_ms * 1000.0) as u64, Ordering::Relaxed);
|
||||
|
||||
for (threshold, count) in &self.buckets {
|
||||
if value_ms <= *threshold {
|
||||
count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn export(&self, name: &str, labels: &str) -> String {
|
||||
let mut output = String::new();
|
||||
let count = self.count.load(Ordering::Relaxed);
|
||||
let sum = self.sum.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
|
||||
let label_str = if labels.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
format!("{{{}}}", labels.trim_start_matches(','))
|
||||
};
|
||||
|
||||
output.push_str(&format!("{}_sum{} {}\n", name, label_str, sum));
|
||||
output.push_str(&format!("{}_count{} {}\n", name, label_str, count));
|
||||
|
||||
for (threshold, bucket_count) in &self.buckets {
|
||||
let c = bucket_count.load(Ordering::Relaxed);
|
||||
let bucket_labels = if labels.is_empty() {
|
||||
format!("le=\"{}\"", threshold)
|
||||
} else {
|
||||
format!("le=\"{}\"{}", threshold, labels)
|
||||
};
|
||||
output.push_str(&format!("{}_bucket{{{}}} {}\n", name, bucket_labels, c));
|
||||
}
|
||||
let inf_labels = if labels.is_empty() {
|
||||
"le=\"+Inf\"".to_string()
|
||||
} else {
|
||||
format!("le=\"+Inf\"{}", labels)
|
||||
};
|
||||
output.push_str(&format!("{}_bucket{{{}}} {}\n", name, inf_labels, count));
|
||||
|
||||
output
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for AtomicHistogram {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Metrics {
|
||||
events_total: RwLock<HashMap<String, AtomicU64>>,
|
||||
processing_errors: RwLock<HashMap<String, AtomicU64>>,
|
||||
processing_duration: RwLock<HashMap<String, AtomicHistogram>>,
|
||||
}
|
||||
|
||||
impl Metrics {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
events_total: RwLock::new(HashMap::new()),
|
||||
processing_errors: RwLock::new(HashMap::new()),
|
||||
processing_duration: RwLock::new(HashMap::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn increment_events_total(&self, view_type: &str, tenant_id: &str) {
|
||||
let key = format!("{}:{}", view_type, tenant_id);
|
||||
let map = self.events_total.read().unwrap();
|
||||
if let Some(counter) = map.get(&key) {
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
drop(map);
|
||||
let mut map = self.events_total.write().unwrap();
|
||||
let counter = map.entry(key).or_insert_with(|| AtomicU64::new(0));
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn increment_processing_errors(&self, view_type: &str, tenant_id: &str) {
|
||||
let key = format!("{}:{}", view_type, tenant_id);
|
||||
let map = self.processing_errors.read().unwrap();
|
||||
if let Some(counter) = map.get(&key) {
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
drop(map);
|
||||
let mut map = self.processing_errors.write().unwrap();
|
||||
let counter = map.entry(key).or_insert_with(|| AtomicU64::new(0));
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn record_processing_duration(&self, duration: Duration, view_type: &str) {
|
||||
let mut map = self.processing_duration.write().unwrap();
|
||||
let histogram = map.entry(view_type.to_string()).or_default();
|
||||
histogram.observe(duration);
|
||||
}
|
||||
|
||||
pub fn export_prometheus(&self) -> String {
|
||||
let mut output = String::new();
|
||||
|
||||
output.push_str("# HELP projection_events_total Total number of events processed\n");
|
||||
output.push_str("# TYPE projection_events_total counter\n");
|
||||
{
|
||||
let map = self.events_total.read().unwrap();
|
||||
for (key, counter) in map.iter() {
|
||||
let parts: Vec<&str> = key.split(':').collect();
|
||||
if parts.len() == 2 {
|
||||
let value = counter.load(Ordering::Relaxed);
|
||||
output.push_str(&format!(
|
||||
"projection_events_total{{view_type=\"{}\",tenant_id=\"{}\"}} {}\n",
|
||||
parts[0], parts[1], value
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output.push_str("\n# HELP projection_processing_errors_total Total processing errors\n");
|
||||
output.push_str("# TYPE projection_processing_errors_total counter\n");
|
||||
{
|
||||
let map = self.processing_errors.read().unwrap();
|
||||
for (key, counter) in map.iter() {
|
||||
let parts: Vec<&str> = key.split(':').collect();
|
||||
if parts.len() == 2 {
|
||||
let value = counter.load(Ordering::Relaxed);
|
||||
output.push_str(&format!(
|
||||
"projection_processing_errors_total{{view_type=\"{}\",tenant_id=\"{}\"}} {}\n",
|
||||
parts[0], parts[1], value
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output.push_str(
|
||||
"\n# HELP projection_processing_duration_seconds Event processing duration\n",
|
||||
);
|
||||
output.push_str("# TYPE projection_processing_duration_seconds histogram\n");
|
||||
{
|
||||
let map = self.processing_duration.read().unwrap();
|
||||
for (view_type, histogram) in map.iter() {
|
||||
let labels = format!(",view_type=\"{}\"", view_type);
|
||||
output
|
||||
.push_str(&histogram.export("projection_processing_duration_seconds", &labels));
|
||||
}
|
||||
}
|
||||
|
||||
output
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Metrics {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn metrics_increment_events() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.increment_events_total("User", "tenant-a");
|
||||
metrics.increment_events_total("User", "tenant-a");
|
||||
|
||||
let output = metrics.export_prometheus();
|
||||
assert!(
|
||||
output.contains("projection_events_total{view_type=\"User\",tenant_id=\"tenant-a\"} 2")
|
||||
);
|
||||
}
|
||||
}
|
||||
128
projection/src/observability/mod.rs
Normal file
128
projection/src/observability/mod.rs
Normal file
@@ -0,0 +1,128 @@
|
||||
mod metrics;
|
||||
|
||||
pub use metrics::Metrics;
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ProcessingSpan {
|
||||
view_type: String,
|
||||
tenant_id: String,
|
||||
correlation_id: Option<String>,
|
||||
trace_id: Option<String>,
|
||||
start_time: Instant,
|
||||
}
|
||||
|
||||
impl ProcessingSpan {
|
||||
pub fn new(
|
||||
view_type: impl Into<String>,
|
||||
tenant_id: impl Into<String>,
|
||||
correlation_id: Option<String>,
|
||||
trace_id: Option<String>,
|
||||
) -> Self {
|
||||
Self {
|
||||
view_type: view_type.into(),
|
||||
tenant_id: tenant_id.into(),
|
||||
correlation_id,
|
||||
trace_id,
|
||||
start_time: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn elapsed(&self) -> std::time::Duration {
|
||||
self.start_time.elapsed()
|
||||
}
|
||||
|
||||
pub fn view_type(&self) -> &str {
|
||||
&self.view_type
|
||||
}
|
||||
|
||||
pub fn tenant_id(&self) -> &str {
|
||||
&self.tenant_id
|
||||
}
|
||||
|
||||
pub fn correlation_id(&self) -> Option<&str> {
|
||||
self.correlation_id.as_deref()
|
||||
}
|
||||
|
||||
pub fn trace_id(&self) -> Option<&str> {
|
||||
self.trace_id.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Observability {
|
||||
metrics: Arc<Metrics>,
|
||||
}
|
||||
|
||||
impl Observability {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
metrics: Arc::new(Metrics::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn metrics(&self) -> &Arc<Metrics> {
|
||||
&self.metrics
|
||||
}
|
||||
|
||||
pub fn start_processing_span(
|
||||
&self,
|
||||
view_type: &str,
|
||||
tenant_id: &str,
|
||||
correlation_id: Option<&str>,
|
||||
trace_id: Option<&str>,
|
||||
) -> ProcessingSpan {
|
||||
tracing::info_span!(
|
||||
"projection_event",
|
||||
view_type = %view_type,
|
||||
tenant_id = %tenant_id,
|
||||
correlation_id = correlation_id.unwrap_or(""),
|
||||
trace_id = trace_id.unwrap_or(""),
|
||||
);
|
||||
ProcessingSpan::new(
|
||||
view_type,
|
||||
tenant_id,
|
||||
correlation_id.map(|s| s.to_string()),
|
||||
trace_id.map(|s| s.to_string()),
|
||||
)
|
||||
}
|
||||
|
||||
pub fn record_processed(&self, span: &ProcessingSpan) {
|
||||
self.metrics
|
||||
.increment_events_total(span.view_type(), span.tenant_id());
|
||||
self.metrics
|
||||
.record_processing_duration(span.elapsed(), span.view_type());
|
||||
}
|
||||
|
||||
pub fn record_error(&self, span: &ProcessingSpan) {
|
||||
self.metrics
|
||||
.increment_events_total(span.view_type(), span.tenant_id());
|
||||
self.metrics
|
||||
.increment_processing_errors(span.view_type(), span.tenant_id());
|
||||
self.metrics
|
||||
.record_processing_duration(span.elapsed(), span.view_type());
|
||||
}
|
||||
|
||||
pub fn export_metrics(&self) -> String {
|
||||
self.metrics.export_prometheus()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for Observability {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn observability_is_default() {
|
||||
let obs = Observability::default();
|
||||
let _ = obs.export_metrics();
|
||||
}
|
||||
}
|
||||
89
projection/src/project/manifest.rs
Normal file
89
projection/src/project/manifest.rs
Normal file
@@ -0,0 +1,89 @@
|
||||
use crate::types::{ProjectionError, ViewType};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ProjectionDefinition {
|
||||
pub view_type: ViewType,
|
||||
pub project_program: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
||||
pub struct ProjectionManifest {
|
||||
projections: HashMap<String, ProjectionDefinition>,
|
||||
}
|
||||
|
||||
impl ProjectionManifest {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
projections: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn register(&mut self, definition: ProjectionDefinition) {
|
||||
self.projections
|
||||
.insert(definition.view_type.as_str().to_string(), definition);
|
||||
}
|
||||
|
||||
pub fn get(&self, view_type: &ViewType) -> Option<&ProjectionDefinition> {
|
||||
self.projections.get(view_type.as_str())
|
||||
}
|
||||
|
||||
pub fn all(&self) -> impl Iterator<Item = &ProjectionDefinition> {
|
||||
self.projections.values()
|
||||
}
|
||||
|
||||
pub fn load_from_yaml(yaml: &str) -> Result<Self, serde_yaml::Error> {
|
||||
serde_yaml::from_str(yaml)
|
||||
}
|
||||
|
||||
pub fn load_from_json(json: &str) -> Result<Self, serde_json::Error> {
|
||||
serde_json::from_str(json)
|
||||
}
|
||||
|
||||
pub fn validate(&self) -> Result<(), ProjectionError> {
|
||||
for def in self.projections.values() {
|
||||
if def.project_program.is_empty() {
|
||||
return Err(ProjectionError::ManifestError(format!(
|
||||
"Missing project_program for view_type {}",
|
||||
def.view_type.as_str()
|
||||
)));
|
||||
}
|
||||
|
||||
if !std::path::Path::new(&def.project_program).exists() {
|
||||
return Err(ProjectionError::ManifestError(format!(
|
||||
"Project program not found for view_type {}: {}",
|
||||
def.view_type.as_str(),
|
||||
def.project_program
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn manifest_loads_and_validates() {
|
||||
let dir = tempdir().unwrap();
|
||||
let program_path = dir.path().join("proj.js");
|
||||
std::fs::write(&program_path, "function project() { return null; }").unwrap();
|
||||
|
||||
let yaml = format!(
|
||||
r#"
|
||||
projections:
|
||||
User:
|
||||
view_type: "User"
|
||||
project_program: "{}"
|
||||
"#,
|
||||
program_path.to_string_lossy()
|
||||
);
|
||||
|
||||
let manifest = ProjectionManifest::load_from_yaml(&yaml).unwrap();
|
||||
manifest.validate().unwrap();
|
||||
}
|
||||
}
|
||||
5
projection/src/project/mod.rs
Normal file
5
projection/src/project/mod.rs
Normal file
@@ -0,0 +1,5 @@
|
||||
mod manifest;
|
||||
mod runtime;
|
||||
|
||||
pub use manifest::{ProjectionDefinition, ProjectionManifest};
|
||||
pub use runtime::{ProjectionOutput, ProjectionRuntime};
|
||||
274
projection/src/project/runtime.rs
Normal file
274
projection/src/project/runtime.rs
Normal file
@@ -0,0 +1,274 @@
|
||||
use crate::types::{EventEnvelope, ProjectionError};
|
||||
use serde_json::Value as JsonValue;
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ProjectionOutput {
|
||||
pub view_id: String,
|
||||
pub new_view: JsonValue,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct ProjectionRuntime {
|
||||
pub gas_limit: u64,
|
||||
pub timeout: Duration,
|
||||
engine: runtime_function::Engine,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ProjectionRuntime {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ProjectionRuntime")
|
||||
.field("gas_limit", &self.gas_limit)
|
||||
.field("timeout", &self.timeout)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ProjectionRuntime {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
gas_limit: 1_000_000,
|
||||
timeout: Duration::from_secs(5),
|
||||
engine: runtime_function::Engine::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ProjectionRuntime {
|
||||
pub fn new(gas_limit: u64, timeout: Duration) -> Self {
|
||||
Self {
|
||||
gas_limit,
|
||||
timeout,
|
||||
engine: runtime_function::Engine::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn project(
|
||||
&self,
|
||||
current_view: &JsonValue,
|
||||
event: &EventEnvelope,
|
||||
program_path: &str,
|
||||
) -> Result<Option<ProjectionOutput>, ProjectionError> {
|
||||
let program = std::fs::read_to_string(program_path)
|
||||
.map_err(|e| ProjectionError::ProjectError(e.to_string()))?;
|
||||
|
||||
let program = runtime_function::Program::from_json(&program)
|
||||
.map_err(|e| ProjectionError::ProjectError(format!("Program parse error: {}", e)))?;
|
||||
|
||||
self.project_program(current_view, event, &program).await
|
||||
}
|
||||
|
||||
pub async fn project_program(
|
||||
&self,
|
||||
current_view: &JsonValue,
|
||||
event: &EventEnvelope,
|
||||
program: &runtime_function::Program,
|
||||
) -> Result<Option<ProjectionOutput>, ProjectionError> {
|
||||
let current_view_value = json_to_runtime_value(current_view)
|
||||
.map_err(|e| ProjectionError::ProjectError(e.to_string()))?;
|
||||
|
||||
let event_json = serde_json::to_value(event)
|
||||
.map_err(|e| ProjectionError::ProjectError(e.to_string()))?;
|
||||
let event_value = json_to_runtime_value(&event_json)
|
||||
.map_err(|e| ProjectionError::ProjectError(e.to_string()))?;
|
||||
|
||||
let mut inputs = BTreeMap::new();
|
||||
inputs.insert("current_view".to_string(), current_view_value);
|
||||
inputs.insert("event".to_string(), event_value);
|
||||
|
||||
let timeout_secs = self.timeout.as_secs().max(1);
|
||||
let options = runtime_function::engine::ExecutionOptions {
|
||||
gas_limit: self.gas_limit,
|
||||
timeout_secs,
|
||||
trace: false,
|
||||
};
|
||||
|
||||
let now = event.timestamp.unwrap_or_else(chrono::Utc::now);
|
||||
let causation_id = format!("{}:{}", event.aggregate_id, event.event_type);
|
||||
let context = runtime_function::Context::new(now, causation_id)
|
||||
.with_tenant_id(event.tenant_id.as_str());
|
||||
|
||||
let result = self
|
||||
.engine
|
||||
.execute_with_options(program, inputs, context, options);
|
||||
|
||||
if !result.success {
|
||||
return Err(ProjectionError::ProjectError(
|
||||
result
|
||||
.error
|
||||
.map(|e| e.to_string())
|
||||
.unwrap_or_else(|| "runtime execution failed".to_string()),
|
||||
));
|
||||
}
|
||||
|
||||
let Some(output) = result.output else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if output.is_null() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let output_json = runtime_value_to_json(&output);
|
||||
let view_id = output_json
|
||||
.get("view_id")
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| ProjectionError::ProjectError("missing view_id".to_string()))?
|
||||
.to_string();
|
||||
let new_view = output_json
|
||||
.get("new_view")
|
||||
.cloned()
|
||||
.ok_or_else(|| ProjectionError::ProjectError("missing new_view".to_string()))?;
|
||||
|
||||
Ok(Some(ProjectionOutput { view_id, new_view }))
|
||||
}
|
||||
}
|
||||
|
||||
fn json_to_runtime_value(value: &JsonValue) -> Result<runtime_function::Value, serde_json::Error> {
|
||||
serde_json::from_value(value.clone())
|
||||
}
|
||||
|
||||
fn runtime_value_to_json(value: &runtime_function::Value) -> JsonValue {
|
||||
match value {
|
||||
runtime_function::Value::Null => JsonValue::Null,
|
||||
runtime_function::Value::Bool(b) => JsonValue::Bool(*b),
|
||||
runtime_function::Value::Decimal(d) => {
|
||||
let s = d.to_string();
|
||||
if !s.contains('.') && !s.contains('e') && !s.contains('E') {
|
||||
if let Ok(i) = s.parse::<i64>() {
|
||||
return JsonValue::Number(i.into());
|
||||
}
|
||||
if let Ok(u) = s.parse::<u64>() {
|
||||
return JsonValue::Number(u.into());
|
||||
}
|
||||
}
|
||||
JsonValue::String(s)
|
||||
}
|
||||
runtime_function::Value::String(s) => JsonValue::String(s.to_string()),
|
||||
runtime_function::Value::DateTime(dt) => JsonValue::String(dt.to_rfc3339()),
|
||||
runtime_function::Value::Array(arr) => {
|
||||
JsonValue::Array(arr.iter().map(runtime_value_to_json).collect::<Vec<_>>())
|
||||
}
|
||||
runtime_function::Value::Object(obj) => {
|
||||
let mut map = serde_json::Map::new();
|
||||
for (k, v) in obj.iter() {
|
||||
map.insert(k.clone(), runtime_value_to_json(v));
|
||||
}
|
||||
JsonValue::Object(map)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_function_is_deterministic() {
|
||||
let rt = ProjectionRuntime::default();
|
||||
let event = EventEnvelope {
|
||||
tenant_id: crate::types::TenantId::new("t1"),
|
||||
event_id: None,
|
||||
aggregate_id: "a1".to_string(),
|
||||
aggregate_type: "Account".to_string(),
|
||||
version: None,
|
||||
event_type: "created".to_string(),
|
||||
payload: json!({"x": 1}),
|
||||
command_id: None,
|
||||
timestamp: Some(
|
||||
chrono::DateTime::parse_from_rfc3339("2026-02-09T12:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&chrono::Utc),
|
||||
),
|
||||
correlation_id: None,
|
||||
traceparent: None,
|
||||
trace_id: None,
|
||||
};
|
||||
|
||||
let program_json = r#"
|
||||
{
|
||||
"specVersion": "1.1",
|
||||
"id": "proj",
|
||||
"name": "Projection",
|
||||
"inputs": [
|
||||
{"name": "current_view", "type": "Any", "required": true},
|
||||
{"name": "event", "type": "Any", "required": true}
|
||||
],
|
||||
"nodes": [
|
||||
{"id": "event", "type": "InputRef", "data": {"input_name": "event"}},
|
||||
{"id": "expr", "type": "Expr", "data": {"expression": "({ view_id: input.aggregate_id, new_view: input.payload })"}},
|
||||
{"id": "output", "type": "Output", "data": {}}
|
||||
],
|
||||
"edges": [
|
||||
{"id": "e1", "source": "event", "sourceHandle": "out", "target": "expr", "targetHandle": "input"},
|
||||
{"id": "e2", "source": "expr", "sourceHandle": "out", "target": "output", "targetHandle": "value"}
|
||||
],
|
||||
"outputNodeId": "output"
|
||||
}
|
||||
"#;
|
||||
|
||||
let program: runtime_function::Program = serde_json::from_str(program_json).unwrap();
|
||||
|
||||
let out1 = rt
|
||||
.project_program(&json!({"a": 1}), &event, &program)
|
||||
.await
|
||||
.unwrap();
|
||||
let out2 = rt
|
||||
.project_program(&json!({"a": 1}), &event, &program)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(out1, out2);
|
||||
let out = out1.unwrap();
|
||||
assert_eq!(out.view_id, "a1");
|
||||
assert_eq!(out.new_view["x"], 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn runtime_contract_requires_fields() {
|
||||
let rt = ProjectionRuntime::default();
|
||||
let event = EventEnvelope {
|
||||
tenant_id: crate::types::TenantId::new("t1"),
|
||||
event_id: None,
|
||||
aggregate_id: "a1".to_string(),
|
||||
aggregate_type: "Account".to_string(),
|
||||
version: None,
|
||||
event_type: "created".to_string(),
|
||||
payload: json!({"x": 1}),
|
||||
command_id: None,
|
||||
timestamp: None,
|
||||
correlation_id: None,
|
||||
traceparent: None,
|
||||
trace_id: None,
|
||||
};
|
||||
|
||||
let program_json = r#"
|
||||
{
|
||||
"specVersion": "1.1",
|
||||
"id": "proj",
|
||||
"name": "Projection",
|
||||
"inputs": [
|
||||
{"name": "current_view", "type": "Any", "required": true},
|
||||
{"name": "event", "type": "Any", "required": true}
|
||||
],
|
||||
"nodes": [
|
||||
{"id": "const", "type": "Const", "data": {"value": {"ok": true}}},
|
||||
{"id": "output", "type": "Output", "data": {}}
|
||||
],
|
||||
"edges": [
|
||||
{"id": "e1", "source": "const", "sourceHandle": "out", "target": "output", "targetHandle": "value"}
|
||||
],
|
||||
"outputNodeId": "output"
|
||||
}
|
||||
"#;
|
||||
|
||||
let program: runtime_function::Program = serde_json::from_str(program_json).unwrap();
|
||||
let err = rt
|
||||
.project_program(&json!({}), &event, &program)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(format!("{}", err).contains("missing view_id"));
|
||||
}
|
||||
}
|
||||
3
projection/src/query/mod.rs
Normal file
3
projection/src/query/mod.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
mod uqf;
|
||||
|
||||
pub use uqf::{QueryError, QueryRequest, QueryResponse, QueryService};
|
||||
266
projection/src/query/uqf.rs
Normal file
266
projection/src/query/uqf.rs
Normal file
@@ -0,0 +1,266 @@
|
||||
use crate::storage::KvClient;
|
||||
use crate::types::{ProjectionError, TenantId, ViewType};
|
||||
use query_engine::query::QueryHint;
|
||||
use query_engine::{canonicalize_filter, ExecutionLimits, Query, QueryExecutor, QueryMode};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct QueryRequest {
|
||||
pub tenant_id: TenantId,
|
||||
pub view_type: ViewType,
|
||||
pub uqf: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "mode", rename_all = "lowercase")]
|
||||
pub enum QueryResponse {
|
||||
Find {
|
||||
took_ms: u64,
|
||||
hits: Vec<serde_json::Value>,
|
||||
count: usize,
|
||||
next_cursor: Option<String>,
|
||||
metrics: Option<query_engine::query::QueryMetrics>,
|
||||
},
|
||||
Count {
|
||||
took_ms: u64,
|
||||
count: usize,
|
||||
metrics: Option<query_engine::query::QueryMetrics>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum QueryError {
|
||||
#[error("Invalid query: {0}")]
|
||||
InvalidQuery(String),
|
||||
#[error("Query execution error: {0}")]
|
||||
Execution(String),
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct QueryService {
|
||||
storage: KvClient,
|
||||
engine: Arc<query_engine::Engine>,
|
||||
max_results: usize,
|
||||
max_scan: usize,
|
||||
timeout_ms: u64,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for QueryService {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("QueryService")
|
||||
.field("max_results", &self.max_results)
|
||||
.field("max_scan", &self.max_scan)
|
||||
.field("timeout_ms", &self.timeout_ms)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl QueryService {
|
||||
pub fn new(storage: KvClient) -> Self {
|
||||
Self {
|
||||
storage,
|
||||
engine: Arc::new(query_engine::Engine::new()),
|
||||
max_results: 1000,
|
||||
max_scan: 100_000,
|
||||
timeout_ms: 1_000,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_limits(mut self, max_results: usize, max_scan: usize, timeout_ms: u64) -> Self {
|
||||
self.max_results = max_results;
|
||||
self.max_scan = max_scan;
|
||||
self.timeout_ms = timeout_ms;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn query(&self, request: QueryRequest) -> Result<QueryResponse, QueryError> {
|
||||
let mut query =
|
||||
Query::from_json(&request.uqf).map_err(|e| QueryError::InvalidQuery(e.to_string()))?;
|
||||
|
||||
if matches!(query.mode, QueryMode::Explain) {
|
||||
return Err(QueryError::InvalidQuery(
|
||||
"Explain mode is not supported".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
query.filter = canonicalize_filter(query.filter);
|
||||
|
||||
query.limit = Some(
|
||||
query
|
||||
.limit
|
||||
.unwrap_or(self.max_results)
|
||||
.min(self.max_results),
|
||||
);
|
||||
|
||||
let effective_timeout_ms = query
|
||||
.hint
|
||||
.as_ref()
|
||||
.and_then(|h| h.timeout_ms)
|
||||
.unwrap_or(self.timeout_ms)
|
||||
.min(self.timeout_ms);
|
||||
let effective_max_scan = query
|
||||
.hint
|
||||
.as_ref()
|
||||
.and_then(|h| h.max_scan)
|
||||
.unwrap_or(self.max_scan)
|
||||
.min(self.max_scan);
|
||||
|
||||
let hint = query
|
||||
.hint
|
||||
.clone()
|
||||
.unwrap_or_else(QueryHint::new)
|
||||
.with_timeout(effective_timeout_ms)
|
||||
.with_max_scan(effective_max_scan);
|
||||
query.hint = Some(hint);
|
||||
|
||||
self.engine
|
||||
.validate(&query)
|
||||
.map_err(|e| QueryError::InvalidQuery(e.to_string()))?;
|
||||
|
||||
let prefix = format!(
|
||||
"view:{}:{}:",
|
||||
request.tenant_id.as_str(),
|
||||
request.view_type.as_str()
|
||||
);
|
||||
|
||||
let docs = self
|
||||
.storage
|
||||
.scan_documents_by_prefix(prefix.as_bytes(), effective_max_scan)
|
||||
.map_err(QueryError::from)?;
|
||||
|
||||
let source = query_engine::InMemorySource::from_documents(docs);
|
||||
|
||||
let limits = ExecutionLimits::new()
|
||||
.with_max_scan(effective_max_scan)
|
||||
.with_timeout_ms(effective_timeout_ms)
|
||||
.with_cancel_token(Arc::new(query_engine::CancelToken::new()));
|
||||
|
||||
let executor = QueryExecutor::new(self.engine.config());
|
||||
let result = executor
|
||||
.execute_with_limits(&query, &source, limits)
|
||||
.map_err(|e| QueryError::Execution(e.to_string()))?;
|
||||
|
||||
match result {
|
||||
query_engine::exec::ExecutionResult::Find(resp) => Ok(QueryResponse::Find {
|
||||
took_ms: resp.took_ms,
|
||||
hits: resp.hits,
|
||||
count: resp.count,
|
||||
next_cursor: resp.next_cursor,
|
||||
metrics: resp.metrics,
|
||||
}),
|
||||
query_engine::exec::ExecutionResult::Count(resp) => Ok(QueryResponse::Count {
|
||||
took_ms: resp.took_ms,
|
||||
count: resp.count,
|
||||
metrics: resp.metrics,
|
||||
}),
|
||||
query_engine::exec::ExecutionResult::Explain(_) => Err(QueryError::InvalidQuery(
|
||||
"Explain mode is not supported".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ProjectionError> for QueryError {
|
||||
fn from(e: ProjectionError) -> Self {
|
||||
QueryError::Execution(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{CheckpointKey, StreamSequence, ViewId, ViewKey};
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn tenant_scoped_queries_never_return_other_tenant_keys() {
|
||||
let storage = KvClient::in_memory();
|
||||
let service = QueryService::new(storage.clone()).with_limits(100, 1000, 1000);
|
||||
|
||||
let tenant_a = TenantId::new("t1");
|
||||
let tenant_b = TenantId::new("t2");
|
||||
let view_type = ViewType::new("User");
|
||||
|
||||
let cp_a = CheckpointKey::new(&tenant_a, &view_type);
|
||||
let cp_b = CheckpointKey::new(&tenant_b, &view_type);
|
||||
|
||||
let key_a1 = ViewKey::new(&tenant_a, &view_type, &ViewId::new("u1"));
|
||||
let key_a2 = ViewKey::new(&tenant_a, &view_type, &ViewId::new("u2"));
|
||||
let key_b1 = ViewKey::new(&tenant_b, &view_type, &ViewId::new("u1"));
|
||||
|
||||
storage
|
||||
.commit_view_and_checkpoint(&key_a1, &json!({"tenant": "t1", "id": "u1"}), &cp_a, 1)
|
||||
.unwrap();
|
||||
storage
|
||||
.commit_view_and_checkpoint(&key_a2, &json!({"tenant": "t1", "id": "u2"}), &cp_a, 2)
|
||||
.unwrap();
|
||||
storage
|
||||
.commit_view_and_checkpoint(&key_b1, &json!({"tenant": "t2", "id": "u1"}), &cp_b, 1)
|
||||
.unwrap();
|
||||
|
||||
let uqf = Query::new(query_engine::FilterNode::r#true())
|
||||
.with_limit(100)
|
||||
.to_json()
|
||||
.unwrap();
|
||||
|
||||
let response = service
|
||||
.query(QueryRequest {
|
||||
tenant_id: tenant_a,
|
||||
view_type,
|
||||
uqf,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
match response {
|
||||
QueryResponse::Find { hits, .. } => {
|
||||
assert_eq!(hits.len(), 2);
|
||||
assert!(hits.iter().all(|v| v["tenant"] == "t1"));
|
||||
}
|
||||
_ => panic!("expected find response"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn uqf_filter_works_on_fixture_dataset() {
|
||||
let storage = KvClient::in_memory();
|
||||
let service = QueryService::new(storage.clone()).with_limits(100, 1000, 1000);
|
||||
|
||||
let tenant = TenantId::new("t1");
|
||||
let view_type = ViewType::new("User");
|
||||
let cp = CheckpointKey::new(&tenant, &view_type);
|
||||
|
||||
let docs = vec![
|
||||
(ViewId::new("u1"), json!({"age": 20, "name": "a"})),
|
||||
(ViewId::new("u2"), json!({"age": 35, "name": "b"})),
|
||||
(ViewId::new("u3"), json!({"age": 40, "name": "c"})),
|
||||
];
|
||||
|
||||
for (i, (id, doc)) in docs.into_iter().enumerate() {
|
||||
let key = ViewKey::new(&tenant, &view_type, &id);
|
||||
storage
|
||||
.commit_view_and_checkpoint(&key, &doc, &cp, (i + 1) as StreamSequence)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let query = Query::new(query_engine::FilterNode::gt("age", json!(30)));
|
||||
let uqf = query.to_json().unwrap();
|
||||
|
||||
let response = service
|
||||
.query(QueryRequest {
|
||||
tenant_id: tenant,
|
||||
view_type,
|
||||
uqf,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
match response {
|
||||
QueryResponse::Find { hits, count, .. } => {
|
||||
assert_eq!(count, 2);
|
||||
assert_eq!(hits.len(), 2);
|
||||
assert!(hits.iter().all(|v| v["age"].as_i64().unwrap() > 30));
|
||||
}
|
||||
_ => panic!("expected find response"),
|
||||
}
|
||||
}
|
||||
}
|
||||
522
projection/src/storage/kv.rs
Normal file
522
projection/src/storage/kv.rs
Normal file
@@ -0,0 +1,522 @@
|
||||
use crate::types::{
|
||||
Checkpoint, CheckpointKey, ProjectionError, StreamSequence, TenantId, ViewKey, ViewType,
|
||||
};
|
||||
use edge_storage::{Config as EdgeConfig, EdgeStorage, KvStore, TableNames, Writer};
|
||||
use libmdbx::{NoWriteMap, WriteFlags, RW};
|
||||
use query_engine::Document;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct KvClient {
|
||||
storage: Arc<EdgeStorage>,
|
||||
kv: KvStore,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for KvClient {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("KvClient").finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl KvClient {
|
||||
pub fn open(storage_path: impl Into<String>) -> Result<Self, StorageInitError> {
|
||||
let config = EdgeConfig::new(storage_path.into());
|
||||
let storage = EdgeStorage::open(config.clone())?;
|
||||
let writer = Arc::new(Writer::new(storage.db().clone(), &config));
|
||||
let kv = KvStore::new(storage.db().clone(), writer);
|
||||
|
||||
Ok(Self {
|
||||
storage: Arc::new(storage),
|
||||
kv,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub fn in_memory() -> Self {
|
||||
use tempfile::tempdir;
|
||||
let dir = tempdir().expect("failed to create temp dir");
|
||||
let path = dir.path().join("test.mdbx");
|
||||
std::mem::forget(dir);
|
||||
Self::open(path.to_string_lossy().to_string()).expect("failed to open in-memory storage")
|
||||
}
|
||||
|
||||
pub fn get_view(&self, key: &ViewKey) -> Result<Option<serde_json::Value>, ProjectionError> {
|
||||
let bytes = self
|
||||
.kv
|
||||
.get(key.as_str().as_bytes())
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
|
||||
match bytes {
|
||||
Some(bytes) => serde_json::from_slice(&bytes)
|
||||
.map(Some)
|
||||
.map_err(|e| ProjectionError::DecodeError(e.to_string())),
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_checkpoint(
|
||||
&self,
|
||||
key: &CheckpointKey,
|
||||
) -> Result<Option<StreamSequence>, ProjectionError> {
|
||||
let bytes = self
|
||||
.kv
|
||||
.get(key.as_str().as_bytes())
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
|
||||
match bytes {
|
||||
Some(bytes) => {
|
||||
let cp: Checkpoint = serde_json::from_slice(&bytes)
|
||||
.map_err(|e| ProjectionError::DecodeError(e.to_string()))?;
|
||||
Ok(Some(cp.sequence))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn put_checkpoint(
|
||||
&self,
|
||||
key: &CheckpointKey,
|
||||
sequence: StreamSequence,
|
||||
) -> Result<(), ProjectionError> {
|
||||
let checkpoint_bytes = serde_json::to_vec(&Checkpoint::new(sequence))
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
|
||||
self.commit_kv_txn(|txn, table| {
|
||||
txn.put(
|
||||
table,
|
||||
key.as_str().as_bytes(),
|
||||
checkpoint_bytes.as_slice(),
|
||||
WriteFlags::empty(),
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn delete_checkpoint(&self, key: &CheckpointKey) -> Result<(), ProjectionError> {
|
||||
self.delete_key(key.as_str().as_bytes())
|
||||
}
|
||||
|
||||
pub fn delete_key(&self, key: &[u8]) -> Result<(), ProjectionError> {
|
||||
self.commit_kv_txn(|txn, table| {
|
||||
let _ = txn.del(table, key, None)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn delete_view_prefix(
|
||||
&self,
|
||||
tenant_id: &TenantId,
|
||||
view_type: &ViewType,
|
||||
) -> Result<(), ProjectionError> {
|
||||
let prefix = format!("view:{}:{}:", tenant_id.as_str(), view_type.as_str());
|
||||
let txn = self
|
||||
.storage
|
||||
.db()
|
||||
.begin_ro_txn()
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
|
||||
let keys = self
|
||||
.kv
|
||||
.prefix_scan(&txn, prefix.as_bytes())
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?
|
||||
.filter_map(|res| match res {
|
||||
Ok((k, _)) => Some(k),
|
||||
Err(_) => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if keys.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let ops = keys
|
||||
.into_iter()
|
||||
.map(|key| edge_storage::writer::KvOp::Delete { key })
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let result = self
|
||||
.kv
|
||||
.batch_sync(ops)
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
|
||||
match result {
|
||||
edge_storage::WriteResult::Success => Ok(()),
|
||||
edge_storage::WriteResult::Error(e) => Err(ProjectionError::StorageError(e)),
|
||||
other => Err(ProjectionError::StorageError(format!(
|
||||
"Unexpected write result: {:?}",
|
||||
other
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn commit_view_and_checkpoint(
|
||||
&self,
|
||||
view_key: &ViewKey,
|
||||
view_value: &serde_json::Value,
|
||||
checkpoint_key: &CheckpointKey,
|
||||
sequence: StreamSequence,
|
||||
) -> Result<(), ProjectionError> {
|
||||
let view_bytes = serde_json::to_vec(view_value)
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
let checkpoint_bytes = serde_json::to_vec(&Checkpoint::new(sequence))
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
|
||||
self.commit_kv_txn(|txn, table| {
|
||||
txn.put(
|
||||
table,
|
||||
view_key.as_str().as_bytes(),
|
||||
view_bytes.as_slice(),
|
||||
WriteFlags::empty(),
|
||||
)?;
|
||||
txn.put(
|
||||
table,
|
||||
checkpoint_key.as_str().as_bytes(),
|
||||
checkpoint_bytes.as_slice(),
|
||||
WriteFlags::empty(),
|
||||
)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn commit_view_and_advance_checkpoint_ordered(
|
||||
&self,
|
||||
view_key: &ViewKey,
|
||||
view_value: &serde_json::Value,
|
||||
checkpoint_key: &CheckpointKey,
|
||||
sequence: StreamSequence,
|
||||
) -> Result<(), ProjectionError> {
|
||||
let view_bytes = serde_json::to_vec(view_value)
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
self.commit_kv_txn_projection(|txn, table| {
|
||||
txn.put(
|
||||
table,
|
||||
view_key.as_str().as_bytes(),
|
||||
view_bytes.as_slice(),
|
||||
WriteFlags::empty(),
|
||||
)
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
self.mark_processed_and_advance_checkpoint_in_txn(txn, table, checkpoint_key, sequence)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn advance_checkpoint_ordered(
|
||||
&self,
|
||||
checkpoint_key: &CheckpointKey,
|
||||
sequence: StreamSequence,
|
||||
) -> Result<(), ProjectionError> {
|
||||
self.commit_kv_txn_projection(|txn, table| {
|
||||
self.mark_processed_and_advance_checkpoint_in_txn(txn, table, checkpoint_key, sequence)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn put_json(&self, key: &str, value: &serde_json::Value) -> Result<(), ProjectionError> {
|
||||
let bytes =
|
||||
serde_json::to_vec(value).map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
self.commit_kv_txn(|txn, table| {
|
||||
txn.put(table, key.as_bytes(), bytes.as_slice(), WriteFlags::empty())?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
pub fn scan_documents_by_prefix(
|
||||
&self,
|
||||
prefix: &[u8],
|
||||
max_scan: usize,
|
||||
) -> Result<Vec<Document>, ProjectionError> {
|
||||
let txn = self
|
||||
.storage
|
||||
.db()
|
||||
.begin_ro_txn()
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
let table = txn
|
||||
.open_table(TableNames::KV_STORE)
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
let mut cursor = txn
|
||||
.cursor(&table)
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
|
||||
let mut docs = Vec::new();
|
||||
let mut scanned = 0usize;
|
||||
|
||||
let start_key = prefix.to_vec();
|
||||
let mut iter = match cursor.set_range::<Vec<u8>, Vec<u8>>(&start_key) {
|
||||
Ok(Some((key, value))) => {
|
||||
if key.starts_with(prefix) {
|
||||
Some((key, value))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Ok(None) => None,
|
||||
Err(e) => return Err(ProjectionError::StorageError(e.to_string())),
|
||||
};
|
||||
|
||||
while let Some((key_bytes, value_bytes)) = iter {
|
||||
scanned += 1;
|
||||
if scanned > max_scan {
|
||||
break;
|
||||
}
|
||||
|
||||
if let Ok(mut value) = serde_json::from_slice::<serde_json::Value>(&value_bytes) {
|
||||
let key = String::from_utf8_lossy(&key_bytes).to_string();
|
||||
match &mut value {
|
||||
serde_json::Value::Object(map) => {
|
||||
map.entry("_id".to_string())
|
||||
.or_insert_with(|| serde_json::Value::String(key));
|
||||
}
|
||||
other => {
|
||||
value = serde_json::json!({"_id": key, "value": other.clone()});
|
||||
}
|
||||
}
|
||||
docs.push(Document::new(value));
|
||||
}
|
||||
|
||||
iter = match cursor.next::<Vec<u8>, Vec<u8>>() {
|
||||
Ok(Some((key, value))) => {
|
||||
if key.starts_with(prefix) {
|
||||
Some((key, value))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Ok(None) => None,
|
||||
Err(e) => return Err(ProjectionError::StorageError(e.to_string())),
|
||||
};
|
||||
}
|
||||
|
||||
Ok(docs)
|
||||
}
|
||||
|
||||
fn commit_kv_txn<F>(&self, f: F) -> Result<(), ProjectionError>
|
||||
where
|
||||
F: FnOnce(
|
||||
&libmdbx::Transaction<'_, RW, NoWriteMap>,
|
||||
&libmdbx::Table<'_>,
|
||||
) -> Result<(), libmdbx::Error>,
|
||||
{
|
||||
let txn = self
|
||||
.storage
|
||||
.db()
|
||||
.begin_rw_txn()
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
let table = txn
|
||||
.open_table(TableNames::KV_STORE)
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
f(&txn, &table).map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
txn.commit()
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn commit_kv_txn_projection<F>(&self, f: F) -> Result<(), ProjectionError>
|
||||
where
|
||||
F: FnOnce(
|
||||
&libmdbx::Transaction<'_, RW, NoWriteMap>,
|
||||
&libmdbx::Table<'_>,
|
||||
) -> Result<(), ProjectionError>,
|
||||
{
|
||||
let txn = self
|
||||
.storage
|
||||
.db()
|
||||
.begin_rw_txn()
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
let table = txn
|
||||
.open_table(TableNames::KV_STORE)
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
f(&txn, &table)?;
|
||||
txn.commit()
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mark_processed_and_advance_checkpoint_in_txn(
|
||||
&self,
|
||||
txn: &libmdbx::Transaction<'_, RW, NoWriteMap>,
|
||||
table: &libmdbx::Table<'_>,
|
||||
checkpoint_key: &CheckpointKey,
|
||||
sequence: StreamSequence,
|
||||
) -> Result<(), ProjectionError> {
|
||||
if let Some(current) = self.get_checkpoint_in_txn(txn, table, checkpoint_key)? {
|
||||
if sequence <= current {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
|
||||
let marker_key = processed_marker_key(checkpoint_key, sequence);
|
||||
txn.put(table, marker_key.as_bytes(), b"1", WriteFlags::empty())
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
|
||||
let mut checkpoint = self
|
||||
.get_checkpoint_in_txn(txn, table, checkpoint_key)?
|
||||
.unwrap_or(0);
|
||||
|
||||
loop {
|
||||
let next = checkpoint.saturating_add(1);
|
||||
if next == 0 {
|
||||
break;
|
||||
}
|
||||
let next_key = processed_marker_key(checkpoint_key, next);
|
||||
let exists: Option<Vec<u8>> = txn
|
||||
.get(table, next_key.as_bytes())
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
if exists.is_none() {
|
||||
break;
|
||||
}
|
||||
let _ = txn
|
||||
.del(table, next_key.as_bytes(), None)
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
checkpoint = next;
|
||||
}
|
||||
|
||||
let checkpoint_bytes = serde_json::to_vec(&Checkpoint::new(checkpoint))
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
txn.put(
|
||||
table,
|
||||
checkpoint_key.as_str().as_bytes(),
|
||||
checkpoint_bytes.as_slice(),
|
||||
WriteFlags::empty(),
|
||||
)
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_checkpoint_in_txn(
|
||||
&self,
|
||||
txn: &libmdbx::Transaction<'_, RW, NoWriteMap>,
|
||||
table: &libmdbx::Table<'_>,
|
||||
checkpoint_key: &CheckpointKey,
|
||||
) -> Result<Option<StreamSequence>, ProjectionError> {
|
||||
let bytes: Option<Vec<u8>> = txn
|
||||
.get(table, checkpoint_key.as_str().as_bytes())
|
||||
.map_err(|e| ProjectionError::StorageError(e.to_string()))?;
|
||||
let Some(bytes) = bytes else {
|
||||
return Ok(None);
|
||||
};
|
||||
let cp: Checkpoint = serde_json::from_slice(&bytes)
|
||||
.map_err(|e| ProjectionError::DecodeError(e.to_string()))?;
|
||||
Ok(Some(cp.sequence))
|
||||
}
|
||||
}
|
||||
|
||||
fn processed_marker_key(checkpoint_key: &CheckpointKey, sequence: StreamSequence) -> String {
|
||||
format!("processed:{}:{}", checkpoint_key.as_str(), sequence)
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum StorageInitError {
|
||||
#[error("Failed to open storage: {0}")]
|
||||
OpenError(#[from] edge_storage::Error),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{CheckpointKey, TenantId, ViewId, ViewKey, ViewType};
|
||||
use serde_json::json;
|
||||
use tempfile::tempdir;
|
||||
|
||||
fn create_test_client() -> (tempfile::TempDir, KvClient) {
|
||||
let dir = tempdir().unwrap();
|
||||
let path = dir.path().join("test.mdbx");
|
||||
let client = KvClient::open(path.to_string_lossy().to_string()).unwrap();
|
||||
(dir, client)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn view_roundtrip_put_get() {
|
||||
let (_dir, client) = create_test_client();
|
||||
let tenant = TenantId::new("t1");
|
||||
let view_type = ViewType::new("User");
|
||||
let view_id = ViewId::new("u1");
|
||||
let view_key = ViewKey::new(&tenant, &view_type, &view_id);
|
||||
let cp_key = CheckpointKey::new(&tenant, &view_type);
|
||||
|
||||
client
|
||||
.commit_view_and_checkpoint(&view_key, &json!({"a": 1}), &cp_key, 5)
|
||||
.unwrap();
|
||||
|
||||
let loaded = client.get_view(&view_key).unwrap().unwrap();
|
||||
assert_eq!(loaded["a"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_roundtrip_put_get() {
|
||||
let (_dir, client) = create_test_client();
|
||||
let tenant = TenantId::new("t1");
|
||||
let view_type = ViewType::new("User");
|
||||
let view_id = ViewId::new("u1");
|
||||
let view_key = ViewKey::new(&tenant, &view_type, &view_id);
|
||||
let cp_key = CheckpointKey::new(&tenant, &view_type);
|
||||
|
||||
client
|
||||
.commit_view_and_checkpoint(&view_key, &json!({"a": 1}), &cp_key, 42)
|
||||
.unwrap();
|
||||
|
||||
let cp = client.get_checkpoint(&cp_key).unwrap().unwrap();
|
||||
assert_eq!(cp, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefix_delete_removes_all_keys_for_tenant_view_type() {
|
||||
let (_dir, client) = create_test_client();
|
||||
let tenant = TenantId::new("t1");
|
||||
let view_type = ViewType::new("User");
|
||||
let other_view_type = ViewType::new("Other");
|
||||
|
||||
let view_id_1 = ViewId::new("u1");
|
||||
let view_id_2 = ViewId::new("u2");
|
||||
let view_key_1 = ViewKey::new(&tenant, &view_type, &view_id_1);
|
||||
let view_key_2 = ViewKey::new(&tenant, &view_type, &view_id_2);
|
||||
let other_key = ViewKey::new(&tenant, &other_view_type, &view_id_1);
|
||||
let cp_key = CheckpointKey::new(&tenant, &view_type);
|
||||
|
||||
client
|
||||
.commit_view_and_checkpoint(&view_key_1, &json!({"a": 1}), &cp_key, 1)
|
||||
.unwrap();
|
||||
client
|
||||
.commit_view_and_checkpoint(&view_key_2, &json!({"a": 2}), &cp_key, 2)
|
||||
.unwrap();
|
||||
client
|
||||
.commit_view_and_checkpoint(&other_key, &json!({"a": 3}), &cp_key, 3)
|
||||
.unwrap();
|
||||
|
||||
client.delete_view_prefix(&tenant, &view_type).unwrap();
|
||||
assert!(client.get_view(&view_key_1).unwrap().is_none());
|
||||
assert!(client.get_view(&view_key_2).unwrap().is_none());
|
||||
assert!(client.get_view(&other_key).unwrap().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomicity_neither_view_nor_checkpoint_committed_on_error() {
|
||||
let (_dir, client) = create_test_client();
|
||||
let tenant = TenantId::new("t1");
|
||||
let view_type = ViewType::new("User");
|
||||
let view_id = ViewId::new("u1");
|
||||
let view_key = ViewKey::new(&tenant, &view_type, &view_id);
|
||||
let cp_key = CheckpointKey::new(&tenant, &view_type);
|
||||
|
||||
let view_bytes = serde_json::to_vec(&json!({"a": 1})).unwrap();
|
||||
let checkpoint_bytes = serde_json::to_vec(&Checkpoint::new(10)).unwrap();
|
||||
|
||||
let result = client.commit_kv_txn(|txn, table| {
|
||||
txn.put(
|
||||
table,
|
||||
view_key.as_str().as_bytes(),
|
||||
view_bytes.as_slice(),
|
||||
WriteFlags::empty(),
|
||||
)?;
|
||||
txn.put(
|
||||
table,
|
||||
cp_key.as_str().as_bytes(),
|
||||
checkpoint_bytes.as_slice(),
|
||||
WriteFlags::empty(),
|
||||
)?;
|
||||
Err(libmdbx::Error::Other(1))
|
||||
});
|
||||
|
||||
assert!(result.is_err());
|
||||
assert!(client.get_view(&view_key).unwrap().is_none());
|
||||
assert!(client.get_checkpoint(&cp_key).unwrap().is_none());
|
||||
}
|
||||
}
|
||||
3
projection/src/storage/mod.rs
Normal file
3
projection/src/storage/mod.rs
Normal file
@@ -0,0 +1,3 @@
|
||||
mod kv;
|
||||
|
||||
pub use kv::{KvClient, StorageInitError};
|
||||
90
projection/src/stream/jetstream.rs
Normal file
90
projection/src/stream/jetstream.rs
Normal file
@@ -0,0 +1,90 @@
|
||||
use crate::config::Settings;
|
||||
use crate::types::ProjectionError;
|
||||
use async_nats::jetstream::{
|
||||
self, consumer::pull::Config as PullConfig, consumer::AckPolicy, consumer::DeliverPolicy,
|
||||
consumer::ReplayPolicy,
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct JetStreamClient {
|
||||
stream: jetstream::stream::Stream,
|
||||
consumer: jetstream::consumer::PullConsumer,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ConsumerOptions {
|
||||
pub durable_name: String,
|
||||
pub filter_subject: String,
|
||||
pub deliver_policy: DeliverPolicy,
|
||||
}
|
||||
|
||||
impl JetStreamClient {
|
||||
pub async fn connect(settings: &Settings) -> Result<Self, ProjectionError> {
|
||||
let filter_subject = settings
|
||||
.subject_filters
|
||||
.first()
|
||||
.cloned()
|
||||
.unwrap_or_else(|| "tenant.*.aggregate.*.*".to_string());
|
||||
|
||||
let options = ConsumerOptions {
|
||||
durable_name: settings.durable_name.clone(),
|
||||
filter_subject,
|
||||
deliver_policy: DeliverPolicy::All,
|
||||
};
|
||||
|
||||
Self::connect_with(settings, options).await
|
||||
}
|
||||
|
||||
pub async fn connect_with(
|
||||
settings: &Settings,
|
||||
options: ConsumerOptions,
|
||||
) -> Result<Self, ProjectionError> {
|
||||
let client = async_nats::connect(&settings.nats_url).await.map_err(|e| {
|
||||
ProjectionError::StreamError(format!("Failed to connect to NATS: {}", e))
|
||||
})?;
|
||||
|
||||
let jetstream = jetstream::new(client);
|
||||
|
||||
let stream = jetstream
|
||||
.get_stream(&settings.stream_name)
|
||||
.await
|
||||
.map_err(|e| ProjectionError::StreamError(format!("Stream not found: {}", e)))?;
|
||||
|
||||
let consumer_config = PullConfig {
|
||||
durable_name: Some(options.durable_name.clone()),
|
||||
deliver_policy: options.deliver_policy,
|
||||
ack_policy: AckPolicy::Explicit,
|
||||
ack_wait: std::time::Duration::from_millis(settings.ack_timeout_ms),
|
||||
filter_subject: options.filter_subject,
|
||||
replay_policy: ReplayPolicy::Instant,
|
||||
max_ack_pending: settings.max_in_flight as i64,
|
||||
max_deliver: settings.max_deliver,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let consumer = stream
|
||||
.get_or_create_consumer(&options.durable_name, consumer_config)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
ProjectionError::StreamError(format!("Consumer creation failed: {}", e))
|
||||
})?;
|
||||
|
||||
Ok(Self { stream, consumer })
|
||||
}
|
||||
|
||||
pub async fn messages(&self) -> Result<jetstream::consumer::pull::Stream, ProjectionError> {
|
||||
self.consumer
|
||||
.messages()
|
||||
.await
|
||||
.map_err(|e| ProjectionError::StreamError(format!("Message stream error: {}", e)))
|
||||
}
|
||||
|
||||
pub async fn stream_last_sequence(&self) -> Result<u64, ProjectionError> {
|
||||
let mut stream = self.stream.clone();
|
||||
let info = stream
|
||||
.info()
|
||||
.await
|
||||
.map_err(|e| ProjectionError::StreamError(e.to_string()))?;
|
||||
Ok(info.state.last_sequence)
|
||||
}
|
||||
}
|
||||
1839
projection/src/stream/mod.rs
Normal file
1839
projection/src/stream/mod.rs
Normal file
File diff suppressed because it is too large
Load Diff
181
projection/src/tenant_placement.rs
Normal file
181
projection/src/tenant_placement.rs
Normal file
@@ -0,0 +1,181 @@
|
||||
use crate::config::Settings;
|
||||
use crate::types::TenantId;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use std::sync::RwLock;
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct TenantPlacement {
|
||||
inner: Arc<RwLock<Inner>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Inner {
|
||||
hosted: Option<HashSet<String>>,
|
||||
draining: HashSet<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct TenantPlacementSnapshot {
|
||||
pub hosted: Option<Vec<String>>,
|
||||
pub draining: Vec<String>,
|
||||
}
|
||||
|
||||
impl TenantPlacement {
|
||||
pub fn load(settings: &Settings) -> Result<Self, String> {
|
||||
let hosted = hosted_tenants_from_settings(settings)?;
|
||||
Ok(Self {
|
||||
inner: Arc::new(RwLock::new(Inner {
|
||||
hosted,
|
||||
draining: HashSet::new(),
|
||||
})),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn reload(&self, settings: &Settings) -> Result<(), String> {
|
||||
let hosted = hosted_tenants_from_settings(settings)?;
|
||||
let mut inner = self
|
||||
.inner
|
||||
.write()
|
||||
.map_err(|_| "tenant placement lock poisoned".to_string())?;
|
||||
inner.hosted = hosted;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn is_hosted(&self, tenant_id: &TenantId) -> bool {
|
||||
let inner = match self.inner.read() {
|
||||
Ok(i) => i,
|
||||
Err(_) => return true,
|
||||
};
|
||||
let Some(hosted) = &inner.hosted else {
|
||||
return true;
|
||||
};
|
||||
hosted.contains(tenant_id.as_str())
|
||||
}
|
||||
|
||||
pub fn is_draining(&self, tenant_id: &TenantId) -> bool {
|
||||
let inner = match self.inner.read() {
|
||||
Ok(i) => i,
|
||||
Err(_) => return false,
|
||||
};
|
||||
inner.draining.contains(tenant_id.as_str())
|
||||
}
|
||||
|
||||
pub fn set_draining(&self, tenant_id: TenantId, draining: bool) -> Result<(), String> {
|
||||
let mut inner = self
|
||||
.inner
|
||||
.write()
|
||||
.map_err(|_| "tenant placement lock poisoned".to_string())?;
|
||||
if draining {
|
||||
inner.draining.insert(tenant_id.as_str().to_string());
|
||||
} else {
|
||||
inner.draining.remove(tenant_id.as_str());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn hosted_count(&self) -> Option<usize> {
|
||||
let inner = self.inner.read().ok()?;
|
||||
inner.hosted.as_ref().map(|s| s.len())
|
||||
}
|
||||
|
||||
pub fn single_hosted_tenant(&self) -> Option<TenantId> {
|
||||
let inner = self.inner.read().ok()?;
|
||||
let hosted = inner.hosted.as_ref()?;
|
||||
if hosted.len() != 1 {
|
||||
return None;
|
||||
}
|
||||
hosted.iter().next().map(|s| TenantId::new(s.to_string()))
|
||||
}
|
||||
|
||||
pub fn snapshot(&self) -> TenantPlacementSnapshot {
|
||||
let inner = self.inner.read();
|
||||
let Ok(inner) = inner else {
|
||||
return TenantPlacementSnapshot {
|
||||
hosted: None,
|
||||
draining: vec![],
|
||||
};
|
||||
};
|
||||
|
||||
let hosted = inner.hosted.as_ref().map(|set| {
|
||||
let mut out = set.iter().cloned().collect::<Vec<_>>();
|
||||
out.sort();
|
||||
out
|
||||
});
|
||||
|
||||
let mut draining = inner.draining.iter().cloned().collect::<Vec<_>>();
|
||||
draining.sort();
|
||||
|
||||
TenantPlacementSnapshot { hosted, draining }
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum PlacementDoc {
|
||||
List(Vec<String>),
|
||||
Map(PlacementMap),
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct PlacementMap {
|
||||
#[serde(default)]
|
||||
hosted_tenants: Vec<String>,
|
||||
#[serde(default)]
|
||||
tenants: Vec<String>,
|
||||
}
|
||||
|
||||
fn hosted_tenants_from_settings(settings: &Settings) -> Result<Option<HashSet<String>>, String> {
|
||||
let mut tenants = Vec::new();
|
||||
|
||||
if let Ok(raw) = std::env::var("PROJECTION_HOSTED_TENANTS") {
|
||||
tenants.extend(
|
||||
raw.split(',')
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty()),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(path) = settings.tenant_placement_path.as_ref() {
|
||||
let raw = std::fs::read_to_string(path)
|
||||
.map_err(|e| format!("failed to read tenant placement file: {}", e))?;
|
||||
let ext = std::path::Path::new(path)
|
||||
.extension()
|
||||
.and_then(|e| e.to_str())
|
||||
.unwrap_or("")
|
||||
.to_ascii_lowercase();
|
||||
|
||||
let doc = if ext == "json" {
|
||||
serde_json::from_str::<PlacementDoc>(&raw)
|
||||
.map_err(|e| format!("failed to parse placement json: {}", e))?
|
||||
} else {
|
||||
serde_yaml::from_str::<PlacementDoc>(&raw)
|
||||
.map_err(|e| format!("failed to parse placement yaml: {}", e))?
|
||||
};
|
||||
|
||||
match doc {
|
||||
PlacementDoc::List(items) => tenants.extend(items),
|
||||
PlacementDoc::Map(map) => {
|
||||
if !map.hosted_tenants.is_empty() {
|
||||
tenants.extend(map.hosted_tenants);
|
||||
} else if !map.tenants.is_empty() {
|
||||
tenants.extend(map.tenants);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut set = tenants
|
||||
.into_iter()
|
||||
.map(|s| s.trim().to_string())
|
||||
.filter(|s| !s.is_empty())
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
if set.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
set.retain(|t| !t.is_empty());
|
||||
Ok(Some(set))
|
||||
}
|
||||
70
projection/src/types/checkpoint.rs
Normal file
70
projection/src/types/checkpoint.rs
Normal file
@@ -0,0 +1,70 @@
|
||||
use crate::types::{TenantId, ViewType};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt;
|
||||
|
||||
pub type StreamSequence = u64;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct CheckpointKey(String);
|
||||
|
||||
impl CheckpointKey {
|
||||
pub fn new(tenant_id: &TenantId, view_type: &ViewType) -> Self {
|
||||
Self(format!(
|
||||
"checkpoint:{}:{}",
|
||||
tenant_id.as_str(),
|
||||
view_type.as_str()
|
||||
))
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for CheckpointKey {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
|
||||
#[serde(default)]
|
||||
pub struct Checkpoint {
|
||||
pub sequence: StreamSequence,
|
||||
pub metadata: Option<serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Checkpoint {
|
||||
pub fn new(sequence: StreamSequence) -> Self {
|
||||
Self {
|
||||
sequence,
|
||||
metadata: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{TenantId, ViewType};
|
||||
|
||||
#[test]
|
||||
fn checkpoint_roundtrip() {
|
||||
let cp = Checkpoint {
|
||||
sequence: 42,
|
||||
metadata: Some(serde_json::json!({"a": 1})),
|
||||
};
|
||||
let bytes = serde_json::to_vec(&cp).unwrap();
|
||||
let decoded: Checkpoint = serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(decoded.sequence, 42);
|
||||
assert_eq!(decoded.metadata.unwrap()["a"], 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_key_format_is_stable() {
|
||||
let tenant = TenantId::new("tenant-a");
|
||||
let view_type = ViewType::new("User");
|
||||
let key = CheckpointKey::new(&tenant, &view_type);
|
||||
assert_eq!(key.as_str(), "checkpoint:tenant-a:User");
|
||||
}
|
||||
}
|
||||
52
projection/src/types/error.rs
Normal file
52
projection/src/types/error.rs
Normal file
@@ -0,0 +1,52 @@
|
||||
use crate::types::{TenantId, ViewType};
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Clone, Error)]
|
||||
pub enum ProjectionError {
|
||||
#[error("Tenant access denied for tenant: {tenant_id}")]
|
||||
TenantAccessDenied { tenant_id: TenantId },
|
||||
|
||||
#[error("Validation error: {0}")]
|
||||
ValidationError(String),
|
||||
|
||||
#[error("Storage error: {0}")]
|
||||
StorageError(String),
|
||||
|
||||
#[error("Stream error: {0}")]
|
||||
StreamError(String),
|
||||
|
||||
#[error("Decode error: {0}")]
|
||||
DecodeError(String),
|
||||
|
||||
#[error("Project error: {0}")]
|
||||
ProjectError(String),
|
||||
|
||||
#[error("Manifest error: {0}")]
|
||||
ManifestError(String),
|
||||
|
||||
#[error("Not found: {0}")]
|
||||
NotFound(String),
|
||||
|
||||
#[error("Unsupported view type: {view_type}")]
|
||||
UnsupportedViewType { view_type: ViewType },
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn error_implements_traits() {
|
||||
let err = ProjectionError::TenantAccessDenied {
|
||||
tenant_id: TenantId::new("other"),
|
||||
};
|
||||
let _ = format!("{}", err);
|
||||
let _: &dyn std::error::Error = &err;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_is_send_sync() {
|
||||
fn assert_send_sync<T: Send + Sync>() {}
|
||||
assert_send_sync::<ProjectionError>();
|
||||
}
|
||||
}
|
||||
80
projection/src/types/event.rs
Normal file
80
projection/src/types/event.rs
Normal file
@@ -0,0 +1,80 @@
|
||||
use crate::types::TenantId;
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use uuid::Uuid;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(default)]
|
||||
pub struct EventEnvelope {
|
||||
pub tenant_id: TenantId,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub event_id: Option<Uuid>,
|
||||
pub aggregate_id: String,
|
||||
pub aggregate_type: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub version: Option<u64>,
|
||||
pub event_type: String,
|
||||
pub payload: serde_json::Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub command_id: Option<Uuid>,
|
||||
pub timestamp: Option<DateTime<Utc>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub correlation_id: Option<shared::CorrelationId>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub traceparent: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub trace_id: Option<shared::TraceId>,
|
||||
}
|
||||
|
||||
impl Default for EventEnvelope {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
tenant_id: TenantId::default(),
|
||||
event_id: None,
|
||||
aggregate_id: String::new(),
|
||||
aggregate_type: String::new(),
|
||||
version: None,
|
||||
event_type: String::new(),
|
||||
payload: serde_json::Value::Null,
|
||||
command_id: None,
|
||||
timestamp: None,
|
||||
correlation_id: None,
|
||||
traceparent: None,
|
||||
trace_id: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn envelope_decoding_ignores_unknown_fields() {
|
||||
let raw = r#"
|
||||
{
|
||||
"tenant_id": "tenant-a",
|
||||
"aggregate_id": "a1",
|
||||
"aggregate_type": "Account",
|
||||
"event_type": "created",
|
||||
"payload": {"x": 1},
|
||||
"timestamp": "2020-01-01T00:00:00Z",
|
||||
"correlation_id": "corr-1",
|
||||
"traceparent": "00-0123456789abcdef0123456789abcdef-1111111111111111-01",
|
||||
"unknown": "ignored"
|
||||
}"#;
|
||||
|
||||
let decoded: EventEnvelope = serde_json::from_str(raw).unwrap();
|
||||
assert_eq!(decoded.tenant_id.as_str(), "tenant-a");
|
||||
assert_eq!(decoded.aggregate_id, "a1");
|
||||
assert_eq!(decoded.payload["x"], 1);
|
||||
assert_eq!(
|
||||
decoded.correlation_id.as_ref().map(|v| v.as_str()),
|
||||
Some("corr-1")
|
||||
);
|
||||
assert_eq!(
|
||||
decoded.traceparent.as_deref(),
|
||||
Some("00-0123456789abcdef0123456789abcdef-1111111111111111-01")
|
||||
);
|
||||
}
|
||||
}
|
||||
104
projection/src/types/id.rs
Normal file
104
projection/src/types/id.rs
Normal file
@@ -0,0 +1,104 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt;
|
||||
use std::str::FromStr;
|
||||
|
||||
pub type TenantId = shared::TenantId;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ViewType(String);
|
||||
|
||||
impl ViewType {
|
||||
pub fn new(ty: impl Into<String>) -> Self {
|
||||
Self(ty.into())
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ViewType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for ViewType {
|
||||
type Err = std::convert::Infallible;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
Ok(Self(s.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for ViewType {
|
||||
fn from(s: &str) -> Self {
|
||||
Self(s.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for ViewType {
|
||||
fn from(s: String) -> Self {
|
||||
Self(s)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ViewId(String);
|
||||
|
||||
impl ViewId {
|
||||
pub fn new(id: impl Into<String>) -> Self {
|
||||
Self(id.into())
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ViewId {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for ViewId {
|
||||
type Err = std::convert::Infallible;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
Ok(Self(s.to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<str> for ViewId {
|
||||
fn as_ref(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn tenant_id_serialization_roundtrip() {
|
||||
let id = TenantId::new("acme-corp");
|
||||
let json = serde_json::to_string(&id).unwrap();
|
||||
let decoded: TenantId = serde_json::from_str(&json).unwrap();
|
||||
assert_eq!(id, decoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tenant_id_default() {
|
||||
let id = TenantId::default();
|
||||
assert!(id.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn view_key_types_are_send_sync() {
|
||||
fn assert_send_sync<T: Send + Sync>() {}
|
||||
assert_send_sync::<TenantId>();
|
||||
assert_send_sync::<ViewType>();
|
||||
assert_send_sync::<ViewId>();
|
||||
}
|
||||
}
|
||||
11
projection/src/types/mod.rs
Normal file
11
projection/src/types/mod.rs
Normal file
@@ -0,0 +1,11 @@
|
||||
mod checkpoint;
|
||||
mod error;
|
||||
mod event;
|
||||
mod id;
|
||||
mod view;
|
||||
|
||||
pub use checkpoint::{Checkpoint, CheckpointKey, StreamSequence};
|
||||
pub use error::ProjectionError;
|
||||
pub use event::EventEnvelope;
|
||||
pub use id::{TenantId, ViewId, ViewType};
|
||||
pub use view::ViewKey;
|
||||
42
projection/src/types/view.rs
Normal file
42
projection/src/types/view.rs
Normal file
@@ -0,0 +1,42 @@
|
||||
use crate::types::{TenantId, ViewId, ViewType};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct ViewKey(String);
|
||||
|
||||
impl ViewKey {
|
||||
pub fn new(tenant_id: &TenantId, view_type: &ViewType, view_id: &ViewId) -> Self {
|
||||
Self(format!(
|
||||
"view:{}:{}:{}",
|
||||
tenant_id.as_str(),
|
||||
view_type.as_str(),
|
||||
view_id.as_str()
|
||||
))
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ViewKey {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::types::{TenantId, ViewId, ViewType};
|
||||
|
||||
#[test]
|
||||
fn view_key_format_is_stable() {
|
||||
let tenant = TenantId::new("tenant-a");
|
||||
let view_type = ViewType::new("User");
|
||||
let view_id = ViewId::new("u1");
|
||||
let key = ViewKey::new(&tenant, &view_type, &view_id);
|
||||
assert_eq!(key.as_str(), "view:tenant-a:User:u1");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user