transport: complete M0–M7

shared: add stream+consumer policy helpers; NATS context header builder

aggregate/runner/projection: centralize stream validation and header usage; set bounded consumer params

projection: add QueryService gRPC and wire into main; settings include PROJECTION_GRPC_ADDR

gateway: gRPC routing to Projection/Runner with deadlines; bounded read-only retries; pooled gRPC channels (bounded LRU+TTL); admin proxy forwards to gRPC; probes use concurrency limiter + TTL cache

runner: add RunnerAdmin gRPC server (drain, status, reload) and wire into main; settings include RUNNER_GRPC_ADDR

tests: add gateway authz for runner admin, projection tenant isolation, runner admin drain semantics

docs: update TRANSPORT_DEVELOPMENT_PLAN to reflect completed milestones and details

projection/Cargo.toml

@@ -29,8 +29,14 @@ uuid = { version = "1", features = ["v7", "serde"] }
 chrono = { version = "0.4", features = ["serde"] }
 futures = "0.3"
 axum = "0.7"
+prost = "0.13"
+tonic = { version = "0.12", default-features = false, features = ["codegen", "prost", "transport"] }
 v8 = { version = "0.106", optional = true }
 
 [dev-dependencies]
 tempfile = "3"
 tower = "0.5"
+
+[build-dependencies]
+tonic-build = { version = "0.12", default-features = false, features = ["prost"] }
+protoc-bin-vendored = "3"

projection/build.rs

@@ -0,0 +1,8 @@
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let protoc = protoc_bin_vendored::protoc_bin_path()?;
+    std::env::set_var("PROTOC", protoc);
+
+    tonic_build::configure().compile_protos(&["proto/query.proto"], &["proto"])?;
+
+    Ok(())
+}

projection/proto/query.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+package projection.gateway.v1;
+
+service QueryService {
+  rpc ExecuteQuery(QueryRequest) returns (QueryResponse);
+}
+
+message QueryRequest {
+  string tenant_id = 1;
+  string view_type = 2;
+  string uqf = 3;
+}
+
+message QueryResponse {
+  string json = 1;
+}

projection/src/config/settings.rs

@@ -18,6 +18,7 @@ pub struct Settings {
     pub max_deliver: i64,
     pub consumer_mode: ConsumerMode,
     pub http_addr: String,
+    pub grpc_addr: String,
     pub storage_backoff_ms: u64,
     pub storage_backoff_max_ms: u64,
 }
@@ -47,6 +48,7 @@ impl Default for Settings {
             max_deliver: 10,
             consumer_mode: ConsumerMode::Single,
             http_addr: "0.0.0.0:8080".to_string(),
+            grpc_addr: "0.0.0.0:9090".to_string(),
             storage_backoff_ms: 50,
             storage_backoff_max_ms: 2_000,
         }
@@ -181,6 +183,12 @@ impl Settings {
             }
         }
 
+        if let Ok(addr) = std::env::var("PROJECTION_GRPC_ADDR") {
+            if !addr.trim().is_empty() {
+                self.grpc_addr = addr;
+            }
+        }
+
         if let Ok(ms) = std::env::var("PROJECTION_STORAGE_BACKOFF_MS") {
             if let Ok(value) = ms.parse() {
                 self.storage_backoff_ms = value;
@@ -210,6 +218,12 @@ impl Settings {
         if self.durable_name.is_empty() {
             return Err("Durable name is required".to_string());
         }
+        if self.http_addr.trim().is_empty() {
+            return Err("HTTP addr is required".to_string());
+        }
+        if self.grpc_addr.trim().is_empty() {
+            return Err("gRPC addr is required".to_string());
+        }
         Ok(())
     }
 }

projection/src/grpc.rs

@@ -0,0 +1,181 @@
+use crate::config::Settings;
+use crate::query::{QueryError, QueryRequest, QueryService};
+use crate::tenant_placement::TenantPlacement;
+use crate::types::{ProjectionError, TenantId, ViewType};
+use crate::ProjectionManifest;
+
+pub mod proto {
+    tonic::include_proto!("projection.gateway.v1");
+}
+
+#[derive(Clone)]
+pub struct GrpcQueryService {
+    placement: TenantPlacement,
+    manifest: ProjectionManifest,
+    query: QueryService,
+}
+
+impl GrpcQueryService {
+    pub fn new(
+        placement: TenantPlacement,
+        manifest: ProjectionManifest,
+        query: QueryService,
+    ) -> Self {
+        Self {
+            placement,
+            manifest,
+            query,
+        }
+    }
+}
+
+#[tonic::async_trait]
+impl proto::query_service_server::QueryService for GrpcQueryService {
+    async fn execute_query(
+        &self,
+        request: tonic::Request<proto::QueryRequest>,
+    ) -> Result<tonic::Response<proto::QueryResponse>, tonic::Status> {
+        let md_tenant = request
+            .metadata()
+            .get(shared::HEADER_X_TENANT_ID)
+            .and_then(|v| v.to_str().ok())
+            .map(|s| s.trim())
+            .filter(|s| !s.is_empty())
+            .map(|s| s.to_string());
+
+        let req = request.into_inner();
+        let tenant_id = req.tenant_id.trim().to_string();
+        if tenant_id.is_empty() {
+            return Err(tonic::Status::invalid_argument("tenant_id is required"));
+        }
+        if let Some(md_tenant) = md_tenant.as_deref() {
+            if md_tenant != tenant_id {
+                return Err(tonic::Status::permission_denied("tenant mismatch"));
+            }
+        }
+
+        let tenant_id = TenantId::new(tenant_id);
+
+        if self.placement.is_draining(&tenant_id) {
+            return Err(tonic::Status::unavailable("tenant is draining"));
+        }
+        if !self.placement.is_hosted(&tenant_id) {
+            return Err(tonic::Status::permission_denied("tenant not hosted"));
+        }
+
+        let view_type_raw = req.view_type.trim().to_string();
+        if view_type_raw.is_empty() {
+            return Err(tonic::Status::invalid_argument("view_type is required"));
+        }
+        let view_type = ViewType::new(view_type_raw.clone());
+        if self.manifest.get(&view_type).is_none() {
+            return Err(tonic::Status::not_found("unknown view type"));
+        }
+
+        let uqf = req.uqf;
+        if uqf.trim().is_empty() {
+            return Err(tonic::Status::invalid_argument("uqf is required"));
+        }
+
+        let request = QueryRequest {
+            tenant_id,
+            view_type,
+            uqf,
+        };
+
+        let response = self.query.query(request).map_err(map_query_error)?;
+        let json =
+            serde_json::to_string(&response).map_err(|e| tonic::Status::internal(e.to_string()))?;
+
+        Ok(tonic::Response::new(proto::QueryResponse { json }))
+    }
+}
+
+fn map_query_error(err: QueryError) -> tonic::Status {
+    match err {
+        QueryError::InvalidQuery(e) => tonic::Status::invalid_argument(e),
+        QueryError::Execution(e) => tonic::Status::internal(e),
+    }
+}
+
+pub async fn serve(
+    settings: Settings,
+    placement: TenantPlacement,
+    manifest: ProjectionManifest,
+    query: QueryService,
+    shutdown: std::sync::Arc<tokio::sync::Notify>,
+) -> Result<(), ProjectionError> {
+    let addr: std::net::SocketAddr = settings
+        .grpc_addr
+        .parse::<std::net::SocketAddr>()
+        .map_err(|e| ProjectionError::StreamError(e.to_string()))?;
+
+    tonic::transport::Server::builder()
+        .add_service(proto::query_service_server::QueryServiceServer::new(
+            GrpcQueryService::new(placement, manifest, query),
+        ))
+        .serve_with_shutdown(addr, async move { shutdown.notified().await })
+        .await
+        .map_err(|e| ProjectionError::StreamError(e.to_string()))?;
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::proto::query_service_server::QueryService as QueryServiceGrpc;
+    use super::*;
+    use crate::storage::KvClient;
+    use crate::types::{CheckpointKey, StreamSequence, ViewId, ViewKey};
+    use serde_json::json;
+
+    #[tokio::test]
+    async fn rejects_tenant_not_hosted() {
+        let storage = KvClient::in_memory();
+        let tenant_a = TenantId::new("tenant-a");
+        let view_type = ViewType::new("User");
+        let cp = CheckpointKey::new(&tenant_a, &view_type);
+        let key = ViewKey::new(&tenant_a, &view_type, &ViewId::new("u1"));
+        storage
+            .commit_view_and_checkpoint(&key, &json!({"id":"u1"}), &cp, 1 as StreamSequence)
+            .unwrap();
+
+        let query = QueryService::new(storage);
+        let mut manifest = ProjectionManifest::new();
+        manifest.register(crate::project::ProjectionDefinition {
+            view_type: view_type.clone(),
+            project_program: "unused".to_string(),
+        });
+
+        let placement = TenantPlacement::with_hosted(Some(vec!["tenant-a".to_string()]));
+        let svc = GrpcQueryService::new(placement, manifest, query);
+
+        let request = proto::QueryRequest {
+            tenant_id: "tenant-b".to_string(),
+            view_type: "User".to_string(),
+            uqf: "{}".to_string(),
+        };
+        let err = QueryServiceGrpc::execute_query(&svc, tonic::Request::new(request))
+            .await
+            .unwrap_err();
+        assert_eq!(err.code(), tonic::Code::PermissionDenied);
+    }
+
+    #[tokio::test]
+    async fn rejects_unknown_view_type() {
+        let query = QueryService::new(KvClient::in_memory());
+        let placement = TenantPlacement::with_hosted(None);
+        let manifest = ProjectionManifest::new();
+        let svc = GrpcQueryService::new(placement, manifest, query);
+
+        let request = proto::QueryRequest {
+            tenant_id: "tenant-a".to_string(),
+            view_type: "Missing".to_string(),
+            uqf: "{}".to_string(),
+        };
+        let err = QueryServiceGrpc::execute_query(&svc, tonic::Request::new(request))
+            .await
+            .unwrap_err();
+        assert_eq!(err.code(), tonic::Code::NotFound);
+    }
+}

projection/src/http/mod.rs

@@ -289,7 +289,7 @@ fn tenant_from_headers(
     headers: &HeaderMap,
 ) -> Result<TenantId, TenantHeaderError> {
     let header_value = headers
-        .get("x-tenant-id")
+        .get(shared::HEADER_X_TENANT_ID)
         .and_then(|v| v.to_str().ok())
         .map(|s| s.trim())
         .unwrap_or("");

projection/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod config;
+pub mod grpc;
 pub mod http;
 pub mod observability;
 pub mod project;

projection/src/main.rs

@@ -54,10 +54,27 @@ async fn serve() {
         }
     };
 
+    let grpc_manifest = http_state.manifest.clone();
+    let grpc_query = http_state.query.clone();
+
     let http_shutdown = shutdown.clone();
     let http_task =
         tokio::spawn(async move { projection::http::serve(http_state, http_shutdown).await });
 
+    let grpc_shutdown = shutdown.clone();
+    let grpc_settings = settings.clone();
+    let grpc_placement = tenant_placement.clone();
+    let grpc_task = tokio::spawn(async move {
+        projection::grpc::serve(
+            grpc_settings,
+            grpc_placement,
+            grpc_manifest,
+            grpc_query,
+            grpc_shutdown,
+        )
+        .await
+    });
+
     let signal_shutdown = shutdown.clone();
     let signal_ready = ready.clone();
     let signal_draining = draining.clone();
@@ -103,6 +120,7 @@ async fn serve() {
     shutdown.notify_waiters();
 
     let _ = http_task.await;
+    let _ = grpc_task.await;
 
     match worker_result {
         Ok(Ok(())) => {}

projection/src/stream/jetstream.rs

@@ -2,7 +2,7 @@ use crate::config::Settings;
 use crate::types::ProjectionError;
 use async_nats::jetstream::{
     self, consumer::pull::Config as PullConfig, consumer::AckPolicy, consumer::DeliverPolicy,
-    consumer::ReplayPolicy,
+    consumer::ReplayPolicy, stream::Config as StreamConfig,
 };
 
 #[derive(Debug, Clone)]
@@ -24,7 +24,7 @@ impl JetStreamClient {
             .subject_filters
             .first()
             .cloned()
-            .unwrap_or_else(|| "tenant.*.aggregate.*.*".to_string());
+            .unwrap_or_else(|| shared::NATS_SUBJECT_AGGREGATE_EVENTS_ALL.to_string());
 
         let options = ConsumerOptions {
             durable_name: settings.durable_name.clone(),
@@ -45,20 +45,32 @@ impl JetStreamClient {
 
         let jetstream = jetstream::new(client);
 
-        let stream = jetstream
-            .get_stream(&settings.stream_name)
+        let expected = stream_policy_config(&settings.stream_name);
+        let mut stream = jetstream
+            .get_or_create_stream(expected.clone())
             .await
-            .map_err(|e| ProjectionError::StreamError(format!("Stream not found: {}", e)))?;
+            .map_err(|e| ProjectionError::StreamError(format!("Stream error: {}", e)))?;
 
+        let info = stream
+            .info()
+            .await
+            .map_err(|e| ProjectionError::StreamError(format!("Stream info error: {}", e)))?;
+        validate_stream_config(&expected, &info.config)?;
+
+        let policy = shared::consumer_policy_from_parts(
+            settings.ack_timeout_ms,
+            settings.max_in_flight,
+            settings.max_deliver,
+        );
         let consumer_config = PullConfig {
             durable_name: Some(options.durable_name.clone()),
             deliver_policy: options.deliver_policy,
             ack_policy: AckPolicy::Explicit,
-            ack_wait: std::time::Duration::from_millis(settings.ack_timeout_ms),
+            ack_wait: policy.ack_wait,
             filter_subject: options.filter_subject,
             replay_policy: ReplayPolicy::Instant,
-            max_ack_pending: settings.max_in_flight as i64,
-            max_deliver: settings.max_deliver,
+            max_ack_pending: policy.max_ack_pending,
+            max_deliver: policy.max_deliver,
             ..Default::default()
         };
 
@@ -88,3 +100,43 @@ impl JetStreamClient {
         Ok(info.state.last_sequence)
     }
 }
+
+fn stream_policy_config(name: &str) -> StreamConfig {
+    let policy = shared::stream_policy_defaults(
+        name.to_string(),
+        vec![shared::NATS_SUBJECT_AGGREGATE_EVENTS_ALL.to_string()],
+    );
+    StreamConfig {
+        name: policy.name,
+        subjects: policy.subjects,
+        max_messages: policy.max_messages,
+        max_bytes: policy.max_bytes,
+        max_age: policy.max_age,
+        duplicate_window: policy.duplicate_window,
+        ..Default::default()
+    }
+}
+
+fn validate_stream_config(
+    expected: &StreamConfig,
+    actual: &StreamConfig,
+) -> Result<(), ProjectionError> {
+    let expected = shared::stream_policy_from_parts(
+        expected.name.as_str(),
+        expected.subjects.clone(),
+        expected.max_messages,
+        expected.max_bytes,
+        expected.max_age,
+        expected.duplicate_window,
+    );
+    let actual = shared::stream_policy_from_parts(
+        actual.name.as_str(),
+        actual.subjects.clone(),
+        actual.max_messages,
+        actual.max_bytes,
+        actual.max_age,
+        actual.duplicate_window,
+    );
+    shared::validate_stream_policy(&expected, &actual)
+        .map_err(|e| ProjectionError::StreamError(e.to_string()))
+}

projection/src/stream/mod.rs

@@ -101,7 +101,7 @@ async fn run_projection_per_view_with_options(
         .subject_filters
         .first()
         .cloned()
-        .unwrap_or_else(|| "tenant.*.aggregate.*.*".to_string());
+        .unwrap_or_else(|| shared::NATS_SUBJECT_AGGREGATE_EVENTS_ALL.to_string());
 
     let shutdown = options.shutdown.clone();
     let ready = options.ready.clone();
@@ -220,7 +220,7 @@ pub async fn run_projection_with_options(
             .consumer_filter_subject
             .clone()
             .or_else(|| settings.subject_filters.first().cloned())
-            .unwrap_or_else(|| "tenant.*.aggregate.*.*".to_string());
+            .unwrap_or_else(|| shared::NATS_SUBJECT_AGGREGATE_EVENTS_ALL.to_string());
         let deliver_policy = options
             .consumer_deliver_policy
             .unwrap_or(DeliverPolicy::All);
@@ -301,7 +301,7 @@ pub async fn run_projection_with_options(
         let sequence = info.stream_sequence;
         let delivered = info.delivered;
 
-        let envelope: EventEnvelope = match serde_json::from_slice(&msg.payload) {
+        let mut envelope: EventEnvelope = match serde_json::from_slice(&msg.payload) {
             Ok(e) => e,
             Err(e) => {
                 tracing::error!(error = %e, "Failed to decode event envelope");
@@ -310,6 +310,53 @@ pub async fn run_projection_with_options(
             }
         };
 
+        if let Some(headers) = msg.headers.as_ref() {
+            if envelope.correlation_id.is_none() {
+                let correlation_id = headers
+                    .get(shared::NATS_HEADER_CORRELATION_ID)
+                    .or_else(|| headers.get(shared::HEADER_X_CORRELATION_ID))
+                    .map(|v| v.to_string())
+                    .map(|s| s.trim().to_string())
+                    .filter(|s| !s.is_empty());
+                if let Some(correlation_id) = correlation_id {
+                    envelope.correlation_id = Some(shared::CorrelationId::new(correlation_id));
+                }
+            }
+
+            if envelope.traceparent.is_none() {
+                let traceparent = headers
+                    .get(shared::HEADER_TRACEPARENT)
+                    .map(|v| v.to_string())
+                    .map(|s| s.trim().to_string())
+                    .filter(|s| !s.is_empty());
+                if let Some(traceparent) = traceparent {
+                    let normalized = shared::normalize_traceparent(Some(&traceparent));
+                    envelope.traceparent = Some(normalized);
+                }
+            }
+
+            if envelope.trace_id.is_none() {
+                let trace_id = headers
+                    .get(shared::HEADER_TRACE_ID)
+                    .map(|v| v.to_string())
+                    .map(|s| s.trim().to_string())
+                    .filter(|s| !s.is_empty())
+                    .map(shared::TraceId::new)
+                    .filter(|t| t.is_valid_hex_32());
+                if let Some(trace_id) = trace_id {
+                    envelope.trace_id = Some(trace_id);
+                }
+            }
+        }
+
+        if envelope.trace_id.is_none() {
+            if let Some(traceparent) = envelope.traceparent.as_deref() {
+                if let Some(trace_id) = shared::trace_id_from_traceparent(traceparent) {
+                    envelope.trace_id = Some(shared::TraceId::new(trace_id.to_string()));
+                }
+            }
+        }
+
         let tenant_id = resolve_tenant_id(&settings, &envelope);
 
         if let Some(filter) = &options.tenant_filter {
@@ -460,7 +507,7 @@ pub async fn rebuild_view(
         Uuid::now_v7()
     );
     let filter_subject = if tenant_id.is_empty() {
-        "tenant.*.aggregate.*.*".to_string()
+        shared::NATS_SUBJECT_AGGREGATE_EVENTS_ALL.to_string()
     } else {
         format!("tenant.{}.aggregate.*.*", tenant_id.as_str())
     };
@@ -499,7 +546,7 @@ pub async fn backfill_to_tail(
 ) -> Result<(), ProjectionError> {
     let durable_name = format!("{}_backfill_{}", settings.durable_name, Uuid::now_v7());
     let filter_subject = if tenant_id.is_empty() {
-        "tenant.*.aggregate.*.*".to_string()
+        shared::NATS_SUBJECT_AGGREGATE_EVENTS_ALL.to_string()
     } else {
         format!("tenant.{}.aggregate.*.*", tenant_id.as_str())
     };

projection/src/tenant_placement.rs

@@ -23,6 +23,22 @@ pub struct TenantPlacementSnapshot {
 }
 
 impl TenantPlacement {
+    pub fn with_hosted(hosted: Option<Vec<String>>) -> Self {
+        let hosted = hosted.map(|items| {
+            items
+                .into_iter()
+                .map(|s| s.trim().to_string())
+                .filter(|s| !s.is_empty())
+                .collect::<HashSet<_>>()
+        });
+        Self {
+            inner: Arc::new(RwLock::new(Inner {
+                hosted,
+                draining: HashSet::new(),
+            })),
+        }
+    }
+
     pub fn load(settings: &Settings) -> Result<Self, String> {
         let hosted = hosted_tenants_from_settings(settings)?;
         Ok(Self {