feat(billing): implement tenant subscription entitlements system (milestones 0-6)

control/api/tests/observability_s3_docker_gated.rs

@@ -0,0 +1,218 @@
+use reqwest::StatusCode;
+use serde_json::json;
+use std::{
+    net::TcpStream,
+    path::PathBuf,
+    process::Command,
+    time::{Duration, Instant},
+};
+
+fn repo_root() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .parent()
+        .and_then(|p| p.parent())
+        .expect("api crate should live under repo root")
+        .to_path_buf()
+}
+
+fn docker_enabled() -> bool {
+    std::env::var("CONTROL_TEST_DOCKER")
+        .ok()
+        .is_some_and(|v| v.trim() == "1")
+}
+
+fn wait_for_tcp(addr: &str, timeout: Duration) -> bool {
+    let start = Instant::now();
+    while start.elapsed() < timeout {
+        if TcpStream::connect_timeout(
+            &addr.parse().expect("invalid socket addr"),
+            Duration::from_secs(1),
+        )
+        .is_ok()
+        {
+            return true;
+        }
+        std::thread::sleep(Duration::from_millis(250));
+    }
+    false
+}
+
+fn mc_ls_bucket(compose: &PathBuf, bucket: &str) -> std::process::Output {
+    // Run inside compose network so it can reach `minio:9000`.
+    Command::new("docker")
+        .args(["compose", "-f"])
+        .arg(compose)
+        .args([
+            "run",
+            "--rm",
+            "minio-init",
+            "/bin/sh",
+            "-lc",
+            &format!(
+                "mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null && mc ls --recursive local/{bucket}"
+            ),
+        ])
+        .output()
+        .expect("failed to run mc ls")
+}
+
+#[tokio::test]
+async fn loki_and_tempo_write_objects_to_minio_in_s3_mode() {
+    if !docker_enabled() {
+        eprintln!("skipping: set CONTROL_TEST_DOCKER=1 to enable docker tests");
+        return;
+    }
+
+    let root = repo_root();
+    let base = root.join("docker-compose.yml");
+    let obs = root.join("observability/docker-compose.yml");
+    let obs_s3 = root.join("observability/docker-compose.s3.yml");
+
+    let up = Command::new("docker")
+        .args(["compose", "-f"])
+        .arg(&base)
+        .args(["-f"])
+        .arg(&obs)
+        .args(["-f"])
+        .arg(&obs_s3)
+        .args(["up", "-d"])
+        .status()
+        .expect("failed to run docker compose up");
+    assert!(up.success(), "docker compose up failed");
+
+    let reachable = wait_for_tcp("127.0.0.1:3100", Duration::from_secs(45))
+        && wait_for_tcp("127.0.0.1:3200", Duration::from_secs(45))
+        && wait_for_tcp("127.0.0.1:9411", Duration::from_secs(45))
+        && wait_for_tcp("127.0.0.1:9000", Duration::from_secs(45));
+    assert!(reachable, "loki/tempo/minio ports not reachable in time");
+
+    let http = reqwest::Client::builder()
+        .timeout(Duration::from_secs(10))
+        .build()
+        .unwrap();
+
+    // Push one log line into Loki.
+    let ts_ns = (std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_nanos())
+    .to_string();
+
+    let push = http
+        .post("http://127.0.0.1:3100/loki/api/v1/push")
+        .json(&json!({
+            "streams": [{
+                "stream": { "app": "cloudlysis-test" },
+                "values": [[ts_ns, "hello from test"]]
+            }]
+        }))
+        .send()
+        .await
+        .expect("loki push request failed");
+    assert!(
+        push.status() == StatusCode::NO_CONTENT,
+        "unexpected loki push status: {}",
+        push.status()
+    );
+
+    // Emit one trace span via Zipkin v2.
+    let zipkin = http
+        .post("http://127.0.0.1:9411/api/v2/spans")
+        .json(&json!([{
+            "traceId": "463ac35c9f6413ad48485a3953bb6124",
+            "id": "a2fb4a1d1a96d312",
+            "name": "test-span",
+            "timestamp": 1700000000000000u64,
+            "duration": 1000u64,
+            "localEndpoint": { "serviceName": "cloudlysis-test" }
+        }]))
+        .send()
+        .await
+        .expect("zipkin post failed");
+    assert!(
+        zipkin.status().is_success(),
+        "zipkin ingest failed: {}",
+        zipkin.status()
+    );
+
+    // Query Loki back to ensure the line is retrievable (not just accepted).
+    // Loki may need a short delay to index.
+    let loki_deadline = Instant::now() + Duration::from_secs(30);
+    let mut loki_ok = false;
+    while Instant::now() < loki_deadline && !loki_ok {
+        let q = http
+            .get("http://127.0.0.1:3100/loki/api/v1/query")
+            .query(&[("query", r#"{app="cloudlysis-test"}"#)])
+            .send()
+            .await
+            .expect("loki query failed");
+        if q.status().is_success() {
+            let v: serde_json::Value = q.json().await.expect("invalid loki query json");
+            // We only need to see any non-empty result.
+            let has = v
+                .get("data")
+                .and_then(|d| d.get("result"))
+                .and_then(|r| r.as_array())
+                .is_some_and(|a| !a.is_empty());
+            if has {
+                loki_ok = true;
+                break;
+            }
+        }
+        tokio::time::sleep(Duration::from_millis(500)).await;
+    }
+
+    // Query Tempo back by trace id (Zipkin traceId used above).
+    let tempo_deadline = Instant::now() + Duration::from_secs(30);
+    let mut tempo_ok = false;
+    while Instant::now() < tempo_deadline && !tempo_ok {
+        let res = http
+            .get("http://127.0.0.1:3200/api/traces/463ac35c9f6413ad48485a3953bb6124")
+            .send()
+            .await
+            .expect("tempo get trace failed");
+        if res.status().is_success() {
+            tempo_ok = true;
+            break;
+        }
+        tokio::time::sleep(Duration::from_millis(500)).await;
+    }
+
+    // Poll buckets until at least one object appears.
+    let deadline = Instant::now() + Duration::from_secs(45);
+    let mut loki_has_objects = false;
+    let mut tempo_has_objects = false;
+    while Instant::now() < deadline && (!loki_has_objects || !tempo_has_objects) {
+        let loki_out = mc_ls_bucket(&base, "cloudlysis-loki");
+        if loki_out.status.success() && !loki_out.stdout.is_empty() {
+            loki_has_objects = true;
+        }
+
+        let tempo_out = mc_ls_bucket(&base, "cloudlysis-tempo");
+        if tempo_out.status.success() && !tempo_out.stdout.is_empty() {
+            tempo_has_objects = true;
+        }
+
+        if !loki_has_objects || !tempo_has_objects {
+            tokio::time::sleep(Duration::from_millis(500)).await;
+        }
+    }
+
+    let _ = Command::new("docker")
+        .args(["compose", "-f"])
+        .arg(&base)
+        .args(["-f"])
+        .arg(&obs)
+        .args(["-f"])
+        .arg(&obs_s3)
+        .args(["down", "-v"])
+        .status();
+
+    assert!(loki_has_objects, "expected Loki to write objects to MinIO");
+    assert!(
+        tempo_has_objects,
+        "expected Tempo to write objects to MinIO"
+    );
+    assert!(loki_ok, "expected Loki query to return a result");
+    assert!(tempo_ok, "expected Tempo to return the ingested trace");
+}