cloudlysis/control/api/tests/observability_s3_docker_gated.rs

use reqwest::StatusCode;
use serde_json::json;
use std::{
    net::TcpStream,
    path::PathBuf,
    process::Command,
    time::{Duration, Instant},
};

fn repo_root() -> PathBuf {
    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
        .parent()
        .and_then(|p| p.parent())
        .expect("api crate should live under repo root")
        .to_path_buf()
}

fn docker_enabled() -> bool {
    std::env::var("CONTROL_TEST_DOCKER")
        .ok()
        .is_some_and(|v| v.trim() == "1")
}

fn wait_for_tcp(addr: &str, timeout: Duration) -> bool {
    let start = Instant::now();
    while start.elapsed() < timeout {
        if TcpStream::connect_timeout(
            &addr.parse().expect("invalid socket addr"),
            Duration::from_secs(1),
        )
        .is_ok()
        {
            return true;
        }
        std::thread::sleep(Duration::from_millis(250));
    }
    false
}

fn mc_ls_bucket(compose: &PathBuf, bucket: &str) -> std::process::Output {
    // Run inside compose network so it can reach `minio:9000`.
    Command::new("docker")
        .args(["compose", "-f"])
        .arg(compose)
        .args([
            "run",
            "--rm",
            "minio-init",
            "/bin/sh",
            "-lc",
            &format!(
                "mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null && mc ls --recursive local/{bucket}"
            ),
        ])
        .output()
        .expect("failed to run mc ls")
}

#[tokio::test]
async fn loki_and_tempo_write_objects_to_minio_in_s3_mode() {
    if !docker_enabled() {
        eprintln!("skipping: set CONTROL_TEST_DOCKER=1 to enable docker tests");
        return;
    }

    let root = repo_root();
    let base = root.join("docker-compose.yml");
    let obs = root.join("observability/docker-compose.yml");
    let obs_s3 = root.join("observability/docker-compose.s3.yml");

    let up = Command::new("docker")
        .args(["compose", "-f"])
        .arg(&base)
        .args(["-f"])
        .arg(&obs)
        .args(["-f"])
        .arg(&obs_s3)
        .args(["up", "-d"])
        .status()
        .expect("failed to run docker compose up");
    assert!(up.success(), "docker compose up failed");

    let reachable = wait_for_tcp("127.0.0.1:3100", Duration::from_secs(45))
        && wait_for_tcp("127.0.0.1:3200", Duration::from_secs(45))
        && wait_for_tcp("127.0.0.1:9411", Duration::from_secs(45))
        && wait_for_tcp("127.0.0.1:9000", Duration::from_secs(45));
    assert!(reachable, "loki/tempo/minio ports not reachable in time");

    let http = reqwest::Client::builder()
        .timeout(Duration::from_secs(10))
        .build()
        .unwrap();

    // Push one log line into Loki.
    let ts_ns = (std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap()
        .as_nanos())
    .to_string();

    let push = http
        .post("http://127.0.0.1:3100/loki/api/v1/push")
        .json(&json!({
            "streams": [{
                "stream": { "app": "cloudlysis-test" },
                "values": [[ts_ns, "hello from test"]]
            }]
        }))
        .send()
        .await
        .expect("loki push request failed");
    assert!(
        push.status() == StatusCode::NO_CONTENT,
        "unexpected loki push status: {}",
        push.status()
    );

    // Emit one trace span via Zipkin v2.
    let zipkin = http
        .post("http://127.0.0.1:9411/api/v2/spans")
        .json(&json!([{
            "traceId": "463ac35c9f6413ad48485a3953bb6124",
            "id": "a2fb4a1d1a96d312",
            "name": "test-span",
            "timestamp": 1700000000000000u64,
            "duration": 1000u64,
            "localEndpoint": { "serviceName": "cloudlysis-test" }
        }]))
        .send()
        .await
        .expect("zipkin post failed");
    assert!(
        zipkin.status().is_success(),
        "zipkin ingest failed: {}",
        zipkin.status()
    );

    // Query Loki back to ensure the line is retrievable (not just accepted).
    // Loki may need a short delay to index.
    let loki_deadline = Instant::now() + Duration::from_secs(30);
    let mut loki_ok = false;
    while Instant::now() < loki_deadline && !loki_ok {
        let q = http
            .get("http://127.0.0.1:3100/loki/api/v1/query")
            .query(&[("query", r#"{app="cloudlysis-test"}"#)])
            .send()
            .await
            .expect("loki query failed");
        if q.status().is_success() {
            let v: serde_json::Value = q.json().await.expect("invalid loki query json");
            // We only need to see any non-empty result.
            let has = v
                .get("data")
                .and_then(|d| d.get("result"))
                .and_then(|r| r.as_array())
                .is_some_and(|a| !a.is_empty());
            if has {
                loki_ok = true;
                break;
            }
        }
        tokio::time::sleep(Duration::from_millis(500)).await;
    }

    // Query Tempo back by trace id (Zipkin traceId used above).
    let tempo_deadline = Instant::now() + Duration::from_secs(30);
    let mut tempo_ok = false;
    while Instant::now() < tempo_deadline && !tempo_ok {
        let res = http
            .get("http://127.0.0.1:3200/api/traces/463ac35c9f6413ad48485a3953bb6124")
            .send()
            .await
            .expect("tempo get trace failed");
        if res.status().is_success() {
            tempo_ok = true;
            break;
        }
        tokio::time::sleep(Duration::from_millis(500)).await;
    }

    // Poll buckets until at least one object appears.
    let deadline = Instant::now() + Duration::from_secs(45);
    let mut loki_has_objects = false;
    let mut tempo_has_objects = false;
    while Instant::now() < deadline && (!loki_has_objects || !tempo_has_objects) {
        let loki_out = mc_ls_bucket(&base, "cloudlysis-loki");
        if loki_out.status.success() && !loki_out.stdout.is_empty() {
            loki_has_objects = true;
        }

        let tempo_out = mc_ls_bucket(&base, "cloudlysis-tempo");
        if tempo_out.status.success() && !tempo_out.stdout.is_empty() {
            tempo_has_objects = true;
        }

        if !loki_has_objects || !tempo_has_objects {
            tokio::time::sleep(Duration::from_millis(500)).await;
        }
    }

    let _ = Command::new("docker")
        .args(["compose", "-f"])
        .arg(&base)
        .args(["-f"])
        .arg(&obs)
        .args(["-f"])
        .arg(&obs_s3)
        .args(["down", "-v"])
        .status();

    assert!(loki_has_objects, "expected Loki to write objects to MinIO");
    assert!(
        tempo_has_objects,
        "expected Tempo to write objects to MinIO"
    );
    assert!(loki_ok, "expected Loki query to return a result");
    assert!(tempo_ok, "expected Tempo to return the ingested trace");
}