use reqwest::StatusCode; use serde_json::json; use std::{ net::TcpStream, path::PathBuf, process::Command, time::{Duration, Instant}, }; fn repo_root() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() .and_then(|p| p.parent()) .expect("api crate should live under repo root") .to_path_buf() } fn docker_enabled() -> bool { std::env::var("CONTROL_TEST_DOCKER") .ok() .is_some_and(|v| v.trim() == "1") } fn wait_for_tcp(addr: &str, timeout: Duration) -> bool { let start = Instant::now(); while start.elapsed() < timeout { if TcpStream::connect_timeout( &addr.parse().expect("invalid socket addr"), Duration::from_secs(1), ) .is_ok() { return true; } std::thread::sleep(Duration::from_millis(250)); } false } fn mc_ls_bucket(compose: &PathBuf, bucket: &str) -> std::process::Output { // Run inside compose network so it can reach `minio:9000`. Command::new("docker") .args(["compose", "-f"]) .arg(compose) .args([ "run", "--rm", "minio-init", "/bin/sh", "-lc", &format!( "mc alias set local http://minio:9000 minioadmin minioadmin >/dev/null && mc ls --recursive local/{bucket}" ), ]) .output() .expect("failed to run mc ls") } #[tokio::test] async fn loki_and_tempo_write_objects_to_minio_in_s3_mode() { if !docker_enabled() { eprintln!("skipping: set CONTROL_TEST_DOCKER=1 to enable docker tests"); return; } let root = repo_root(); let base = root.join("docker-compose.yml"); let obs = root.join("observability/docker-compose.yml"); let obs_s3 = root.join("observability/docker-compose.s3.yml"); let up = Command::new("docker") .args(["compose", "-f"]) .arg(&base) .args(["-f"]) .arg(&obs) .args(["-f"]) .arg(&obs_s3) .args(["up", "-d"]) .status() .expect("failed to run docker compose up"); assert!(up.success(), "docker compose up failed"); let reachable = wait_for_tcp("127.0.0.1:3100", Duration::from_secs(45)) && wait_for_tcp("127.0.0.1:3200", Duration::from_secs(45)) && wait_for_tcp("127.0.0.1:9411", Duration::from_secs(45)) && wait_for_tcp("127.0.0.1:9000", Duration::from_secs(45)); assert!(reachable, "loki/tempo/minio ports not reachable in time"); let http = reqwest::Client::builder() .timeout(Duration::from_secs(10)) .build() .unwrap(); // Push one log line into Loki. let ts_ns = (std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos()) .to_string(); let push = http .post("http://127.0.0.1:3100/loki/api/v1/push") .json(&json!({ "streams": [{ "stream": { "app": "cloudlysis-test" }, "values": [[ts_ns, "hello from test"]] }] })) .send() .await .expect("loki push request failed"); assert!( push.status() == StatusCode::NO_CONTENT, "unexpected loki push status: {}", push.status() ); // Emit one trace span via Zipkin v2. let zipkin = http .post("http://127.0.0.1:9411/api/v2/spans") .json(&json!([{ "traceId": "463ac35c9f6413ad48485a3953bb6124", "id": "a2fb4a1d1a96d312", "name": "test-span", "timestamp": 1700000000000000u64, "duration": 1000u64, "localEndpoint": { "serviceName": "cloudlysis-test" } }])) .send() .await .expect("zipkin post failed"); assert!( zipkin.status().is_success(), "zipkin ingest failed: {}", zipkin.status() ); // Query Loki back to ensure the line is retrievable (not just accepted). // Loki may need a short delay to index. let loki_deadline = Instant::now() + Duration::from_secs(30); let mut loki_ok = false; while Instant::now() < loki_deadline && !loki_ok { let q = http .get("http://127.0.0.1:3100/loki/api/v1/query") .query(&[("query", r#"{app="cloudlysis-test"}"#)]) .send() .await .expect("loki query failed"); if q.status().is_success() { let v: serde_json::Value = q.json().await.expect("invalid loki query json"); // We only need to see any non-empty result. let has = v .get("data") .and_then(|d| d.get("result")) .and_then(|r| r.as_array()) .is_some_and(|a| !a.is_empty()); if has { loki_ok = true; break; } } tokio::time::sleep(Duration::from_millis(500)).await; } // Query Tempo back by trace id (Zipkin traceId used above). let tempo_deadline = Instant::now() + Duration::from_secs(30); let mut tempo_ok = false; while Instant::now() < tempo_deadline && !tempo_ok { let res = http .get("http://127.0.0.1:3200/api/traces/463ac35c9f6413ad48485a3953bb6124") .send() .await .expect("tempo get trace failed"); if res.status().is_success() { tempo_ok = true; break; } tokio::time::sleep(Duration::from_millis(500)).await; } // Poll buckets until at least one object appears. let deadline = Instant::now() + Duration::from_secs(45); let mut loki_has_objects = false; let mut tempo_has_objects = false; while Instant::now() < deadline && (!loki_has_objects || !tempo_has_objects) { let loki_out = mc_ls_bucket(&base, "cloudlysis-loki"); if loki_out.status.success() && !loki_out.stdout.is_empty() { loki_has_objects = true; } let tempo_out = mc_ls_bucket(&base, "cloudlysis-tempo"); if tempo_out.status.success() && !tempo_out.stdout.is_empty() { tempo_has_objects = true; } if !loki_has_objects || !tempo_has_objects { tokio::time::sleep(Duration::from_millis(500)).await; } } let _ = Command::new("docker") .args(["compose", "-f"]) .arg(&base) .args(["-f"]) .arg(&obs) .args(["-f"]) .arg(&obs_s3) .args(["down", "-v"]) .status(); assert!(loki_has_objects, "expected Loki to write objects to MinIO"); assert!( tempo_has_objects, "expected Tempo to write objects to MinIO" ); assert!(loki_ok, "expected Loki query to return a result"); assert!(tempo_ok, "expected Tempo to return the ingested trace"); }