madapes/cloudlysis
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Monorepo consolidation: workspace, shared types, transport plans, docker/swam assets
Some checks failed
ci / rust (push) Failing after 2m34s
Details
ci / ui (push) Failing after 30s
Details

Browse Source
This commit is contained in:
Vlad Durnea
2026-03-30 11:40:42 +03:00
parent 7e7041cf8b
commit 1298d9a3df
246 changed files with 55434 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
observability/INSTRUMENTATION.md Normal file
View File

@@ -0,0 +1,44 @@
# Instrumentation Requirements (Control Plane Dashboards)
## Global Conventions
- Build/version labeling: every service exports a gauge metric `*_build_info{service,version,git_sha} 1`
- Correlation identifiers:
- Logs include `correlation_id` and `request_id` fields for all request spans
- Traces propagate `traceparent` end-to-end and expose trace IDs in logs
- Cardinality safety:
- `tenant_id` must not be a label on high-frequency metrics unless bounded (sampling, rollups, or explicit allowlist)
## Dashboard: Noisy Neighbor & Tenant Health
Required metrics (examples):
- `http_request_duration_ms_bucket{service,route,method,status}` histogram
- `job_duration_ms_bucket{job_kind,status}` histogram for control-plane jobs
- Optional bounded tenant signals:
- `tenant_active_jobs{tenant_id}` only if tenant count is bounded and enforced
## Dashboard: API Regression & Deployment
Required metrics (examples):
- `*_build_info{service,version,git_sha} 1`
- `http_request_duration_ms_bucket{service,route,method,status}`
- `http_requests_total{service,route,method,status}`
Deploy markers:
- Grafana annotations for deploy events (vertical markers) or a low-cardinality metric like:
- `deploy_event{service,version,git_sha} 1` (sparse, emitted once per deploy)
## Dashboard: Storage & Event Bus Bottlenecks
Required metrics (examples):
- Storage:
- `process_resident_memory_bytes{service}`
- `disk_io_time_seconds_total{device}` (node-exporter)
- `mdbx_*` or equivalent libmdbx metrics if exposed by storage services
- Event bus / JetStream:
- `nats_*` / `jetstream_*` metrics for consumer lag, ack latency, stream bytes, and redeliveries
## Dashboard: Infrastructure Exhaustion
Required metrics (examples):
- Node exporter:
- `node_cpu_seconds_total`
- `node_memory_MemAvailable_bytes`
- `node_filesystem_avail_bytes`
- Container/service level:
- `process_open_fds`
- `tokio_*` runtime metrics (if enabled) for saturation indicators

62
observability/docker-compose.yml Normal file
View File

@@ -0,0 +1,62 @@
services:
victoria-metrics:
image: victoriametrics/victoria-metrics:v1.120.0
ports:
- "8428:8428"
command:
- "-retentionPeriod=30d"
volumes:
- victoria-metrics-data:/victoria-metrics-data
vmagent:
image: victoriametrics/vmagent:v1.120.0
depends_on:
- victoria-metrics
ports:
- "8429:8429"
command:
- "-promscrape.config=/etc/vmagent/scrape.yml"
- "-remoteWrite.url=http://victoria-metrics:8428/api/v1/write"
volumes:
- ./vmagent/scrape.yml:/etc/vmagent/scrape.yml:ro
loki:
image: grafana/loki:3.5.5
ports:
- "3100:3100"
command:
- "-config.file=/etc/loki/config.yml"
volumes:
- ./loki/config.yml:/etc/loki/config.yml:ro
- loki-data:/loki
tempo:
image: grafana/tempo:2.8.2
ports:
- "3200:3200"
- "4317:4317"
- "4318:4318"
command:
- "-config.file=/etc/tempo/config.yml"
volumes:
- ./tempo/config.yml:/etc/tempo/config.yml:ro
- tempo-data:/var/tempo
grafana:
image: grafana/grafana:12.1.1
depends_on:
- victoria-metrics
- loki
- tempo
ports:
- "3000:3000"
volumes:
- grafana-data:/var/lib/grafana
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
volumes:
grafana-data:
loki-data:
tempo-data:
victoria-metrics-data:

39
observability/grafana/dashboards/api_regression_deployment.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "API Regression & Deployment",
"uid": "cloudlysis-api-regression-deploy",
"version": 1
}

39
observability/grafana/dashboards/cluster.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "Cluster / Orchestrator",
"uid": "cloudlysis-cluster",
"version": 1
}

39
observability/grafana/dashboards/event_bus.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "Event Bus / JetStream",
"uid": "cloudlysis-event-bus",
"version": 1
}

39
observability/grafana/dashboards/http_detail.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "HTTP Detail",
"uid": "cloudlysis-http-detail",
"version": 1
}

39
observability/grafana/dashboards/infrastructure_exhaustion.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "Infrastructure Exhaustion",
"uid": "cloudlysis-infra-exhaustion",
"version": 1
}

39
observability/grafana/dashboards/logs.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "Logs",
"uid": "cloudlysis-logs",
"version": 1
}

39
observability/grafana/dashboards/noisy_neighbor_tenant_health.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "Noisy Neighbor & Tenant Health",
"uid": "cloudlysis-noisy-neighbor",
"version": 1
}

39
observability/grafana/dashboards/operations_overview.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "Operations Overview",
"uid": "cloudlysis-ops-overview",
"version": 1
}

39
observability/grafana/dashboards/storage.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "Storage",
"uid": "cloudlysis-storage",
"version": 1
}

39
observability/grafana/dashboards/storage_event_bus_bottlenecks.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "Storage & Event Bus Bottlenecks",
"uid": "cloudlysis-storage-event-bus",
"version": 1
}

39
observability/grafana/dashboards/traces.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "Traces",
"uid": "cloudlysis-traces",
"version": 1
}

39
observability/grafana/dashboards/workers.json Normal file
View File

@@ -0,0 +1,39 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"liveNow": false,
"panels": [],
"refresh": "10s",
"schemaVersion": 39,
"style": "dark",
"tags": ["cloudlysis"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timezone": "",
"title": "Workers (Runner)",
"uid": "cloudlysis-workers",
"version": 1
}

8
observability/grafana/provisioning/dashboards/dashboards.yml Normal file
View File

@@ -0,0 +1,8 @@
apiVersion: 1
providers:
- name: cloudlysis
type: file
updateIntervalSeconds: 10
options:
path: /var/lib/grafana/dashboards

21
observability/grafana/provisioning/datasources/datasources.yml Normal file
View File

@@ -0,0 +1,21 @@
apiVersion: 1
datasources:
- name: VictoriaMetrics
type: prometheus
access: proxy
url: http://victoria-metrics:8428
isDefault: true
editable: false
- name: Loki
type: loki
access: proxy
url: http://loki:3100
editable: false
- name: Tempo
type: tempo
access: proxy
url: http://tempo:3200
editable: false

28
observability/loki/config.yml Normal file
View File

@@ -0,0 +1,28 @@
auth_enabled: false
server:
http_listen_port: 3100
common:
path_prefix: /loki
storage:
filesystem:
chunks_directory: /loki/chunks
rules_directory: /loki/rules
replication_factor: 1
ring:
kvstore:
store: inmemory
schema_config:
configs:
- from: 2025-01-01
store: tsdb
object_store: filesystem
schema: v13
index:
prefix: index_
period: 24h
limits_config:
allow_structured_metadata: true

30
observability/tempo/config.yml Normal file
View File

@@ -0,0 +1,30 @@
server:
http_listen_port: 3200
distributor:
receivers:
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
ingester:
max_block_bytes: 1000000
trace_idle_period: 10s
compactor:
compaction:
block_retention: 24h
storage:
trace:
backend: local
local:
path: /var/tempo/traces
overrides:
defaults:
metrics_generator:
processors: []

16
observability/vmagent/scrape.yml Normal file
View File

@@ -0,0 +1,16 @@
global:
scrape_interval: 15s
scrape_configs:
- job_name: victoria-metrics
static_configs:
- targets: ["victoria-metrics:8428"]
- job_name: vmagent
static_configs:
- targets: ["vmagent:8429"]
- job_name: control-api
metrics_path: /metrics
static_configs:
- targets: ["host.docker.internal:8080"]