wip:milestone 0 fixes

storage/TESTS.md

@@ -0,0 +1,42 @@
+# Storage Backend Tests
+
+## Test Structure
+
+### Unit Tests (storage/src/backend.rs)
+- test_backend_initialization
+- test_put_and_get_object  
+- test_delete_object
+
+### Integration Tests (storage/tests/)
+- integration_tests.rs - Handler tests
+- test_utils.rs - Test utilities
+
+## Running Tests
+
+```bash
+# Start test services
+docker-compose -f docker-compose.test.yml up -d
+
+# Run tests
+cd storage && ./scripts/run_tests.sh
+
+# Or manually
+export TEST_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/madbase_test"
+export TEST_S3_ENDPOINT="http://localhost:9000"
+cargo test --lib storage::backend -- --ignored
+```
+
+## Requirements
+
+- PostgreSQL 15+
+- MinIO or S3-compatible service
+
+## Environment Variables
+
+| Variable | Default |
+|----------|---------|
+| TEST_DATABASE_URL | postgresql://postgres:postgres@localhost:5432/madbase_test |
+| TEST_S3_ENDPOINT | http://localhost:9000 |
+| TEST_S3_ACCESS_KEY | minioadmin |
+| TEST_S3_SECRET_KEY | minioadmin |
+| TEST_S3_BUCKET | madbase-test |

storage/scripts/run_tests.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+echo "Running MadBase Storage Tests"
+export TEST_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/madbase_test"
+export TEST_S3_ENDPOINT="http://localhost:9000"
+export TEST_S3_ACCESS_KEY="minioadmin"
+export TEST_S3_SECRET_KEY="minioadmin"
+export TEST_S3_BUCKET="madbase-test"
+cargo test --lib storage::backend -- --nocapture
+cargo test --test integration_tests -- --nocapture
+echo "Tests complete!"

storage/src/backend.rs

@@ -0,0 +1,163 @@
+use aws_sdk_s3::{primitives::ByteStream, Client as AwsClient};
+use aws_config::BehaviorVersion;
+use aws_sdk_s3::config::Credentials;
+use aws_sdk_s3::config::Region;
+use anyhow::Result;
+use async_trait::async_trait;
+use bytes::Bytes;
+use std::env;
+
+/// Storage backend trait for supporting multiple S3-compatible services
+#[async_trait]
+pub trait StorageBackend: Send + Sync {
+    async fn put_object(&self, bucket: &str, key: &str, data: Bytes) -> Result<()>;
+    async fn get_object(&self, bucket: &str, key: &str) -> Result<Bytes>;
+    async fn delete_object(&self, bucket: &str, key: &str) -> Result<()>;
+    async fn create_bucket(&self, bucket: &str) -> Result<()>;
+}
+
+/// AWS SDK S3 implementation (for Hetzner Bucket Storage and AWS S3)
+pub struct AwsS3Backend {
+    client: AwsClient,
+    bucket_name: String,
+}
+
+impl AwsS3Backend {
+    pub async fn new() -> Result<Self> {
+        let endpoint = env::var("S3_ENDPOINT")
+            .unwrap_or_else(|_| "https://fsn1.your-objectstorage.com".to_string()); // Hetzner default
+        let access_key = env::var("S3_ACCESS_KEY")
+            .or_else(|_| env::var("MINIO_ROOT_USER"))
+            .expect("S3_ACCESS_KEY or MINIO_ROOT_USER must be set");
+        let secret_key = env::var("S3_SECRET_KEY")
+            .or_else(|_| env::var("MINIO_ROOT_PASSWORD"))
+            .expect("S3_SECRET_KEY or MINIO_ROOT_PASSWORD must be set");
+        let bucket_name = env::var("S3_BUCKET")
+            .unwrap_or_else(|_| "madbase".to_string());
+        let region = env::var("S3_REGION")
+            .unwrap_or_else(|_| "us-east-1".to_string());
+
+        tracing::info!("Initializing AWS S3 Backend");
+        tracing::info!("  Endpoint: {}", endpoint);
+        tracing::info!("  Bucket: {}", bucket_name);
+        tracing::info!("  Region: {}", region);
+
+        // Build AWS config with custom endpoint
+        let aws_config = aws_config::defaults(BehaviorVersion::latest())
+            .region(Region::new(region.clone()))
+            .endpoint_url(&endpoint)
+            .credentials_provider(Credentials::new(
+                access_key.clone(),
+                secret_key.clone(),
+                None,
+                None,
+                "static",
+            ))
+            .load()
+            .await;
+
+        let s3_config = aws_sdk_s3::config::Builder::from(&aws_config)
+            .endpoint_url(&endpoint)
+            .force_path_style(true) // Required for MinIO and custom S3 endpoints
+            .build();
+
+        let client = AwsClient::from_conf(s3_config);
+
+        Ok(Self {
+            client,
+            bucket_name,
+        })
+    }
+
+    pub fn bucket_name(&self) -> &str {
+        &self.bucket_name
+    }
+
+    pub fn client(&self) -> &AwsClient {
+        &self.client
+    }
+}
+
+#[async_trait]
+impl StorageBackend for AwsS3Backend {
+    async fn put_object(&self, _bucket: &str, key: &str, data: Bytes) -> Result<()> {
+        self.client
+            .put_object()
+            .bucket(&self.bucket_name)
+            .key(key)
+            .body(ByteStream::from(data))
+            .send()
+            .await?;
+        Ok(())
+    }
+
+    async fn get_object(&self, _bucket: &str, key: &str) -> Result<Bytes> {
+        let resp = self.client
+            .get_object()
+            .bucket(&self.bucket_name)
+            .key(key)
+            .send()
+            .await?;
+        
+        Ok(resp.body.collect().await?.into_bytes())
+    }
+
+    async fn delete_object(&self, _bucket: &str, key: &str) -> Result<()> {
+        self.client
+            .delete_object()
+            .bucket(&self.bucket_name)
+            .key(key)
+            .send()
+            .await?;
+        Ok(())
+    }
+
+    async fn create_bucket(&self, _bucket: &str) -> Result<()> {
+        // Try to create bucket, ignore if it already exists
+        let _ = self.client.create_bucket()
+            .bucket(&self.bucket_name)
+            .send()
+            .await;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use bytes::Bytes;
+
+    /// Helper to create a test backend
+    async fn create_test_backend() -> AwsS3Backend {
+        // Set test environment variables
+        env::set_var("S3_ENDPOINT", "http://localhost:9000");
+        env::set_var("S3_ACCESS_KEY", "test_access_key");
+        env::set_var("S3_SECRET_KEY", "test_secret_key");
+        env::set_var("S3_BUCKET", "test-bucket");
+        env::set_var("S3_REGION", "us-east-1");
+
+        AwsS3Backend::new().await.expect("Failed to create test backend")
+    }
+
+    #[tokio::test]
+    #[ignore]
+    async fn test_backend_initialization() {
+        let backend = create_test_backend().await;
+        assert_eq!(backend.bucket_name(), "test-bucket");
+    }
+
+    #[tokio::test]
+    #[ignore]
+    async fn test_put_and_get_object() {
+        let backend = create_test_backend().await;
+        let test_data = Bytes::from("Hello, World!");
+        let test_key = "test/file.txt";
+
+        let put_result = backend.put_object("test-bucket", test_key, test_data.clone()).await;
+        assert!(put_result.is_ok());
+
+        let get_result = backend.get_object("test-bucket", test_key).await;
+        assert!(get_result.is_ok());
+        assert_eq!(get_result.unwrap(), test_data);
+    }
+}

storage/tests/integration_tests.rs

@@ -0,0 +1,39 @@
+// Integration tests for storage handlers
+use axum::{body::Body, http::{HeaderMap, StatusCode}};
+use bytes::Bytes;
+use sqlx::PgPool;
+
+async fn setup_test_db() -> PgPool {
+    let database_url = std::env::var("TEST_DATABASE_URL")
+        .unwrap_or_else(|_| "postgresql://postgres:postgres@localhost:5432/madbase_test".to_string());
+    sqlx::PgPool::connect(&database_url).await.expect("Failed to connect")
+}
+
+async fn create_test_bucket(pool: &PgPool, bucket_id: &str) {
+    sqlx::query("INSERT INTO storage.buckets (id, name, public) VALUES ($1, $2, true) ON CONFLICT (id) DO NOTHING")
+        .bind(bucket_id)
+        .bind(format!("test_{}", bucket_id))
+        .execute(pool)
+        .await
+        .expect("Failed to create test bucket");
+}
+
+#[cfg(test)]
+mod handler_tests {
+    use super::*;
+
+    #[tokio::test]
+    #[ignore]
+    async fn test_list_buckets_empty() {
+        let pool = setup_test_db().await;
+        assert!(true, "Test placeholder - requires auth context setup");
+    }
+
+    #[tokio::test]
+    #[ignore]
+    async fn test_upload_and_download_file() {
+        let pool = setup_test_db().await;
+        create_test_bucket(&pool, "test-upload").await;
+        assert!(true, "Test placeholder - requires S3 mock");
+    }
+}

storage/tests/test_utils.rs

@@ -0,0 +1,60 @@
+// Test utilities for storage module
+use aws_sdk_s3::{Client, config::Region};
+use aws_config::BehaviorVersion;
+use aws_sdk_s3::config::Credentials;
+use bytes::Bytes;
+use std::env;
+use sqlx::PgPool;
+
+pub struct TestConfig {
+    pub database_url: String,
+    pub s3_endpoint: String,
+    pub s3_access_key: String,
+    pub s3_secret_key: String,
+    pub s3_bucket: String,
+}
+
+impl Default for TestConfig {
+    fn default() -> Self {
+        Self {
+            database_url: env::var("TEST_DATABASE_URL").unwrap_or_else(|_| "postgresql://postgres:postgres@localhost:5432/madbase_test".to_string()),
+            s3_endpoint: env::var("TEST_S3_ENDPOINT").unwrap_or_else(|_| "http://localhost:9000".to_string()),
+            s3_access_key: env::var("TEST_S3_ACCESS_KEY").unwrap_or_else(|_| "minioadmin".to_string()),
+            s3_secret_key: env::var("TEST_S3_SECRET_KEY").unwrap_or_else(|_| "minioadmin".to_string()),
+            s3_bucket: env::var("TEST_S3_BUCKET").unwrap_or_else(|_| "madbase-test".to_string()),
+        }
+    }
+}
+
+pub fn generate_test_data(size: usize) -> Bytes {
+    let data: Vec<u8> = (0..size).map(|i| (i % 256) as u8).collect();
+    Bytes::from(data)
+}
+
+pub async fn create_test_bucket(pool: &PgPool, bucket_id: &str, public: bool) {
+    sqlx::query("INSERT INTO storage.buckets (id, name, public) VALUES ($1, $2, $3) ON CONFLICT (id) DO UPDATE SET name = $2, public = $3")
+        .bind(bucket_id)
+        .bind(format!("test_bucket_{}", bucket_id))
+        .bind(public)
+        .execute(pool)
+        .await
+        .expect("Failed to create test bucket");
+}
+
+pub async fn cleanup_test_data(pool: &PgPool, bucket_id: &str) {
+    sqlx::query("DELETE FROM storage.objects WHERE bucket_id = $1").bind(bucket_id).execute(pool).await.ok();
+    sqlx::query("DELETE FROM storage.buckets WHERE id = $1").bind(bucket_id).execute(pool).await.ok();
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_generate_test_data() {
+        let data = generate_test_data(1024);
+        assert_eq!(data.len(), 1024);
+        assert_eq!(data[0], 0);
+        assert_eq!(data[255], 255);
+    }
+}