chore: add LocalStack for local s3 (#647)

Co-authored-by: Mariano A. Nicolini <mariano.nicolini.91@gmail.com>
Co-authored-by: Santos Rosati <rosatisantos@gmail.com>
Co-authored-by: Tatu <65305492+srosati@users.noreply.github.com>

Author: 3 people

.gitignore

@@ -10,5 +10,6 @@ scripts/test_files/gnark_groth16_bn254_infinite_script/infinite_proofs/**
 batcher/aligned/batch_inclusion_responses/*
 **/aligned_verification_data
 **/broadcast
+volume
 
 nonce_*.bin

Makefile

@@ -191,6 +191,11 @@ batcher_start: ./batcher/aligned-batcher/.env user_fund_payment_service
 	@echo "Starting Batcher..."
 	@cargo +nightly-2024-04-17 run --manifest-path ./batcher/aligned-batcher/Cargo.toml --release -- --config ./config-files/config-batcher.yaml --env-file ./batcher/aligned-batcher/.env
 
+batcher_start_local: ./batcher/aligned-batcher/.env user_fund_payment_service
+	@echo "Starting Batcher..."
+	@$(MAKE) run_storage &
+	@cargo +nightly-2024-04-17 run --manifest-path ./batcher/aligned-batcher/Cargo.toml --release -- --config ./config-files/config-batcher.yaml --env-file ./batcher/aligned-batcher/.env.dev
+
 install_batcher:
 	@cargo +nightly-2024-04-17 install --path batcher/aligned-batcher
 
@@ -396,6 +401,11 @@ run_metrics: ## Run metrics using metrics-docker-compose.yaml
 	@echo "Running metrics..."
 	@docker-compose -f metrics-docker-compose.yaml up
 
+__STORAGE__:
+run_storage: ## Run storage using storage-docker-compose.yaml
+	@echo "Running storage..."
+	@docker-compose -f storage-docker-compose.yaml up
+
 __DEPLOYMENT__:
 deploy_aligned_contracts: ## Deploy Aligned Contracts
 	@echo "Deploying Aligned Contracts..."

batcher/aligned-batcher/.env.dev

@@ -0,0 +1,8 @@
+AWS_SECRET_ACCESS_KEY=test
+AWS_REGION=us-east-2
+AWS_ACCESS_KEY_ID=test
+AWS_BUCKET_NAME=aligned.storage
+STORAGE_ENDPOINT=http://localhost:4566/aligned.storage
+LOCALSTACK_ENDPOINT_URL=http://localhost:4566
+RUST_LOG=info
+RUST_BACKTRACE=1

batcher/aligned-batcher/.env.example

@@ -2,4 +2,6 @@ AWS_SECRET_ACCESS_KEY=<secret_access_key>
 AWS_REGION=<region>
 AWS_ACCESS_KEY_ID=<access_key_id>
 AWS_BUCKET_NAME=<bucket_name>
+STORAGE_ENDPOINT=<storage_endpoint>
+LOCALSTACK_ENDPOINT_URL=<endpoint_url to use for localstack>
 RUST_LOG=<log_level> # info, debug, error, warn, etc.

batcher/aligned-batcher/src/lib.rs

@@ -87,6 +87,7 @@ impl BatchState {
 pub struct Batcher {
     s3_client: S3Client,
     s3_bucket_name: String,
+    storage_endpoint: String,
     eth_ws_provider: Provider<Ws>,
     payment_service: BatcherPaymentService,
     batch_state: Mutex<BatchState>,
@@ -102,10 +103,17 @@ pub struct Batcher {
 impl Batcher {
     pub async fn new(config_file: String) -> Self {
         dotenv().ok();
+
+        // https://docs.aws.amazon.com/sdk-for-rust/latest/dg/localstack.html
+        let endpoint_url = env::var("LOCALSTACK_ENDPOINT_URL").ok();
+
         let s3_bucket_name =
             env::var("AWS_BUCKET_NAME").expect("AWS_BUCKET_NAME not found in environment");
 
-        let s3_client = s3::create_client().await;
+        let storage_endpoint =
+            env::var("STORAGE_ENDPOINT").expect("STORAGE_ENDPOINT not found in environment");
+
+        let s3_client = s3::create_client(endpoint_url).await;
 
         let config = ConfigFromYaml::new(config_file);
         let deployment_output =
@@ -147,6 +155,7 @@ impl Batcher {
         Self {
             s3_client,
             s3_bucket_name,
+            storage_endpoint,
             eth_ws_provider,
             payment_service,
             batch_state: Mutex::new(BatchState::new()),
@@ -593,7 +602,7 @@ impl Batcher {
         info!("Batch sent to S3 with name: {}", file_name);
 
         info!("Uploading batch to contract");
-        let batch_data_pointer = "https://".to_owned() + &self.s3_bucket_name + "/" + &file_name;
+        let batch_data_pointer: String = "".to_owned() + &self.storage_endpoint + "/" + &file_name;
 
         let num_proofs_in_batch = leaves.len();
 

batcher/aligned-batcher/src/s3/mod.rs

@@ -4,14 +4,23 @@ use aws_sdk_s3::error::SdkError;
 use aws_sdk_s3::operation::put_object::{PutObjectError, PutObjectOutput};
 use aws_sdk_s3::primitives::ByteStream;
 use aws_sdk_s3::Client;
+use log::info;
 
-pub async fn create_client() -> Client {
+pub async fn create_client(endpoint_url: Option<String>) -> Client {
     let region_provider = RegionProviderChain::default_provider().or_else("us-east-2");
-    let config = aws_config::defaults(BehaviorVersion::latest())
-        .region(region_provider)
-        .load()
-        .await;
-    Client::new(&config)
+    let mut config = aws_config::defaults(BehaviorVersion::latest()).region(region_provider);
+    if let Some(endpoint_url) = &endpoint_url {
+        info!("Using custom endpoint: {}", endpoint_url);
+        config = config.endpoint_url(endpoint_url);
+    }
+    let config = config.load().await;
+
+    let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&config);
+    if endpoint_url.is_some() {
+        info!("Forcing path style for custom endpoint");
+        s3_config_builder = s3_config_builder.force_path_style(true);
+    }
+    Client::from_conf(s3_config_builder.build())
 }
 
 pub async fn upload_object(

scripts/init-s3.py

@@ -0,0 +1,10 @@
+import boto3
+
+s3_client = boto3.client(
+    "s3",
+    endpoint_url=f"http://localhost:4566",
+    aws_access_key_id="test",
+    aws_secret_access_key="test"
+)
+
+s3_client.create_bucket(Bucket="aligned.storage")

storage-docker-compose.yaml

@@ -0,0 +1,15 @@
+services:
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}"
+    image: localstack/localstack:s3-latest
+    ports:
+      - "127.0.0.1:4566:4566"            # LocalStack Gateway
+    environment:
+      - DEBUG=${DEBUG:-0}
+      - DEFAULT_REGION=us-east-2
+      - AWS_ACCESS_KEY_ID=test
+      - AWS_SECRET_ACCESS_KEY=test
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "./scripts/init-s3.py:/etc/localstack/init/ready.d/init-s3.py"  # bucket initialization script