Merge pull request #180 from andrewm4894/add-deploy-to-flyio

Add deploy to flyio

Author: andrewm4894

.example.env

@@ -18,11 +18,15 @@ ANOMSTACK_SNOWFLAKE_WAREHOUSE=
 ANOMSTACK_AWS_ACCESS_KEY_ID=
 ANOMSTACK_AWS_SECRET_ACCESS_KEY=
 
+# admin authentication (for Fly.io deployment and admin access)
+ANOMSTACK_ADMIN_USERNAME=admin
+ANOMSTACK_ADMIN_PASSWORD=anomstack2024
+
 # local duckdb path for testing/dev quickstart
-# For local development (outside Docker)
+# For local development (outside containers)
 # ANOMSTACK_DUCKDB_PATH=tmpdata/anomstack-duckdb.db
-# For Docker containers (uses named volume)
-ANOMSTACK_DUCKDB_PATH=/metrics_db/duckdb/anomstack.db
+# For container deployments (Docker Compose, Fly.io, etc.)
+ANOMSTACK_DUCKDB_PATH=/data/anomstack.db
 # example using motherduck
 # https://motherduck.com/docs/getting-started/connect-query-from-python/choose-database/
 # ANOMSTACK_DUCKDB_PATH=md:anomstack
@@ -100,6 +104,10 @@ ANOMSTACK_LLM_PLATFORM=openai
 # some dagster env vars
 DAGSTER_LOG_LEVEL=DEBUG
 DAGSTER_CONCURRENCY=4
+# Code server host for workspace configuration
+# For Docker Compose: leave blank (defaults to "anomstack_code")
+# For Fly.io single container: set to "localhost"
+DAGSTER_CODE_SERVER_HOST=
 
 # max runtime for a job in dagster
 # https://docs.dagster.io/deployment/run-monitoring#general-run-timeouts

DOCKER.md

@@ -43,7 +43,7 @@ make docker
 - **Restart Policy**: `always`
 - **Volumes**:
   - `./tmp:/opt/dagster/app/tmp` (temporary files)
-  - `anomstack_metrics_duckdb:/metrics_db/duckdb` (DuckDB data)
+  - `anomstack_metrics_duckdb:/data` (DuckDB data)
 
 ### 3. Dagster UI (`anomstack_dagit`)
 - **Image**: `andrewm4894/anomstack_dagster:latest`
@@ -70,7 +70,7 @@ make docker
 
 ### Named Volumes
 - **`anomstack_metrics_duckdb`**: Persistent storage for DuckDB metrics database
-  - **Mount Point**: `/metrics_db/duckdb/`
+  - **Mount Point**: `/data/`
   - **Purpose**: Stores time-series metrics data
   - **Persistence**: Data survives container restarts
 
@@ -86,7 +86,7 @@ Create a `.env` file based on `.example.env`:
 
 ```bash
 # Database paths
-ANOMSTACK_DUCKDB_PATH=/metrics_db/duckdb/anomstack.db
+ANOMSTACK_DUCKDB_PATH=/data/anomstack.db
 ANOMSTACK_SQLITE_PATH=tmpdata/anomstack-sqlite.db
 
 # PostgreSQL
@@ -191,7 +191,7 @@ make docker-restart-[service]
 ## Data Persistence
 
 ### DuckDB Database
-- **Location**: `/metrics_db/duckdb/anomstack.db` (inside containers)
+- **Location**: `/data/anomstack.db` (inside containers)
 - **Volume**: `anomstack_metrics_duckdb`
 - **Backup**: Use `docker volume` commands to backup
 

Makefile

@@ -131,7 +131,7 @@ docker-restart:
 # reload configuration without restarting containers (hot reload)
 reload-config:
 	@echo "🔄 Reloading Anomstack configuration..."
-	python3 scripts/reload_config.py
+	python3 scripts/configuration/reload_config.py
 
 # enable automatic config reloading via Dagster scheduled job
 enable-auto-reload:
@@ -159,6 +159,39 @@ docker-prune:
 	docker compose down -v --remove-orphans
 	docker system prune -a -f
 
+# =============================================================================
+# FLY.IO DEPLOYMENT
+# =============================================================================
+
+.PHONY: fly-validate fly-preview fly-deploy fly-status fly-logs fly-ssh
+
+# validate fly.io configuration
+fly-validate:
+	./scripts/deployment/validate_fly_config.sh
+
+# preview what environment variables will be set as Fly secrets
+fly-preview:
+	./scripts/deployment/preview_fly_secrets.sh
+
+# deploy to fly.io (reads .env file automatically)
+fly-deploy:
+	./scripts/deployment/deploy_fly.sh
+
+# check fly.io app status (requires app name as FLY_APP env var)
+fly-status:
+	@if [ -z "$$FLY_APP" ]; then echo "Set FLY_APP environment variable"; exit 1; fi
+	fly status -a $$FLY_APP
+
+# view fly.io app logs (requires app name as FLY_APP env var)
+fly-logs:
+	@if [ -z "$$FLY_APP" ]; then echo "Set FLY_APP environment variable"; exit 1; fi
+	fly logs -f -a $$FLY_APP
+
+# ssh into fly.io app (requires app name as FLY_APP env var)
+fly-ssh:
+	@if [ -z "$$FLY_APP" ]; then echo "Set FLY_APP environment variable"; exit 1; fi
+	fly ssh console -a $$FLY_APP
+
 # =============================================================================
 # RESET OPERATIONS
 # =============================================================================
@@ -307,11 +340,11 @@ requirements-install:
 
 # run the PostHog example ingest function
 posthog-example:
-	python scripts/posthog_example.py
+	python scripts/examples/posthog_example.py
 
 # kill any dagster runs exceeding configured timeout
 kill-long-runs:
-	python scripts/kill_long_running_tasks.py
+	python scripts/maintenance/kill_long_running_tasks.py
 
 # run docker in dev mode with correct environment
 docker-dev-env:

README.md

@@ -46,6 +46,7 @@ Anomaly list view:
 - [Quickstart](#quickstart)
   - [GitHub Codespaces](#github-codespaces)
   - [Dagster Cloud](#dagster-cloud)
+  - [Fly.io Deployment](#flyio-deployment)
   - [Docker](#docker)
   - [Replit](#replit)
   - [Local Python env](#local-python-env)
@@ -132,6 +133,7 @@ Supported ways to run this project:
         <tr>
             <th align="center"><a href="#local-python-env">Python Env</a></th>
             <th align="center"><a href="#docker">Docker</a></th>
+            <th align="center"><a href="#fly-io-deployment">Fly.io</a></th>
             <th align="center"><a href="#dagster-cloud">Dagster Cloud</a></th>
             <th align="center"><a href="#github-codespaces">GitHub Codespaces</a></th>
             <th align="center"><a href="#replit">Replit</a></th>
@@ -141,6 +143,7 @@ Supported ways to run this project:
         <tr>
             <td align="center">✅</td>
             <td align="center">✅</td>
+            <td align="center">🆕</td>
             <td align="center">✅</td>
             <td align="center">✅</td>
             <td align="center">✅</td>
@@ -451,6 +454,32 @@ You can run this project in Dagster Cloud. Fork the repo (or make a completely n
 
 You can then manage you metrics via PR's in your GitHub repo ([here](https://github.com/andrewm4894/anomstack/pull/40/files) is a PR to add Google Trends metrics) and run them in Dagster Cloud which will just sync with your repo.
 
+### Fly.io Deployment
+
+Deploy to [Fly.io](https://fly.io) for production-ready, globally distributed anomaly detection.
+
+**🚀 Live Demo**: https://anomstack-demo.fly.dev
+
+```bash
+# New! Automatic .env integration 🎉
+cp .example.env .env          # Edit with your secrets
+make fly-preview              # Preview what will be deployed
+make fly-deploy               # Deploy with your environment variables
+
+# Or using scripts directly
+./scripts/deployment/preview_fly_secrets.sh
+./scripts/deployment/deploy_fly.sh
+```
+
+**✨ Features:**
+- 🔒 **Secure secret management** via Fly secrets
+- 🧠 **Smart filtering** (skips local-only variables)  
+- 🔐 **Configurable admin credentials**
+- 📊 **Public dashboard** + protected admin interface
+- 🌐 **Global edge deployment** with auto-scaling
+
+See [full deployment docs](docs/docs/deployment/fly.md) for detailed instructions.
+
 ### Docker
 
 To get started with Anomstack, you can run it via docker compose.

dagster_docker.yaml

@@ -37,7 +37,7 @@ run_launcher:
         - ${ANOMSTACK_HOME}/tmp:/opt/dagster/app/tmp
         - ${ANOMSTACK_HOME}/dagster_home:/opt/dagster/dagster_home
         - ${ANOMSTACK_HOME}/metrics:/opt/dagster/app/metrics
-        - anomstack_metrics_duckdb:/metrics_db/duckdb
+        - anomstack_metrics_duckdb:/data
 
 run_storage:
   module: dagster_postgres.run_storage

dagster_fly.yaml

@@ -0,0 +1,82 @@
+scheduler:
+  module: dagster.core.scheduler
+  class: DagsterDaemonScheduler
+
+run_coordinator:
+  module: dagster.core.run_coordinator
+  class: QueuedRunCoordinator
+  config:
+    max_concurrent_runs: 12  # Increased for 8GB RAM + 4 performance CPUs (prevents queue backup)
+    tag_concurrency_limits:
+      - key: "dagster/concurrency_key"
+        value: "database"
+        limit: 2
+      - key: "dagster/concurrency_key"
+        value: "ml_training"
+        limit: 1
+
+# Use DefaultRunLauncher instead of DockerRunLauncher for Fly.io
+run_launcher:
+  module: dagster.core.launcher.default_run_launcher
+  class: DefaultRunLauncher
+
+run_storage:
+  module: dagster_postgres.run_storage
+  class: PostgresRunStorage
+  config:
+    postgres_url:
+      env: DATABASE_URL
+
+run_retries:
+  enabled: true
+  max_retries: 1
+
+# Aggressive retention policies optimized for Fly.io disk usage
+retention:
+  schedule:
+    purge_after_days: 2  # Keep for 2 days
+  sensor:
+    purge_after_days:
+      skipped: 1
+      failure: 2
+      success: 1
+
+# Run monitoring for Fly.io environment
+run_monitoring:
+  enabled: true
+  start_timeout_seconds: 180   # 3 minutes to start
+  cancel_timeout_seconds: 120  # 2 minutes to cancel
+  max_runtime_seconds: 2700    # 45 minutes max runtime per run (increased with better resources)
+  poll_interval_seconds: 60    # Check every minute
+
+# Disable telemetry
+telemetry:
+  enabled: false
+
+schedules:
+  use_threads: true
+  num_workers: 4  # Conservative for Fly.io
+
+sensors:
+  use_threads: true
+  num_workers: 2  # Conservative for Fly.io
+
+schedule_storage:
+  module: dagster_postgres.schedule_storage
+  class: PostgresScheduleStorage
+  config:
+    postgres_url:
+      env: DATABASE_URL
+
+event_log_storage:
+  module: dagster_postgres.event_log
+  class: PostgresEventLogStorage
+  config:
+    postgres_url:
+      env: DATABASE_URL
+
+compute_logs:
+  module: dagster.core.storage.local_compute_log_manager
+  class: LocalComputeLogManager
+  config:
+    base_dir: "/tmp/dagster/compute_logs" 

dagster_home/workspace.yaml

@@ -1,6 +1,9 @@
 load_from:
   # Each entry here corresponds to a service in the docker-compose file that exposes user code.
+  # For Fly.io deployment, DAGSTER_CODE_SERVER_HOST=localhost
+  # For Docker Compose, DAGSTER_CODE_SERVER_HOST=anomstack_code (default)
   - grpc_server:
-      host: anomstack_code
+      host:
+        env: DAGSTER_CODE_SERVER_HOST
       port: 4000
       location_name: "anomstack_code"

dashboard/state.py

@@ -21,9 +21,6 @@ class AppState:
     State manager for the dashboard.
     """
 
-    def __init__(self):
-        self._connection = None
-
     def get_connection(self):
         """Get database connection with MotherDuck fallback"""
         import os
@@ -79,34 +76,84 @@ def __init__(self):
         # Lazy initialization flags
         self._specs_loaded = False
         self._metric_batches_loaded = False
+        self._specs = None
+        self._metric_batches = None
+        self._specs_enabled = None
 
         print("AppState initialized with lazy loading")
+
+    @property
+    def specs(self):
+        """Lazy-loaded specs property"""
+        if not self._specs_loaded:
+            self._ensure_specs_loaded()
+        return self._specs
+
+    @specs.setter
+    def specs(self, value):
+        """Setter for specs"""
+        self._specs = value
+        self._specs_loaded = True
+
+    @property
+    def metric_batches(self):
+        """Lazy-loaded metric batches property"""
+        if not self._metric_batches_loaded:
+            self._ensure_metric_batches_loaded()
+        return self._metric_batches
+
+    @metric_batches.setter 
+    def metric_batches(self, value):
+        """Setter for metric_batches"""
+        self._metric_batches = value
+
+    @property
+    def specs_enabled(self):
+        """Lazy-loaded specs_enabled property"""
+        if not self._metric_batches_loaded:
+            self._ensure_metric_batches_loaded()
+        return self._specs_enabled
+
+    @specs_enabled.setter
+    def specs_enabled(self, value):
+        """Setter for specs_enabled"""
+        self._specs_enabled = value
 
     def _ensure_specs_loaded(self):
         """Lazy load specs and metric batches"""
         if not self._specs_loaded:
             try:
-                self.specs = get_specs()
+                self._specs = get_specs()
                 self._specs_loaded = True
                 print("Specs loaded successfully")
             except Exception as e:
                 log.error(f"Error loading specs: {e}")
-                self.specs = {}
+                self._specs = {}
 
     def _ensure_metric_batches_loaded(self):
         """Lazy load metric batches"""
         if not self._metric_batches_loaded:
             try:
-                self.metric_batches = get_metric_batches(source="all")
-                if not self.metric_batches:
+                # Ensure specs are loaded first
+                if not self._specs_loaded:
+                    self._ensure_specs_loaded()
+                    
+                self._metric_batches = get_metric_batches(source="all")
+                if not self._metric_batches:
                     log.warning("No metric batches found.")
-                self.specs_enabled = {batch: self.specs[batch] for batch in self.metric_batches}
+                    self._metric_batches = []
+                
+                if self._specs and self._metric_batches:
+                    self._specs_enabled = {batch: self._specs[batch] for batch in self._metric_batches if batch in self._specs}
+                else:
+                    self._specs_enabled = {}
+                
                 self._metric_batches_loaded = True
                 print("Metric batches loaded successfully")
             except Exception as e:
                 log.error(f"Error loading metric batches: {e}")
-                self.metric_batches = []
-                self.specs_enabled = {}
+                self._metric_batches = []
+                self._specs_enabled = {}
 
     def clear_batch_cache(self, batch_name):
         """