Update dagster_docker.yaml to increase max concurrent runs and add tag concurrency limits. Refactor environment variables for DockerRunLauncher to streamline configuration and improve clarity.

Author: andrewm4894

dagster_docker.yaml

@@ -6,55 +6,35 @@ run_coordinator:
   module: dagster.core.run_coordinator
   class: QueuedRunCoordinator
   config:
-    max_concurrent_runs: 10
+    max_concurrent_runs: 25
+    tag_concurrency_limits:
+      - key: "dagster/concurrency_key"
+        value: "database"
+        limit: 3
+      - key: "dagster/concurrency_key"
+        value: "ml_training"
+        limit: 2
 
 run_launcher:
-  module: dagster_docker.run_launcher
+  module: dagster_docker
   class: DockerRunLauncher
   config:
     env_vars:
       - DAGSTER_POSTGRES_USER
       - DAGSTER_POSTGRES_PASSWORD
       - DAGSTER_POSTGRES_DB
-      - ANOMSTACK_GCP_PROJECT_ID
       - ANOMSTACK_DUCKDB_PATH
       - ANOMSTACK_TABLE_KEY
       - ANOMSTACK_MODEL_PATH
-      - ANOMSTACK_SLACK_BOT_TOKEN
-      - ANOMSTACK_SLACK_CHANNEL
       - ANOMSTACK_IGNORE_EXAMPLES
-      - ANOMSTACK_ALERT_EMAIL_FROM
-      - ANOMSTACK_ALERT_EMAIL_TO
-      - ANOMSTACK_ALERT_EMAIL_SMTP_HOST
-      - ANOMSTACK_ALERT_EMAIL_SMTP_PORT
-      - ANOMSTACK_ALERT_EMAIL_PASSWORD
-      - ANOMSTACK_GOOGLE_APPLICATION_CREDENTIALS
-      - ANOMSTACK_GOOGLE_APPLICATION_CREDENTIALS_JSON
-      - ANOMSTACK_OPENAI_KEY
-      - OPENAI_API_KEY
-      - ANOMSTACK_ANTHROPIC_KEY
-      - ANOMSTACK_MOTHERDUCK_TOKEN
-      - ANOMSTACK_TURSO_DATABASE_URL
-      - ANOMSTACK_TURSO_AUTH_TOKEN
-      - ANOMSTACK_CLICKHOUSE_HOST
-      - ANOMSTACK_CLICKHOUSE_PORT
-      - ANOMSTACK_CLICKHOUSE_USER
-      - ANOMSTACK_CLICKHOUSE_PASSWORD
-      - ANOMSTACK_CLICKHOUSE_DATABASE
-      - ANOMSTACK_AWS_ACCESS_KEY_ID
-      - ANOMSTACK_AWS_SECRET_ACCESS_KEY
-      - ANOMSTACK_SNOWFLAKE_ACCOUNT
-      - ANOMSTACK_SNOWFLAKE_USER
-      - ANOMSTACK_SNOWFLAKE_PASSWORD
-      - ANOMSTACK_SNOWFLAKE_WAREHOUSE
       - DAGSTER_HOME
     network: anomstack_network
     container_kwargs:
       volumes: # Make docker client accessible to any launched containers as well
         - /var/run/docker.sock:/var/run/docker.sock
         - /tmp/io_manager_storage:/tmp/io_manager_storage
-        - ./tmp:/opt/dagster/app/tmp
-        - ./dagster_home:/opt/dagster/dagster_home
+        - /Users/andrewmaguire/Documents/GitHub/anomstack/tmp:/opt/dagster/app/tmp
+        - /Users/andrewmaguire/Documents/GitHub/anomstack/dagster_home:/opt/dagster/dagster_home
         - anomstack_metrics_duckdb:/metrics_db/duckdb
 
 run_storage: