diff --git a/.example.env b/.example.env index 72a2d4ce..3519196c 100644 --- a/.example.env +++ b/.example.env @@ -44,6 +44,10 @@ ANOMSTACK_MODEL_PATH=local://./tmp/models # for s3 bucket # ANOMSTACK_MODEL_PATH=s3://your-bucket/models +# home directory for anomstack (defaults to current working directory) +# used for docker volume mounts in dagster_docker.yaml +ANOMSTACK_HOME=. + # slack bot token for alerting # ANOMSTACK_SLACK_BOT_TOKEN= # slack channel for alerting @@ -66,13 +70,14 @@ ANOMSTACK_ALERT_EMAIL_PASSWORD= # ANOMSTACK_FAILURE_EMAIL_SMTP_PORT=587 # ANOMSTACK_FAILURE_EMAIL_PASSWORD= -# dagster related env vars -ANOMSTACK_DAGSTER_LOCAL_ARTIFACT_STORAGE_DIR=tmp -ANOMSTACK_DAGSTER_OVERALL_CONCURRENCY_LIMIT=10 +# dagster related env vars - LIGHTWEIGHT DEFAULTS +# Use smaller, more manageable directories to prevent disk space issues +ANOMSTACK_DAGSTER_LOCAL_ARTIFACT_STORAGE_DIR=tmp_light/artifacts +ANOMSTACK_DAGSTER_OVERALL_CONCURRENCY_LIMIT=5 # Reduced from 10 ANOMSTACK_DAGSTER_DEQUEUE_USE_THREADS=false -ANOMSTACK_DAGSTER_DEQUEUE_NUM_WORKERS=4 -ANOMSTACK_DAGSTER_LOCAL_COMPUTE_LOG_MANAGER_DIRECTORY=tmp -ANOMSTACK_DAGSTER_SQLITE_STORAGE_BASE_DIR=tmp +ANOMSTACK_DAGSTER_DEQUEUE_NUM_WORKERS=2 # Reduced from 4 +ANOMSTACK_DAGSTER_LOCAL_COMPUTE_LOG_MANAGER_DIRECTORY=tmp_light/compute_logs +ANOMSTACK_DAGSTER_SQLITE_STORAGE_BASE_DIR=tmp_light/storage # OpenAI env vars for LLM based alerts ANOMSTACK_OPENAI_KEY= @@ -131,4 +136,20 @@ POSTHOG_API_KEY= # example to enable some jobs via env vars # ANOMSTACK__PYTHON_INGEST_SIMPLE__INGEST_DEFAULT_SCHEDULE_STATUS=RUNNING -# ANOMSTACK__PYTHON_INGEST_SIMPLE__TRAIN_DEFAULT_SCHEDULE_STATUS=RUNNING \ No newline at end of file +# ANOMSTACK__PYTHON_INGEST_SIMPLE__TRAIN_DEFAULT_SCHEDULE_STATUS=RUNNING +# ANOMSTACK__PYTHON_INGEST_SIMPLE__SCORE_DEFAULT_SCHEDULE_STATUS=RUNNING +# ANOMSTACK__PYTHON_INGEST_SIMPLE__ALERT_DEFAULT_SCHEDULE_STATUS=RUNNING +# ANOMSTACK__PYTHON_INGEST_SIMPLE__PLOT_DEFAULT_SCHEDULE_STATUS=RUNNING +# ANOMSTACK__PYTHON_INGEST_SIMPLE__CHANGE_DEFAULT_SCHEDULE_STATUS=RUNNING +# ANOMSTACK__PYTHON_INGEST_SIMPLE__LLMALERT_DEFAULT_SCHEDULE_STATUS=RUNNING +# ANOMSTACK__PYTHON_INGEST_SIMPLE__DELETE_DEFAULT_SCHEDULE_STATUS=RUNNING + +# automatic configuration reloading +# Enable scheduled config reload job (runs every N minutes) +ANOMSTACK_AUTO_CONFIG_RELOAD=false +ANOMSTACK_CONFIG_RELOAD_CRON="*/5 * * * *" +ANOMSTACK_CONFIG_RELOAD_STATUS=STOPPED + +# Enable smart config file watcher sensor (reloads only when files change) +ANOMSTACK_CONFIG_WATCHER=true +ANOMSTACK_CONFIG_WATCHER_INTERVAL=30 \ No newline at end of file diff --git a/Makefile b/Makefile index 07607f35..d8c5b215 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ dev: .PHONY: docker docker-dev docker-smart docker-build docker-dev-build docker-tag docker-push docker-build-push .PHONY: docker-pull docker-clean docker-logs docker-logs-code docker-logs-dagit docker-logs-daemon docker-logs-dashboard -.PHONY: docker-shell-code docker-shell-dagit docker-shell-dashboard docker-restart-dashboard docker-restart-code +.PHONY: docker-shell-code docker-shell-dagit docker-shell-dashboard docker-restart-dashboard docker-restart-code docker-restart reload-config enable-auto-reload enable-config-watcher .PHONY: docker-stop docker-down docker-rm docker-prune # start docker containers (now uses pre-built images) @@ -124,6 +124,28 @@ docker-restart-dashboard: docker-restart-code: docker compose restart anomstack_code +# restart all containers (useful for .env changes) +docker-restart: + docker compose restart + +# reload configuration without restarting containers (hot reload) +reload-config: + @echo "πŸ”„ Reloading Anomstack configuration..." + python3 scripts/reload_config.py + +# enable automatic config reloading via Dagster scheduled job +enable-auto-reload: + @echo "πŸ€– Enabling automatic configuration reloading..." + @echo "ANOMSTACK_AUTO_CONFIG_RELOAD=true" >> .env + @echo "ANOMSTACK_CONFIG_RELOAD_STATUS=RUNNING" >> .env + @echo "βœ… Auto reload enabled! Restart containers: make docker-restart" + +# enable smart config file watcher sensor +enable-config-watcher: + @echo "πŸ‘οΈ Enabling smart configuration file watcher..." + @echo "ANOMSTACK_CONFIG_WATCHER=true" >> .env + @echo "βœ… Config watcher enabled! Restart containers: make docker-restart" + # alias for docker-stop docker-down: docker compose down @@ -137,6 +159,58 @@ docker-prune: docker compose down -v --remove-orphans docker system prune -a -f +# ============================================================================= +# RESET OPERATIONS +# ============================================================================= + +.PHONY: reset-gentle reset-medium reset-nuclear reset-full-nuclear reset-interactive + +# interactive reset with guided options +reset-interactive: + @scripts/utils/reset_docker.sh + +# gentle reset: rebuild containers with fresh images (safest) +reset-gentle: + @scripts/utils/reset_docker.sh gentle + +# medium reset: remove containers, keep data volumes +reset-medium: + @scripts/utils/reset_docker.sh medium + +# nuclear reset: remove everything including local data +reset-nuclear: + @scripts/utils/reset_docker.sh nuclear + +# full nuclear reset: nuclear + full docker system cleanup (maximum cleanup) +reset-full-nuclear: + @scripts/utils/reset_docker.sh full-nuclear + +# ============================================================================= +# DAGSTER STORAGE CLEANUP +# ============================================================================= + +.PHONY: dagster-cleanup-status dagster-cleanup-minimal dagster-cleanup-standard dagster-cleanup-aggressive + +# show current dagster storage usage and configuration status +dagster-cleanup-status: + @scripts/utils/cleanup_dagster_storage.sh status + +# minimal dagster cleanup - remove old logs only (safe) +dagster-cleanup-minimal: + @scripts/utils/cleanup_dagster_storage.sh minimal + +# standard dagster cleanup - remove runs older than 30 days +dagster-cleanup-standard: + @scripts/utils/cleanup_dagster_storage.sh standard + +# aggressive dagster cleanup - remove runs older than 7 days +dagster-cleanup-aggressive: + @scripts/utils/cleanup_dagster_storage.sh aggressive + +# interactive dagster cleanup menu +dagster-cleanup-menu: + @scripts/utils/cleanup_dagster_storage.sh menu + # ============================================================================= # DASHBOARD OPERATIONS # ============================================================================= @@ -185,11 +259,31 @@ coverage: # DOCUMENTATION # ============================================================================= -.PHONY: docs +.PHONY: docs docs-start docs-build docs-serve docs-clear docs-install -# start documentation development server +# start documentation development server (alias for docs-start) docs: - cd docs && yarn start + @$(MAKE) docs-start + +# install documentation dependencies +docs-install: + cd docs && npm install + +# start development server with live reload +docs-start: + cd docs && npm start + +# build static documentation site +docs-build: + cd docs && npm run build + +# serve built documentation locally +docs-serve: + cd docs && npm run serve + +# clear documentation build cache +docs-clear: + cd docs && npm run clear # ============================================================================= # DEPENDENCIES diff --git a/Makefile.md b/Makefile.md new file mode 100644 index 00000000..28f9dd34 --- /dev/null +++ b/Makefile.md @@ -0,0 +1,649 @@ +# Makefile Documentation + +This document provides comprehensive documentation for the Anomstack Makefile, which contains targets for development, Docker operations, testing, and maintenance. + +## Table of Contents + +- [Overview](#overview) +- [Local Development](#local-development) +- [Docker Operations](#docker-operations) +- [Reset Operations](#reset-operations) +- [Dagster Storage Cleanup](#dagster-storage-cleanup) +- [Dashboard Operations](#dashboard-operations) +- [Testing & Quality](#testing--quality) +- [Documentation](#documentation) +- [Dependencies](#dependencies) +- [Utilities](#utilities) +- [Quick Reference](#quick-reference) + +## Overview + +The Makefile is organized into logical sections for different aspects of Anomstack development and deployment. Each section contains related targets that perform specific tasks. + +**Prerequisites:** +- Docker and Docker Compose installed +- Python 3.x with required dependencies +- Node.js and Yarn (for documentation) + +## Local Development + +Targets for running Anomstack locally without Docker. + +### `make local` +**Start Dagster locally (simple mode)** +- Sets `DAGSTER_HOME` to local `dagster_home/` directory +- Runs Dagster in development mode with hot reloading +- Uses the main Anomstack module (`anomstack/main.py`) + +```bash +make local +``` + +*Use when: You want to develop and test locally with immediate feedback* + +### `make locald` +**Start Dagster as background daemon** +- Same as `local` but runs in background +- Output redirected to `/dev/null` +- Useful for long-running local development + +```bash +make locald +``` + +*Use when: You want Dagster running in background while you work on other things* + +### `make kill-locald` +**Kill running Dagster processes** +- Finds and kills all local Dagster processes +- Uses `kill -9` for forceful termination + +```bash +make kill-locald +``` + +*Use when: You need to stop background Dagster processes* + +### `make ps-locald` +**List running Dagster processes** +- Shows all currently running Dagster processes +- Helpful for debugging multiple instances + +```bash +make ps-locald +``` + +### `make dev` +**Setup development environment** +- Installs pre-commit hooks +- Prepares development dependencies + +```bash +make dev +``` + +*Use when: Setting up a new development environment* + +## Docker Operations + +Comprehensive Docker management for containerized deployment. + +### Starting Containers + +#### `make docker` +**Start containers with pre-built images (Production)** +- Uses images from Docker Hub (`andrewm4894/*`) +- Fast startup, no building required +- Production-ready configuration + +```bash +make docker +``` + +#### `make docker-dev` +**Start containers with local development images** +- Uses locally built images for development +- Includes development overrides +- Live code mounting for development + +```bash +make docker-dev +``` + +#### `make docker-smart` +**Intelligent container startup** +- Tries to pull pre-built images first +- Falls back to local building if pull fails +- Best of both worlds approach + +```bash +make docker-smart +``` + +### Building Images + +#### `make docker-build` +**Build all images locally** +- Builds: `anomstack_code_image`, `anomstack_dagster_image`, `anomstack_dashboard_image` +- Uses individual Dockerfiles for each service +- Local tags for development use + +```bash +make docker-build +``` + +#### `make docker-dev-build` +**Build development images with no cache** +- Uses docker-compose for coordinated building +- `--no-cache` ensures completely fresh builds +- Includes development configurations + +```bash +make docker-dev-build +``` + +### Docker Hub Operations + +#### `make docker-tag` +**Tag images for Docker Hub** +- Tags local images with Docker Hub repository names +- Prepares images for pushing to registry + +```bash +make docker-tag +``` + +#### `make docker-push` +**Push images to Docker Hub** +- Uploads tagged images to Docker Hub +- Requires Docker Hub authentication + +```bash +make docker-push +``` + +#### `make docker-build-push` +**Complete build and push workflow** +- Combines: `docker-build` β†’ `docker-tag` β†’ `docker-push` +- One-command CI/CD pipeline + +```bash +make docker-build-push +``` + +#### `make docker-pull` +**Pull latest images from Docker Hub** +- Downloads latest versions of all images +- Updates local image cache + +```bash +make docker-pull +``` + +### Container Management + +#### `make docker-down` / `make docker-stop` +**Stop containers** +- Stops and removes containers +- Preserves volumes and networks +- Equivalent commands + +```bash +make docker-down +# or +make docker-stop +``` + +#### `make docker-rm` +**Remove containers and networks** +- More thorough cleanup than `docker-down` +- Removes orphaned containers +- Preserves volumes + +```bash +make docker-rm +``` + +#### `make docker-prune` +**⚠️ Nuclear cleanup - removes everything** +- Removes containers, networks, **and volumes** +- Runs Docker system prune +- **Deletes all data** - use with caution! + +```bash +make docker-prune +``` + +### Debugging & Monitoring + +#### `make docker-logs` +**View logs from all containers** +- Shows combined logs with following (`-f`) +- Color-coded by service + +```bash +make docker-logs +``` + +#### Service-Specific Logs +**View logs from individual services** +```bash +make docker-logs-code # Anomstack code service +make docker-logs-daemon # Dagster daemon +make docker-logs-dashboard # Dashboard service +``` + +#### Shell Access +**Get shell access to running containers** +```bash +make docker-shell-code # Shell into code container +make docker-shell-dagit # Shell into webserver container +make docker-shell-dashboard # Shell into dashboard container +``` + +#### Service Restart +**Restart individual services** +```bash +make docker-restart-code # Restart code service +make docker-restart-dashboard # Restart dashboard service +``` + +#### `make docker-clean` +**Clean unused Docker resources** +- Runs `docker system prune -f` +- Runs `docker volume prune -f` +- Frees disk space without removing data + +```bash +make docker-clean +``` + +## Reset Operations + +**πŸ†• NEW SECTION** - Comprehensive reset utilities for different cleanup levels. + +All reset operations use the `scripts/utils/reset_docker.sh` script with safety checks and confirmations. + +### `make reset-interactive` +**Interactive guided reset** +- Presents menu with clear options +- Shows current data usage +- Guides you through reset process +- **Recommended for most users** + +```bash +make reset-interactive +``` + +### `make reset-gentle` +**Gentle reset (safest)** +- βœ… Preserves all data and volumes +- πŸ”„ Rebuilds containers with fresh images +- 🎯 Use when: You want fresh containers but keep data + +```bash +make reset-gentle +``` + +### `make reset-medium` +**Medium reset (balanced)** +- βœ… Preserves Docker volumes and data files +- πŸ—‘οΈ Removes containers and networks +- 🎯 Use when: Container issues but want to keep data + +```bash +make reset-medium +``` + +### `make reset-nuclear` +**Nuclear reset (destructive)** +- ⚠️ **Removes ALL data** including volumes and local files +- πŸ—‘οΈ Clears tmp/, storage/, history/, logs/ +- 🎯 Use when: You want completely fresh start (like your 63GB cleanup case) + +```bash +make reset-nuclear +``` + +### `make reset-full-nuclear` +**Full nuclear reset (maximum cleanup)** +- ⚠️ Everything from nuclear reset +- πŸ—‘οΈ **Plus** complete Docker system cleanup +- πŸ’Ύ Frees maximum possible disk space +- 🎯 Use when: You want absolute maximum cleanup + +```bash +make reset-full-nuclear +``` + +## Dagster Storage Cleanup + +**Purpose**: Manage Dagster storage accumulation to prevent disk space issues (like the 63GB+ buildup problem). + +### `make dagster-cleanup-status` +**Show storage analysis and configuration status** +- πŸ“Š Analyzes current tmp/ directory usage +- πŸ”’ Counts Dagster run directories +- πŸ’Ύ Reports storage sizes +- βœ… Checks if retention policies are configured +- πŸ”§ Verifies run monitoring settings + +```bash +make dagster-cleanup-status +``` + +### `make dagster-cleanup-minimal` +**Safe cleanup - Remove old logs only** +- πŸ—‘οΈ Removes compute logs older than 7 days +- βœ… Preserves all run metadata and artifacts +- πŸ›‘οΈ Safe for production use +- πŸ“… **Recommended frequency**: Weekly + +```bash +make dagster-cleanup-minimal +``` + +### `make dagster-cleanup-standard` +**Standard cleanup - Remove old runs** +- πŸ—‘οΈ Removes run directories older than 30 days +- πŸ“‹ Cleans associated compute logs and artifacts +- πŸ› Keeps recent runs for debugging +- πŸ“… **Recommended frequency**: Monthly + +```bash +make dagster-cleanup-standard +``` + +### `make dagster-cleanup-aggressive` +**Aggressive cleanup - Keep only recent data** +- πŸ—‘οΈ Removes run directories older than 7 days +- πŸ”₯ Aggressive log cleanup (3+ days old) +- ⏱️ Only keeps very recent data +- 🎯 **Use when**: Disk space is becoming an issue + +```bash +make dagster-cleanup-aggressive +``` + +### `make dagster-cleanup-menu` +**Interactive cleanup menu** +- πŸŽ›οΈ Interactive menu with all cleanup options +- πŸ“Š Shows real-time storage analysis +- ☒️ Includes nuclear and CLI-based options +- 🎯 **Best for**: One-time cleanup or exploration + +```bash +make dagster-cleanup-menu +``` + +**πŸ’‘ Pro Tip**: Run `make dagster-cleanup-status` first to understand your current storage usage before choosing a cleanup level. + +## Dashboard Operations + +Targets for running the Anomstack dashboard independently. + +### `make dashboard` +**Start dashboard locally with Python** +- Runs `python dashboard/app.py` +- Development server on default port +- Direct Python execution + +```bash +make dashboard +``` + +### `make dashboard-uvicorn` +**Start dashboard with Uvicorn ASGI server** +- Production-ready ASGI server +- Runs on `0.0.0.0:5003` with auto-reload +- Better performance than direct Python + +```bash +make dashboard-uvicorn +``` + +### Background Dashboard + +#### `make dashboardd` +**Start dashboard as background daemon (Python)** +```bash +make dashboardd +``` + +#### `make dashboardd-uvicorn` +**Start dashboard as background daemon (Uvicorn)** +```bash +make dashboardd-uvicorn +``` + +#### `make kill-dashboardd` +**Kill running dashboard processes** +```bash +make kill-dashboardd +``` + +## Testing & Quality + +Targets for code quality and testing. + +### `make pre-commit` +**Run pre-commit hooks on all files** +- Runs linting, formatting, and checks +- Uses `.pre-commit-config.yaml` configuration +- Ensures code quality standards + +```bash +make pre-commit +``` + +### `make tests` +**Run test suite** +- Executes pytest with verbose output +- Runs all tests in `tests/` directory + +```bash +make tests +``` + +### `make coverage` +**Run tests with coverage report** +- Generates code coverage analysis +- Shows which code is tested +- Displays missing coverage areas + +```bash +make coverage +``` + +## Documentation + +**Purpose**: Manage the Docusaurus documentation site for Anomstack. + +### `make docs-install` +**Install documentation dependencies** +- Runs `npm install` in the docs directory +- Required before first use or after dependency changes +- Sets up Docusaurus and all required packages + +```bash +make docs-install +``` + +### `make docs` / `make docs-start` +**Start development server with live reload** +- Starts Docusaurus development server on `http://localhost:3000` +- Enables live editing with hot reload +- Perfect for writing and previewing documentation changes + +```bash +make docs +# or +make docs-start +``` + +### `make docs-build` +**Build static documentation site** +- Creates production-ready static files +- Outputs to `docs/build/` directory +- Required before serving or deploying + +```bash +make docs-build +``` + +### `make docs-serve` +**Serve built documentation locally** +- Serves the built static site locally +- Good for testing the production build +- Requires `make docs-build` to be run first + +```bash +make docs-serve +``` + +### `make docs-clear` +**Clear documentation build cache** +- Clears Docusaurus build cache +- Useful when experiencing build issues +- Forces a fresh build on next command + +```bash +make docs-clear +``` + +**πŸ’‘ Typical workflow:** +1. `make docs-install` (first time only) +2. `make docs` (for development with live reload) +3. `make docs-build` (when ready to test production build) +4. `make docs-serve` (to test the built site) + +## Dependencies + +### `make requirements` +**Compile requirements file** +- Uses `pip-compile` to generate `requirements.txt` +- Based on `requirements.compile` source file +- Pins dependency versions for reproducibility + +```bash +make requirements +``` + +### `make requirements-install` +**Install Python dependencies** +- Installs packages from `requirements.txt` +- Ensures consistent environment setup + +```bash +make requirements-install +``` + +## Utilities + +Miscellaneous utility targets for maintenance and examples. + +### `make posthog-example` +**Run PostHog integration example** +- Tests PostHog credentials and connectivity +- Runs example ingest function +- Useful for validating PostHog setup + +```bash +make posthog-example +``` + +### `make kill-long-runs` +**Kill long-running Dagster tasks** +- Identifies and kills tasks exceeding timeout +- Prevents runaway processes +- Uses `scripts/kill_long_running_tasks.py` + +```bash +make kill-long-runs +``` + +### Legacy Targets + +#### `make docker-dev-env` +**Legacy: Run Docker in development mode** +```bash +make docker-dev-env # Use `make docker-dev` instead +``` + +## Quick Reference + +### Most Common Workflows + +**πŸš€ Quick Start (Development):** +```bash +make docker-dev +``` + +**πŸš€ Quick Start (Production):** +```bash +make docker +``` + +**🧹 Clean Restart:** +```bash +make reset-interactive # Choose your reset level +``` + +**πŸ” Debug Issues:** +```bash +make docker-logs # View all logs +make docker-shell-code # Get shell access +make docker-restart-dashboard # Restart specific service +``` + +**πŸ“Š Development Workflow:** +```bash +make dev # Setup environment +make pre-commit # Check code quality +make tests # Run tests +make docker-dev # Start development +``` + +**🚒 CI/CD Workflow:** +```bash +make docker-build-push # Build and push to registry +make docker-pull # Pull latest images +make docker # Start with latest images +``` + +### Emergency Commands + +**πŸ†˜ Something's broken:** +```bash +make reset-interactive # Guided reset options +``` + +**πŸ’Ύ Free disk space:** +```bash +make reset-full-nuclear # Maximum cleanup +``` + +**πŸ”₯ Kill everything:** +```bash +make kill-locald # Kill local processes +make kill-dashboardd # Kill dashboard processes +make kill-long-runs # Kill long-running tasks +``` + +--- + +## Tips & Best Practices + +1. **Start with `make reset-interactive`** - It provides guidance and shows data usage +2. **Use `make docker-smart`** - Intelligently handles image availability +3. **Check logs first** - `make docker-logs` often reveals the issue +4. **Use gentle resets** - Start with `reset-gentle` before more destructive options +5. **Regular cleanup** - `make docker-clean` frees space without losing data +6. **Development cycle** - Use `make docker-dev` for active development + +## Getting Help + +- Run `make reset-interactive` for guided Docker operations +- Check `scripts/utils/README.md` for detailed reset documentation +- Use `make docker-logs` to diagnose container issues +- Most targets have descriptive comments in the Makefile itself \ No newline at end of file diff --git a/README.md b/README.md index 5f085536..e6f60829 100644 --- a/README.md +++ b/README.md @@ -478,6 +478,15 @@ docker compose build docker compose up -d ``` +**πŸ’‘ Pro Tip:** Anomstack includes a comprehensive Makefile with many useful targets for development and deployment. For detailed documentation of all available commands, see [`Makefile.md`](./Makefile.md). Quick examples: + +```bash +make docker-dev # Start with local development images +make reset-interactive # Interactive reset with cleanup options +make docker-logs # View all container logs +make tests # Run test suite +``` + ### Replit You can run Anomstack directly on Replit: diff --git a/anomstack/config.py b/anomstack/config.py index a37a368b..193a8eb1 100644 --- a/anomstack/config.py +++ b/anomstack/config.py @@ -49,12 +49,15 @@ def process_yaml_file(yaml_file: str): return # Apply global environment variable overrides + # Only override if the parameter is NOT specified in either defaults or specific YAML file for env_var in env_vars: if env_var in os.environ: param_key = env_var.replace("ANOMSTACK_", "").lower() - yaml_value = metric_specs.get(param_key) or defaults.get(param_key) - merged_specs[param_key] = os.getenv(env_var) - logger.info(f"ENV OVERRIDE: {env_var} replaces {param_key} (was: {yaml_value}, now: {merged_specs[param_key]})") + # Only override if the parameter is NOT specified in either defaults or specific YAML file + if param_key not in metric_specs and param_key not in defaults: + yaml_value = merged_specs.get(param_key) + merged_specs[param_key] = os.getenv(env_var) + logger.info(f"ENV OVERRIDE: {env_var} replaces {param_key} (was: {yaml_value}, now: {merged_specs[param_key]})") # Apply metric batch-specific environment variable overrides # Pattern: ANOMSTACK____ @@ -86,3 +89,151 @@ def process_yaml_file(yaml_file: str): process_yaml_file(yaml_path) return specs + + +# Hot configuration reload functionality +def reload_code_location(location_name="anomstack_code"): + """Reload a specific code location in Dagster.""" + import requests + + logger = get_dagster_logger() + host = os.getenv('DAGSTER_HOST', 'localhost') + port = os.getenv('DAGSTER_PORT', '3000') + url = f"http://{host}:{port}/graphql" + + # GraphQL mutation to reload code location + mutation = """ + mutation ReloadRepositoryLocation($locationName: String!) { + reloadRepositoryLocation(repositoryLocationName: $locationName) { + __typename + ... on WorkspaceLocationEntry { + name + locationOrLoadError { + __typename + ... on RepositoryLocation { + name + repositories { + name + } + } + ... on PythonError { + message + stack + } + } + } + ... on ReloadNotSupported { + message + } + ... on RepositoryLocationNotFound { + message + } + } + } + """ + + variables = {"locationName": location_name} + + try: + logger.info(f"πŸ”„ Attempting to reload code location '{location_name}'...") + response = requests.post( + url, + json={"query": mutation, "variables": variables}, + timeout=30 + ) + + if response.status_code != 200: + logger.error(f"❌ HTTP Error {response.status_code}: {response.text}") + return False + + data = response.json() + + if "errors" in data: + logger.error(f"❌ GraphQL Errors: {data['errors']}") + return False + + result = data["data"]["reloadRepositoryLocation"] + + if result["__typename"] == "WorkspaceLocationEntry": + logger.info(f"βœ… Successfully reloaded code location '{location_name}'") + + # Show loaded repositories + location_data = result["locationOrLoadError"] + if location_data["__typename"] == "RepositoryLocation": + repos = location_data["repositories"] + logger.info(f"πŸ“¦ Loaded {len(repos)} repositories:") + for repo in repos: + logger.info(f" - {repo['name']}") + else: + logger.warning(f"⚠️ Location loaded but with errors: {location_data}") + + return True + + elif result["__typename"] == "ReloadNotSupported": + logger.error(f"❌ Reload not supported: {result['message']}") + return False + + elif result["__typename"] == "RepositoryLocationNotFound": + logger.error(f"❌ Location not found: {result['message']}") + return False + + else: + logger.error(f"❌ Unexpected result type: {result}") + return False + + except Exception as e: + logger.error(f"❌ Unexpected error during configuration reload: {str(e)}") + return False + + +def execute_config_reload(): + """ + Execute configuration reload with validation. + + Returns: + bool: True if reload was successful, False otherwise + """ + import requests + + logger = get_dagster_logger() + + try: + # Check if Dagster is accessible + host = os.getenv('DAGSTER_HOST', 'localhost') + port = os.getenv('DAGSTER_PORT', '3000') + url = f"http://{host}:{port}/server_info" + + response = requests.get(url, timeout=5) + if response.status_code != 200: + logger.error(f"❌ Dagster webserver returned {response.status_code}") + return False + + logger.info("βœ… Dagster webserver is accessible") + + # Check if metrics directory is accessible + metrics_dir = Path("./metrics") + if not metrics_dir.exists(): + logger.error(f"❌ Metrics directory not found at {metrics_dir.absolute()}") + return False + + defaults_file = metrics_dir / "defaults" / "defaults.yaml" + if not defaults_file.exists(): + logger.error(f"❌ Defaults file not found at {defaults_file.absolute()}") + return False + + logger.info(f"βœ… Configuration files accessible at {metrics_dir.absolute()}") + + # Execute reload + logger.info("πŸ”„ Starting configuration reload...") + success = reload_code_location("anomstack_code") + + if success: + logger.info("πŸŽ‰ Configuration reload complete!") + return True + else: + logger.error("πŸ’₯ Configuration reload failed!") + return False + + except Exception as e: + logger.error(f"❌ Configuration reload error: {str(e)}") + return False diff --git a/anomstack/jobs/reload.py b/anomstack/jobs/reload.py new file mode 100644 index 00000000..0d748e0d --- /dev/null +++ b/anomstack/jobs/reload.py @@ -0,0 +1,128 @@ +""" +Generate configuration reload jobs and schedules. +""" + +import os +from dagster import ( + MAX_RUNTIME_SECONDS_TAG, + DefaultScheduleStatus, + JobDefinition, + ScheduleDefinition, + get_dagster_logger, + job, + op, +) + +from anomstack.config import execute_config_reload + +ANOMSTACK_MAX_RUNTIME_SECONDS_TAG = os.getenv("ANOMSTACK_MAX_RUNTIME_SECONDS_TAG", 3600) + + +def build_config_reload_job() -> JobDefinition: + """ + Build job definition for configuration reload job. + + This job automatically reloads Dagster configurations when run, + allowing for hot updates without container restarts. + + Returns: + JobDefinition: A job definition for the config reload job. + """ + + @job( + name="config_reload", + tags={MAX_RUNTIME_SECONDS_TAG: ANOMSTACK_MAX_RUNTIME_SECONDS_TAG}, + ) + def _job(): + """ + Reload Dagster configuration from updated YAML files. + """ + + @op(name="reload_configuration") + def reload_configuration() -> str: + """ + Execute the configuration reload. + + Returns: + str: Result of the reload operation. + """ + import os + logger = get_dagster_logger() + + logger.info("πŸ”„ Starting configuration reload via Dagster job...") + + # Ensure environment variables are set for Dagster execution context + # These might not be available when running as a scheduled job + if not os.getenv("DAGSTER_HOST"): + os.environ["DAGSTER_HOST"] = "anomstack_webserver" + logger.info("πŸ”§ Set DAGSTER_HOST=anomstack_webserver for job execution") + + if not os.getenv("DAGSTER_PORT"): + os.environ["DAGSTER_PORT"] = "3000" + logger.info("πŸ”§ Set DAGSTER_PORT=3000 for job execution") + + logger.info(f"🌐 Using Dagster host: {os.getenv('DAGSTER_HOST')}:{os.getenv('DAGSTER_PORT')}") + + # Execute the reload using the internal function + success = execute_config_reload() + + if success: + logger.info("βœ… Configuration reload completed successfully") + return "Configuration reload completed successfully" + else: + error_msg = "Configuration reload failed" + logger.error(f"❌ {error_msg}") + raise Exception(error_msg) + + reload_configuration() + + return _job + + +def should_enable_config_reload_job() -> bool: + """ + Determine if the config reload job should be enabled. + + Returns: + bool: True if the job should be enabled. + """ + # Check environment variable to enable/disable the feature + enabled = os.getenv("ANOMSTACK_AUTO_CONFIG_RELOAD", "false").lower() == "true" + + # Also check if we're in development mode + dev_mode = os.getenv("DAGSTER_DEV", "false").lower() == "true" + + return enabled or dev_mode + + +# Build config reload job and schedule +reload_jobs = [] +reload_schedules = [] + +if should_enable_config_reload_job(): + logger = get_dagster_logger() + logger.info("πŸ”§ Config reload job is enabled") + + config_reload_job = build_config_reload_job() + reload_jobs.append(config_reload_job) + + # Get reload schedule from environment or default to every 5 minutes + reload_cron_schedule = os.getenv("ANOMSTACK_CONFIG_RELOAD_CRON", "*/5 * * * *") + reload_default_status = os.getenv("ANOMSTACK_CONFIG_RELOAD_STATUS", "STOPPED") + + if reload_default_status == "RUNNING": + reload_default_schedule_status = DefaultScheduleStatus.RUNNING + else: + reload_default_schedule_status = DefaultScheduleStatus.STOPPED + + config_reload_schedule = ScheduleDefinition( + job=config_reload_job, + cron_schedule=reload_cron_schedule, + default_status=reload_default_schedule_status, + ) + reload_schedules.append(config_reload_schedule) + + logger.info(f"πŸ“… Config reload scheduled: {reload_cron_schedule} (status: {reload_default_status})") +else: + logger = get_dagster_logger() + logger.info("⏸️ Config reload job is disabled (set ANOMSTACK_AUTO_CONFIG_RELOAD=true to enable)") \ No newline at end of file diff --git a/anomstack/main.py b/anomstack/main.py index 741f27b8..aeea9063 100644 --- a/anomstack/main.py +++ b/anomstack/main.py @@ -10,11 +10,13 @@ from anomstack.jobs.ingest import ingest_jobs, ingest_schedules from anomstack.jobs.llmalert import llmalert_jobs, llmalert_schedules from anomstack.jobs.plot import plot_jobs, plot_schedules +from anomstack.jobs.reload import reload_jobs, reload_schedules from anomstack.jobs.score import score_jobs, score_schedules from anomstack.jobs.summary import summary_jobs, summary_schedules from anomstack.jobs.train import train_jobs, train_schedules from anomstack.sensors.failure import email_on_run_failure from anomstack.sensors.timeout import kill_long_running_runs +from anomstack.sensors.config_reload import config_file_watcher jobs = ( ingest_jobs @@ -26,8 +28,9 @@ + change_jobs + summary_jobs + delete_jobs + + reload_jobs ) -sensors = [email_on_run_failure, kill_long_running_runs] +sensors = [email_on_run_failure, kill_long_running_runs, config_file_watcher] schedules = ( ingest_schedules + train_schedules @@ -38,6 +41,7 @@ + change_schedules + summary_schedules + delete_schedules + + reload_schedules ) defs = Definitions( diff --git a/anomstack/sensors/config_reload.py b/anomstack/sensors/config_reload.py new file mode 100644 index 00000000..b4015046 --- /dev/null +++ b/anomstack/sensors/config_reload.py @@ -0,0 +1,184 @@ +""" +Sensor for automatic configuration reloading when YAML files change. +""" + +import os +import hashlib +import json +from pathlib import Path +from typing import Dict, Optional + +from dagster import ( + RunRequest, + SensorEvaluationContext, + SkipReason, + sensor, + get_dagster_logger, +) +from anomstack.config import execute_config_reload + + +def get_metrics_directory() -> Path: + """Get the metrics directory path.""" + return Path("./metrics").resolve() + + +def calculate_directory_hash(directory: Path) -> str: + """ + Calculate a hash of all YAML files in the metrics directory. + + Args: + directory: Path to the metrics directory + + Returns: + str: MD5 hash of all YAML file contents and modification times + """ + hash_md5 = hashlib.md5() + + try: + # Get all YAML files recursively, sorted for consistent hashing + yaml_files = sorted(directory.rglob("*.yaml")) + + for yaml_file in yaml_files: + try: + # Include file path in hash + hash_md5.update(str(yaml_file.relative_to(directory)).encode()) + + # Include modification time + mtime = yaml_file.stat().st_mtime + hash_md5.update(str(mtime).encode()) + + # Include file content + with open(yaml_file, 'rb') as f: + hash_md5.update(f.read()) + + except (OSError, IOError) as e: + # File might have been deleted or is unreadable, skip it + hash_md5.update(f"ERROR:{e}".encode()) + + return hash_md5.hexdigest() + + except Exception as e: + # Return a hash that includes the error for consistency + return hashlib.md5(f"ERROR:{e}".encode()).hexdigest() + + +def get_stored_hash(context: SensorEvaluationContext) -> Optional[str]: + """Get the last stored directory hash from sensor context.""" + cursor = context.cursor + if cursor: + try: + cursor_data = json.loads(cursor) + return cursor_data.get("last_hash") + except (json.JSONDecodeError, KeyError): + return None + return None + + +def store_hash(context: SensorEvaluationContext, directory_hash: str) -> str: + """Store the directory hash in sensor context.""" + return json.dumps({"last_hash": directory_hash}) + + +def execute_sensor_config_reload() -> bool: + """ + Execute the configuration reload via sensor. + + Returns: + bool: True if reload was successful, False otherwise + """ + logger = get_dagster_logger() + + try: + logger.info("πŸ”„ Executing configuration reload via sensor...") + + # Use the internal reload function + return execute_config_reload() + + except Exception as e: + logger.error(f"πŸ’₯ Unexpected error during configuration reload: {str(e)}") + return False + + +def should_enable_config_watcher() -> bool: + """ + Determine if the config watcher sensor should be enabled. + + Returns: + bool: True if the sensor should be enabled. + """ + # Check environment variable to enable/disable the feature + enabled = os.getenv("ANOMSTACK_CONFIG_WATCHER", "false").lower() == "true" + + # Also check if we're in development mode + dev_mode = os.getenv("DAGSTER_DEV", "false").lower() == "true" + + return enabled or dev_mode + + +# Get sensor configuration +SENSOR_ENABLED = should_enable_config_watcher() +SENSOR_INTERVAL = int(os.getenv("ANOMSTACK_CONFIG_WATCHER_INTERVAL", "30")) # seconds + +if SENSOR_ENABLED: + @sensor( + name="config_file_watcher", + minimum_interval_seconds=SENSOR_INTERVAL, + ) + def config_file_watcher(context: SensorEvaluationContext): + """ + Watch for changes to YAML configuration files and reload when necessary. + + This sensor monitors the metrics directory for changes to .yaml files + and triggers a configuration reload when changes are detected. + """ + logger = get_dagster_logger() + + try: + metrics_dir = get_metrics_directory() + + if not metrics_dir.exists(): + return SkipReason(f"Metrics directory not found: {metrics_dir}") + + # Calculate current hash of all YAML files + current_hash = calculate_directory_hash(metrics_dir) + last_hash = get_stored_hash(context) + + # Check if this is the first run + if last_hash is None: + logger.info("πŸ” First run of config file watcher - storing initial hash") + context.update_cursor(store_hash(context, current_hash)) + return SkipReason("First run - establishing baseline") + + # Check if files have changed + if current_hash == last_hash: + return SkipReason("No configuration file changes detected") + + logger.info("πŸ“ Configuration file changes detected!") + logger.info(f"Previous hash: {last_hash[:8]}...") + logger.info(f"Current hash: {current_hash[:8]}...") + + # Execute the reload + reload_success = execute_sensor_config_reload() + + if reload_success: + # Update cursor with new hash only if reload was successful + context.update_cursor(store_hash(context, current_hash)) + logger.info("πŸŽ‰ Configuration watcher successfully reloaded configs") + return SkipReason("Configuration reloaded successfully") + else: + logger.error("❌ Configuration reload failed - keeping old hash") + return SkipReason("Configuration reload failed") + + except Exception as e: + logger.error(f"πŸ’₯ Error in config file watcher: {str(e)}") + return SkipReason(f"Sensor error: {str(e)}") +else: + # Create a dummy sensor that's always disabled + @sensor( + name="config_file_watcher_disabled", + minimum_interval_seconds=3600, # Check once per hour + ) + def config_file_watcher(context: SensorEvaluationContext): + """Disabled config file watcher.""" + return SkipReason("Config file watcher is disabled (set ANOMSTACK_CONFIG_WATCHER=true to enable)") \ No newline at end of file diff --git a/dagster.yaml b/dagster.yaml index eed0c679..6c89991c 100644 --- a/dagster.yaml +++ b/dagster.yaml @@ -27,12 +27,12 @@ local_artifact_storage: retention: schedule: - purge_after_days: 7 + purge_after_days: 3 # Reduced from 7 to prevent disk buildup sensor: purge_after_days: - skipped: 7 - failure: 7 - success: 7 + skipped: 1 # Keep only 1 day of skipped runs + failure: 3 # Keep 3 days of failures for debugging + success: 1 # Keep only 1 day of successful runs schedules: use_threads: true diff --git a/dagster_docker.yaml b/dagster_docker.yaml index 5039e23f..69713ba8 100644 --- a/dagster_docker.yaml +++ b/dagster_docker.yaml @@ -6,14 +6,14 @@ run_coordinator: module: dagster.core.run_coordinator class: QueuedRunCoordinator config: - max_concurrent_runs: 25 + max_concurrent_runs: 10 # Reduced from 25 to prevent excessive storage tag_concurrency_limits: - key: "dagster/concurrency_key" value: "database" - limit: 3 + limit: 2 # Reduced from 3 - key: "dagster/concurrency_key" value: "ml_training" - limit: 2 + limit: 1 # Reduced from 2 run_launcher: module: dagster_docker @@ -28,13 +28,15 @@ run_launcher: - ANOMSTACK_MODEL_PATH - ANOMSTACK_IGNORE_EXAMPLES - DAGSTER_HOME + - ANOMSTACK_HOME network: anomstack_network container_kwargs: volumes: # Make docker client accessible to any launched containers as well - /var/run/docker.sock:/var/run/docker.sock - /tmp/io_manager_storage:/tmp/io_manager_storage - - /Users/andrewmaguire/Documents/GitHub/anomstack/tmp:/opt/dagster/app/tmp - - /Users/andrewmaguire/Documents/GitHub/anomstack/dagster_home:/opt/dagster/dagster_home + - ${ANOMSTACK_HOME}/tmp:/opt/dagster/app/tmp + - ${ANOMSTACK_HOME}/dagster_home:/opt/dagster/dagster_home + - ${ANOMSTACK_HOME}/metrics:/opt/dagster/app/metrics - anomstack_metrics_duckdb:/metrics_db/duckdb run_storage: @@ -55,10 +57,27 @@ run_retries: enabled: true max_retries: 1 +# Aggressive retention policies to prevent disk space issues +retention: + schedule: + purge_after_days: 3 # Keep schedule runs for 3 days + sensor: + purge_after_days: + skipped: 1 # Keep only 1 day of skipped sensor runs + failure: 3 # Keep 3 days of failed runs for debugging + success: 1 # Keep only 1 day of successful sensor runs + +# Run monitoring to detect and clean up stuck runs run_monitoring: enabled: true - start_timeout_seconds: 600 - cancel_timeout_seconds: 300 + start_timeout_seconds: 300 # 5 minutes to start + cancel_timeout_seconds: 180 # 3 minutes to cancel + max_runtime_seconds: 3600 # 1 hour max runtime per run + poll_interval_seconds: 60 # Check every minute + +# Disable telemetry to reduce disk writes +telemetry: + enabled: false schedules: use_threads: true diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 9898f6ce..1c42f7ff 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -7,6 +7,9 @@ services: context: . dockerfile: docker/Dockerfile.anomstack_code image: anomstack_code_image + environment: + DAGSTER_HOST: "anomstack_webserver" + DAGSTER_PORT: "3000" deploy: resources: limits: diff --git a/docker-compose.yaml b/docker-compose.yaml index 2f103f40..80a7a479 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -35,7 +35,9 @@ services: # Multiple containers like this can be deployed separately - each just needs to run on # its own port, and have its own entry in the workspace.yaml file that's loaded by the webserver. anomstack_code: - image: andrewm4894/anomstack_code:latest + build: + context: . + dockerfile: docker/Dockerfile.anomstack_code container_name: anomstack_code restart: always expose: @@ -45,6 +47,7 @@ services: - anomstack_metrics_duckdb:/metrics_db/duckdb - ./dagster_home:/opt/dagster/dagster_home - ./dagster_docker.yaml:/opt/dagster/dagster_home/dagster.yaml + - ./metrics:/opt/dagster/app/metrics # πŸ”₯ ENABLES HOT CONFIG UPDATES env_file: - .env environment: @@ -104,6 +107,7 @@ services: - anomstack_metrics_duckdb:/metrics_db/duckdb - ./dagster_home:/opt/dagster/dagster_home - ./dagster_docker.yaml:/opt/dagster/dagster_home/dagster.yaml + - ./metrics:/opt/dagster/app/metrics # πŸ”₯ ENABLES HOT CONFIG UPDATES networks: - anomstack_network depends_on: @@ -147,6 +151,7 @@ services: - anomstack_metrics_duckdb:/metrics_db/duckdb - ./dagster_home:/opt/dagster/dagster_home - ./dagster_docker.yaml:/opt/dagster/dagster_home/dagster.yaml + - ./metrics:/opt/dagster/app/metrics # πŸ”₯ ENABLES HOT CONFIG UPDATES networks: - anomstack_network depends_on: diff --git a/docker/Dockerfile.anomstack_code b/docker/Dockerfile.anomstack_code index 88379cf1..ee5c6c97 100644 --- a/docker/Dockerfile.anomstack_code +++ b/docker/Dockerfile.anomstack_code @@ -9,7 +9,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ WORKDIR /opt/dagster/app -COPY ../requirements.txt /opt/dagster/app/requirements.txt +COPY requirements.txt /opt/dagster/app/requirements.txt RUN pip install -r requirements.txt @@ -22,10 +22,10 @@ COPY .example.env .env # Run dagster gRPC server on port 4000 EXPOSE 4000 -# Health check for the gRPC server +# Health check for the code server HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \ CMD dagster api grpc-health-check -p 4000 # CMD allows this to be overridden from run launchers or executors that want # to run other commands against your repository -CMD ["dagster", "api", "grpc", "-h", "0.0.0.0", "-p", "4000", "-f", "anomstack/main.py"] +CMD ["dagster", "code-server", "start", "-h", "0.0.0.0", "-p", "4000", "-f", "anomstack/main.py"] diff --git a/docs/docs/configuration/hot-reload.md b/docs/docs/configuration/hot-reload.md new file mode 100644 index 00000000..935beb51 --- /dev/null +++ b/docs/docs/configuration/hot-reload.md @@ -0,0 +1,337 @@ +--- +sidebar_position: 2 +--- + +# Hot Configuration Updates + +This guide explains how to update Anomstack configurations dynamically without restarting Docker containers. + +## πŸ”₯ What's New + +With the updated Docker setup, you can now: + +- **Update YAML configs** in `/metrics` and see changes immediately +- **Modify environment variables** in `.env` (requires container restart) +- **Add new metric batches** without rebuilding images +- **Change SQL queries or Python functions** on the fly +- **Reload Dagster** with a single command + +## πŸš€ Quick Start + +### Option 1: Manual Reload (On-Demand) + +```bash +# 1. Make changes to YAML files in /metrics/ +vim metrics/my_batch/config.yaml + +# 2. Reload manually +make reload-config +``` + +### Option 2: Automatic Scheduled Reload + +```bash +# Enable automatic reload every 5 minutes +make enable-auto-reload + +# Restart containers to activate +make docker-restart +``` + +### Option 3: Smart File Watcher (Recommended) + +```bash +# Enable smart file watching (reloads only when files change) +make enable-config-watcher + +# Restart containers to activate +make docker-restart +``` + +That's it! Your changes will now be automatically detected and applied. + +## πŸ“‹ What Works with Hot Reload + +### βœ… YAML Configuration Changes +- Metric batch configurations (`metrics/*/*.yaml`) +- Default settings (`metrics/defaults/defaults.yaml`) +- Schedule configurations +- Alert thresholds and settings +- Database connection parameters (except connection strings) + +### βœ… SQL Query Updates +- Ingest SQL queries (`ingest_sql` or `ingest_sql_file`) +- Custom SQL templates +- Query logic changes + +### βœ… Python Function Updates +- Custom ingest functions (`ingest_fn`) +- Preprocessing functions +- Python-based metric calculations + +### βœ… New Metric Batches +- Adding entirely new metric batch folders +- New YAML + SQL/Python combinations + +### ⚠️ Requires Manual Container Restart +- Environment variable changes in `.env` +- Database connection string changes +- Infrastructure-level changes + +For `.env` file changes, use this workflow: +```bash +# 1. Edit environment variables +vim .env + +# 2. Restart containers to pick up new environment variables +make docker-restart # or docker compose restart + +# 3. Reload configuration (optional - to be safe) +make reload-config +``` + +## πŸ€– Automated Reloading Options + +### Scheduled Job (Option 2) +- **How it works**: Runs a Dagster job every N minutes to reload configurations +- **Best for**: Development environments where you want periodic updates +- **Configuration**: + ```bash + ANOMSTACK_AUTO_CONFIG_RELOAD=true # Enable the feature + ANOMSTACK_CONFIG_RELOAD_CRON="*/5 * * * *" # Every 5 minutes + ANOMSTACK_CONFIG_RELOAD_STATUS=RUNNING # Start the schedule + ``` +- **Pros**: Simple, predictable timing +- **Cons**: May reload unnecessarily, slight resource usage + +### Smart File Watcher (Option 3) - **Recommended** +- **How it works**: Monitors YAML files for changes, reloads only when needed +- **Best for**: Production and development - most efficient approach +- **Scope**: **YAML files only** - .env changes still require manual restart +- **Configuration**: + ```bash + ANOMSTACK_CONFIG_WATCHER=true # Enable the sensor + ANOMSTACK_CONFIG_WATCHER_INTERVAL=30 # Check every 30 seconds + ``` +- **Pros**: Only reloads when necessary, most responsive, resource efficient +- **Cons**: Doesn't handle .env changes (by design for safety) + +### How File Watching Works +1. **Hash Calculation**: Creates MD5 hash of all YAML files (content + modification time) +2. **Change Detection**: Compares current hash with last known state +3. **Smart Reloading**: Only triggers reload when YAML files actually change +4. **State Persistence**: Remembers last known state across Dagster restarts + +**Note**: File watcher intentionally excludes `.env` files to avoid complex container restart scenarios that could cause side effects. + +## πŸ”§ How It Works + +The hot reload system works by: + +1. **Volume Mounting**: The `/metrics` directory is now mounted as a volume in Docker containers +2. **Dagster Code Location Reload**: Uses Dagster's built-in GraphQL API to reload code locations +3. **Configuration Re-parsing**: The `get_specs()` function re-reads YAML files from the mounted directory + +### Before (Required Container Rebuild) +``` +Host /metrics β†’ Docker Image (frozen at build time) β†’ Container +``` + +### After (Hot Reload Enabled) +``` +Host /metrics β†’ Docker Volume Mount β†’ Container (live updates) +``` + +## πŸ“ Updated Docker Configuration + +### Volume Mounts Added +The following services now mount the metrics directory: + +```yaml +# docker-compose.yaml +anomstack_code: + volumes: + - ./metrics:/opt/dagster/app/metrics # πŸ”₯ NEW + +anomstack_webserver: + volumes: + - ./metrics:/opt/dagster/app/metrics # πŸ”₯ NEW + +anomstack_daemon: + volumes: + - ./metrics:/opt/dagster/app/metrics # πŸ”₯ NEW + +# dagster_docker.yaml (for job containers) +run_launcher: + config: + container_kwargs: + volumes: + - /path/to/metrics:/opt/dagster/app/metrics # πŸ”₯ NEW +``` + +## πŸ”„ Configuration Reload Script + +The `scripts/reload_config.py` script: + +- βœ… Validates configuration files are accessible +- βœ… Checks Dagster health before attempting reload +- βœ… Uses GraphQL API to reload code locations +- βœ… Provides clear success/error messages +- βœ… Shows loaded repositories after reload + +### Script Usage +```bash +# From anomstack root directory +python3 scripts/reload_config.py + +# Via Makefile +make reload-config +``` + +## πŸ› οΈ Troubleshooting + +### Configuration Reload Failed +```bash +# Check container status +docker compose ps + +# Check Dagster logs +docker compose logs anomstack_code + +# Restart specific service if needed +docker compose restart anomstack_code + +# Full restart if required +docker compose restart +``` + +### Volume Mount Issues +If configurations aren't updating: + +1. Verify volume mounts: + ```bash + docker compose exec anomstack_code ls -la /opt/dagster/app/metrics + ``` + +2. Check file permissions: + ```bash + # Ensure metrics directory is readable + chmod -R 755 metrics/ + ``` + +### Environment Variable Updates +For `.env` file changes, use the manual workflow: + +```bash +# 1. Edit your environment variables +vim .env + +# 2. Restart containers (required for env var changes) +make docker-restart + +# 3. Optional: Reload configs to be safe +make reload-config +``` + +**Why manual?** Container restarts require careful orchestration. Automated restart could cause side effects, so we keep this as an explicit, predictable user action. + +## 🎯 Best Practices + +### 1. Test Changes Gradually +- Make small, incremental changes +- Test each change with `make reload-config` +- Monitor Dagster UI for successful reloads + +### 2. Validate Configurations +- Use the reload script to catch syntax errors early +- Check Dagster logs if reload fails +- Keep backup copies of working configurations + +### 3. Environment vs YAML +- Use YAML for business logic (thresholds, schedules, SQL) +- Use environment variables for secrets and deployment-specific settings +- Use environment variable overrides for testing different values + +### 4. Version Control +- Commit configuration changes to git +- Use branching for experimental configurations +- Document breaking changes in commit messages + +## πŸ” Monitoring Changes + +### Dagster UI +- Visit http://localhost:3000 +- Check "Workspace" tab for reload status +- Monitor "Runs" tab for job execution + +### Dashboard +- Visit http://localhost:5001 +- Verify metric batches appear correctly +- Check that data flows as expected + +### Logs +```bash +# Watch all services +docker compose logs -f + +# Focus on specific service +docker compose logs -f anomstack_code +``` + +## 🚨 Migration Notes + +If you're updating from a previous version: + +1. **Pull latest changes** including Docker configuration updates +2. **Restart containers** to apply volume mounts: + ```bash + docker compose down + docker compose up -d + ``` +3. **Test hot reload** with a small configuration change +4. **Update your workflow** to use `make reload-config` instead of container restarts + +## πŸ’‘ Advanced Usage + +### Custom Dagster Host/Port +```bash +# If running Dagster on different host/port +DAGSTER_HOST=my-host DAGSTER_PORT=3001 python3 scripts/reload_config.py +``` + +### Multiple Code Locations +```python +# Modify scripts/reload_config.py to reload specific locations +reload_code_location("my_custom_location") +``` + +### Automated Reloading +```bash +# Use with file watching tools for automatic reload +# (Not recommended for production) +fswatch -o metrics/ | xargs -n1 -I{} make reload-config +``` + +## πŸŽ‰ Benefits + +### Manual Reload +- **πŸš€ Faster Development**: No more waiting for container rebuilds +- **πŸ”„ Quick Iterations**: Test configuration changes in seconds +- **πŸ’‘ Better UX**: Immediate feedback on configuration changes + +### Automated Reload (YAML Files Only) +- **πŸ€– Zero Intervention**: Set it and forget it - YAML configs update automatically +- **⚑ Real-time Updates**: YAML changes detected and applied within seconds +- **🎯 Smart Detection**: Only reloads when YAML files actually change +- **πŸ“Š Dagster Integration**: Full visibility in Dagster UI (jobs, sensors, logs) +- **πŸ›‘οΈ Production Ready**: Robust error handling and logging +- **πŸ”’ Safe by Design**: Avoids complex container restart automation + +### Overall +- **πŸ›‘οΈ Safer Operations**: Avoid full container restarts in production +- **πŸ“ˆ Improved Productivity**: Spend more time on analysis, less on deployment +- **πŸ” Full Observability**: All reload attempts logged and trackable in Dagster + +--- + +*This feature requires Anomstack v1.x+ with updated Docker configurations.* \ No newline at end of file diff --git a/docs/docs/deployment/docker.md b/docs/docs/deployment/docker.md index 055932df..1c7b252a 100644 --- a/docs/docs/deployment/docker.md +++ b/docs/docs/deployment/docker.md @@ -39,4 +39,12 @@ Coming soon... ## Best Practices -Coming soon... \ No newline at end of file +### Storage Management + +Dagster can accumulate significant storage over time. To prevent disk space issues: + +- **Monitor storage regularly**: Use `make dagster-cleanup-status` +- **Configure retention policies**: Ensure your `dagster_docker.yaml` has appropriate cleanup settings +- **Schedule regular cleanup**: Run weekly/monthly cleanup jobs + +For detailed guidance, see the [Storage Optimization](../storage-optimization.md) guide. \ No newline at end of file diff --git a/docs/docs/storage-optimization.md b/docs/docs/storage-optimization.md new file mode 100644 index 00000000..1285f8a7 --- /dev/null +++ b/docs/docs/storage-optimization.md @@ -0,0 +1,214 @@ +--- +sidebar_position: 3 +title: Storage Optimization +description: Prevent Dagster storage buildup and manage disk space effectively +--- + +# Dagster Storage Optimization + +## The Problem: Excessive Storage Buildup + +During development, Dagster can accumulate massive amounts of storage data if not properly configured. This can lead to: + +- **212,000+ run directories** in `tmp/` +- **63GB+ of accumulated storage** +- System performance degradation +- Disk space exhaustion + +## Root Causes + +1. **No retention policies** configured by default +2. **High concurrent runs** (25+ simultaneous) creating many artifacts +3. **Indefinite storage** of compute logs and run metadata +4. **No automatic cleanup** of old runs +5. **Large temp directories** without size limits + +## Solutions Implemented + +### 1. Configuration Improvements + +#### Updated `dagster.yaml` +```yaml +retention: + schedule: + purge_after_days: 3 # Reduced from 7 + sensor: + purge_after_days: + skipped: 1 # Keep only 1 day of skipped runs + failure: 3 # Keep 3 days of failures for debugging + success: 1 # Keep only 1 day of successful runs +``` + +#### Updated `dagster_docker.yaml` +```yaml +run_coordinator: + config: + max_concurrent_runs: 10 # Reduced from 25 + +# Added retention policies +retention: + schedule: + purge_after_days: 3 + sensor: + purge_after_days: + skipped: 1 + failure: 3 + success: 1 + +# Added run monitoring +run_monitoring: + enabled: true + start_timeout_seconds: 300 + cancel_timeout_seconds: 180 + max_runtime_seconds: 3600 + poll_interval_seconds: 60 + +# Disabled telemetry to reduce disk writes +telemetry: + enabled: false +``` + +### 2. Automated Cleanup Tools + +#### Cleanup Script: `scripts/utils/cleanup_dagster_storage.sh` + +**Interactive Menu:** +```bash +./scripts/utils/cleanup_dagster_storage.sh +# or +make dagster-cleanup-menu +``` + +**Direct Commands:** +```bash +# Check current status +make dagster-cleanup-status + +# Safe cleanup (recommended) +make dagster-cleanup-minimal + +# Remove old runs (30+ days) +make dagster-cleanup-standard + +# Aggressive cleanup (7+ days) +make dagster-cleanup-aggressive +``` + +#### Cleanup Levels: + +1. **Minimal** (πŸ”§): Remove old logs only - safe for production +2. **Standard** (🧹): Remove runs older than 30 days +3. **Aggressive** (πŸ”₯): Remove runs older than 7 days +4. **Nuclear** (☒️): Remove all but last 24 hours +5. **CLI-based** (πŸ› οΈ): Use Dagster's built-in cleanup commands + +### 3. Environment Variable Updates + +Updated `.example.env` with lightweight defaults: +```bash +# Lightweight storage paths +ANOMSTACK_DAGSTER_LOCAL_ARTIFACT_STORAGE_DIR=tmp_light/artifacts +ANOMSTACK_DAGSTER_LOCAL_COMPUTE_LOG_MANAGER_DIRECTORY=tmp_light/compute_logs +ANOMSTACK_DAGSTER_SQLITE_STORAGE_BASE_DIR=tmp_light/storage + +# Reduced concurrency +ANOMSTACK_DAGSTER_OVERALL_CONCURRENCY_LIMIT=5 # Reduced from 10 +ANOMSTACK_DAGSTER_DEQUEUE_NUM_WORKERS=2 # Reduced from 4 +``` + +## Best Practices + +### For Developers + +1. **Regular Monitoring:** + ```bash + make dagster-cleanup-status # Check storage weekly + ``` + +2. **Routine Cleanup:** + ```bash + make dagster-cleanup-minimal # Weekly log cleanup + make dagster-cleanup-standard # Monthly run cleanup + ``` + +3. **Emergency Cleanup:** + ```bash + make dagster-cleanup-aggressive # When disk space is low + ``` + +### For Production + +1. **Configure retention policies** in `dagster_docker.yaml` +2. **Limit concurrent runs** to reasonable numbers (5-15) +3. **Enable run monitoring** to detect stuck runs +4. **Set up automated cleanup** using cron jobs: + ```bash + # Weekly cleanup cron job + 0 2 * * 0 /path/to/cleanup_dagster_storage.sh minimal + + # Monthly aggressive cleanup + 0 3 1 * * /path/to/cleanup_dagster_storage.sh standard + ``` + +### For CI/CD + +1. **Use ephemeral storage** when possible +2. **Clean up after tests:** + ```bash + make dagster-cleanup-aggressive + ``` +3. **Monitor disk usage** in build scripts + +## Storage Size Guidelines + +| Storage Level | Recommended Action | +|---------------|-------------------| +| < 1GB | βœ… Healthy - continue monitoring | +| 1-5GB | ⚠️ Consider weekly cleanup | +| 5-20GB | πŸ”„ Run standard cleanup monthly | +| 20-50GB | πŸ”₯ Run aggressive cleanup | +| > 50GB | ☒️ Emergency cleanup required | + +## Troubleshooting + +### Issue: "212,000+ run directories" +**Solution:** Run nuclear cleanup, then configure retention policies + +### Issue: "Disk space full" +**Solution:** +1. Run `make dagster-cleanup-aggressive` +2. If still full, run `make reset-nuclear` +3. Configure retention policies before restarting + +### Issue: "Slow Dagster performance" +**Solution:** +1. Check storage with `make dagster-cleanup-status` +2. Run appropriate cleanup level +3. Reduce `max_concurrent_runs` + +### Issue: "Old runs not being cleaned up" +**Solution:** +1. Verify retention policies in `dagster_docker.yaml` +2. Ensure Dagster daemon is running +3. Check database connectivity for PostgreSQL storage + +## Prevention Checklist + +- [ ] Retention policies configured in `dagster_docker.yaml` +- [ ] Run monitoring enabled +- [ ] Concurrent runs limited (≀ 15) +- [ ] Regular cleanup scheduled (weekly/monthly) +- [ ] Storage monitoring in place +- [ ] Telemetry disabled in production +- [ ] Environment variables optimized + +## Additional Resources + +- **Interactive Cleanup**: `make dagster-cleanup-menu` +- **Makefile Documentation**: Available in the project root `Makefile.md#dagster-storage-cleanup` +- **Reset Scripts**: Available in `scripts/utils/` directory +- **Dagster Retention Docs**: [Official Documentation](https://docs.dagster.io/deployment/run-monitoring) + +--- + +**πŸ’‘ Key Takeaway**: Proactive storage management prevents the 63GB+ buildup problem. Regular monitoring and cleanup are essential for healthy Dagster deployments. \ No newline at end of file diff --git a/docs/sidebars.js b/docs/sidebars.js index ca910eb4..9bba985c 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -67,6 +67,7 @@ const sidebars = { items: [ 'deployment/docker', 'deployment/gcp', + 'storage-optimization', ], }, { @@ -74,6 +75,7 @@ const sidebars = { label: 'Configuration', items: [ 'configuration/metrics', + 'configuration/hot-reload', ], }, { diff --git a/docs/yarn.lock b/docs/yarn.lock index 20c5adc2..aca210e8 100644 --- a/docs/yarn.lock +++ b/docs/yarn.lock @@ -75,6 +75,11 @@ "@algolia/requester-common" "4.24.0" "@algolia/transporter" "4.24.0" +"@algolia/client-common@5.9.1": + version "5.9.1" + resolved "https://registry.npmjs.org/@algolia/client-common/-/client-common-5.9.1.tgz" + integrity sha512-YWPGDyISFNbPFVswI16c4rgt2CeTgFk82e543FSyw/3H5eNKa0YPb876GguEb50NualXCF7DCuVhcp6XMTpaSg== + "@algolia/client-personalization@4.24.0": version "4.24.0" resolved "https://registry.npmjs.org/@algolia/client-personalization/-/client-personalization-4.24.0.tgz" @@ -84,6 +89,16 @@ "@algolia/requester-common" "4.24.0" "@algolia/transporter" "4.24.0" +"@algolia/client-search@>= 4.9.1 < 6": + version "5.9.1" + resolved "https://registry.npmjs.org/@algolia/client-search/-/client-search-5.9.1.tgz" + integrity sha512-JDK8kv1ZR2uwEbUVOZ6GA2AQyHI1+T5noupyTsu7SY2M6W3wPwQO3oUou6Xq+fWJmXGUB8TUj4Yv3ioYoTjQVg== + dependencies: + "@algolia/client-common" "5.9.1" + "@algolia/requester-browser-xhr" "5.9.1" + "@algolia/requester-fetch" "5.9.1" + "@algolia/requester-node-http" "5.9.1" + "@algolia/client-search@4.24.0": version "4.24.0" resolved "https://registry.npmjs.org/@algolia/client-search/-/client-search-4.24.0.tgz" @@ -134,11 +149,25 @@ dependencies: "@algolia/requester-common" "4.24.0" +"@algolia/requester-browser-xhr@5.9.1": + version "5.9.1" + resolved "https://registry.npmjs.org/@algolia/requester-browser-xhr/-/requester-browser-xhr-5.9.1.tgz" + integrity sha512-EevzJJ2AXu+U2w14XgK9GnJn9Y4q5GNnoAUWS0aErCCb7XhYiM7xa1eJnVq+FoOwRuZj8RmS4GEV7t3CQI3TqA== + dependencies: + "@algolia/client-common" "5.9.1" + "@algolia/requester-common@4.24.0": version "4.24.0" resolved "https://registry.npmjs.org/@algolia/requester-common/-/requester-common-4.24.0.tgz" integrity sha512-k3CXJ2OVnvgE3HMwcojpvY6d9kgKMPRxs/kVohrwF5WMr2fnqojnycZkxPoEg+bXm8fi5BBfFmOqgYztRtHsQA== +"@algolia/requester-fetch@5.9.1": + version "5.9.1" + resolved "https://registry.npmjs.org/@algolia/requester-fetch/-/requester-fetch-5.9.1.tgz" + integrity sha512-gBSi8QotBfOu3BbF25PB2uVbCNfrjVDGyvfeIQ6DukUldjEE8ruusNJnVMHoR00rO1C8G86/USHkbmXx73vf7Q== + dependencies: + "@algolia/client-common" "5.9.1" + "@algolia/requester-node-http@4.24.0": version "4.24.0" resolved "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-4.24.0.tgz" @@ -146,6 +175,13 @@ dependencies: "@algolia/requester-common" "4.24.0" +"@algolia/requester-node-http@5.9.1": + version "5.9.1" + resolved "https://registry.npmjs.org/@algolia/requester-node-http/-/requester-node-http-5.9.1.tgz" + integrity sha512-ImECpAR0A0q+9UfTprA099JJ6VZ+GjUoOC+m5rbyJieA4rUbt/A6QHkqeUq/2fObeezOzLn4DZDAXW93YHM+oQ== + dependencies: + "@algolia/client-common" "5.9.1" + "@algolia/transporter@4.24.0": version "4.24.0" resolved "https://registry.npmjs.org/@algolia/transporter/-/transporter-4.24.0.tgz" @@ -176,7 +212,7 @@ resolved "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.23.3.tgz" integrity sha512-BmR4bWbDIoFJmJ9z2cZ8Gmm2MXgEDgjdWgpKmKWUt54UGFJdlj31ECtbaDvCG/qVdG3AQ1SfpZEs01lUFbzLOQ== -"@babel/core@^7.21.3", "@babel/core@^7.23.3": +"@babel/core@^7.0.0", "@babel/core@^7.0.0-0", "@babel/core@^7.0.0-0 || ^8.0.0-0 <8.0.0", "@babel/core@^7.12.0", "@babel/core@^7.13.0", "@babel/core@^7.21.3", "@babel/core@^7.23.3", "@babel/core@^7.4.0 || ^8.0.0-0 <8.0.0": version "7.23.3" resolved "https://registry.npmjs.org/@babel/core/-/core-7.23.3.tgz" integrity sha512-Jg+msLuNuCJDyBvFv5+OKOUjWMZgd85bKjbICd3zWrKAo+bJ49HJufi7CQE0q0uR8NGyO6xkCACScNqyjHSZew== @@ -1227,7 +1263,7 @@ "@docsearch/css" "3.6.2" algoliasearch "^4.19.1" -"@docusaurus/core@3.5.2", "@docusaurus/core@^3.5.2": +"@docusaurus/core@^3.5.2", "@docusaurus/core@3.5.2": version "3.5.2" resolved "https://registry.npmjs.org/@docusaurus/core/-/core-3.5.2.tgz" integrity sha512-4Z1WkhCSkX4KO0Fw5m/Vuc7Q3NxBG53NE5u59Rs96fWkMPZVSrzEPP16/Nk6cWb/shK7xXPndTmalJtw7twL/w== @@ -1400,7 +1436,7 @@ utility-types "^3.10.0" webpack "^5.88.1" -"@docusaurus/plugin-content-docs@3.5.2": +"@docusaurus/plugin-content-docs@*", "@docusaurus/plugin-content-docs@3.5.2": version "3.5.2" resolved "https://registry.npmjs.org/@docusaurus/plugin-content-docs/-/plugin-content-docs-3.5.2.tgz" integrity sha512-Bt+OXn/CPtVqM3Di44vHjE7rPCEsRCB/DMo2qoOuozB9f7+lsdrHvD0QCHdBs0uhz6deYJDppAr2VgqybKPlVQ== @@ -1601,7 +1637,7 @@ fs-extra "^11.1.1" tslib "^2.6.0" -"@docusaurus/types@3.0.0": +"@docusaurus/types@*", "@docusaurus/types@3.0.0": version "3.0.0" resolved "https://registry.npmjs.org/@docusaurus/types/-/types-3.0.0.tgz" integrity sha512-Qb+l/hmCOVemReuzvvcFdk84bUmUFyD0Zi81y651ie3VwMrXqC7C0E7yZLKMOsLj/vkqsxHbtkAuYMI89YzNzg== @@ -1798,7 +1834,7 @@ "@nodelib/fs.stat" "2.0.5" run-parallel "^1.1.9" -"@nodelib/fs.stat@2.0.5", "@nodelib/fs.stat@^2.0.2": +"@nodelib/fs.stat@^2.0.2", "@nodelib/fs.stat@2.0.5": version "2.0.5" resolved "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz" integrity sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A== @@ -1932,7 +1968,7 @@ "@svgr/babel-plugin-transform-react-native-svg" "8.1.0" "@svgr/babel-plugin-transform-svg-component" "8.0.0" -"@svgr/core@8.1.0": +"@svgr/core@*", "@svgr/core@8.1.0": version "8.1.0" resolved "https://registry.npmjs.org/@svgr/core/-/core-8.1.0.tgz" integrity sha512-8QqtOQT5ACVlmsvKOJNEaWmRPmcojMOzCz4Hs2BGG/toAp/K38LcsMRyLp349glq5AzJbCEeimEoxaX6v/fLrA== @@ -2227,7 +2263,7 @@ "@types/history" "^4.7.11" "@types/react" "*" -"@types/react@*": +"@types/react@*", "@types/react@>= 16.8.0 < 19.0.0", "@types/react@>=16": version "18.2.37" resolved "https://registry.npmjs.org/@types/react/-/react-18.2.37.tgz" integrity sha512-RGAYMi2bhRgEXT3f4B92WTohopH6bIXw05FuGlmJEnv/omEn190+QYEIYxIAuIBdKgboYYdVved2p1AxZVQnaw== @@ -2318,7 +2354,7 @@ resolved "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz" integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== -"@webassemblyjs/ast@1.12.1", "@webassemblyjs/ast@^1.12.1": +"@webassemblyjs/ast@^1.12.1", "@webassemblyjs/ast@1.12.1": version "1.12.1" resolved "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz" integrity sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg== @@ -2419,7 +2455,7 @@ "@webassemblyjs/wasm-gen" "1.12.1" "@webassemblyjs/wasm-parser" "1.12.1" -"@webassemblyjs/wasm-parser@1.12.1", "@webassemblyjs/wasm-parser@^1.12.1": +"@webassemblyjs/wasm-parser@^1.12.1", "@webassemblyjs/wasm-parser@1.12.1": version "1.12.1" resolved "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz" integrity sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ== @@ -2472,7 +2508,7 @@ acorn-walk@^8.0.0: resolved "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.0.tgz" integrity sha512-FS7hV565M5l1R08MXqo8odwMTB02C2UqzB17RVgu9EyuYFBqJZ3/ZY97sQD5FewVu1UyDFc1yztUDrAwT0EypA== -acorn@^8.0.0, acorn@^8.0.4, acorn@^8.7.1, acorn@^8.8.2: +"acorn@^6.0.0 || ^7.0.0 || ^8.0.0", acorn@^8, acorn@^8.0.0, acorn@^8.0.4, acorn@^8.7.1, acorn@^8.8.2: version "8.11.2" resolved "https://registry.npmjs.org/acorn/-/acorn-8.11.2.tgz" integrity sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w== @@ -2497,7 +2533,12 @@ ajv-formats@^2.1.1: dependencies: ajv "^8.0.0" -ajv-keywords@^3.4.1, ajv-keywords@^3.5.2: +ajv-keywords@^3.4.1: + version "3.5.2" + resolved "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz" + integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== + +ajv-keywords@^3.5.2: version "3.5.2" resolved "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz" integrity sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ== @@ -2509,7 +2550,7 @@ ajv-keywords@^5.1.0: dependencies: fast-deep-equal "^3.1.3" -ajv@^6.12.2, ajv@^6.12.5: +ajv@^6.12.2, ajv@^6.12.5, ajv@^6.9.1: version "6.12.6" resolved "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz" integrity sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g== @@ -2519,7 +2560,7 @@ ajv@^6.12.2, ajv@^6.12.5: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -ajv@^8.0.0, ajv@^8.9.0: +ajv@^8.0.0, ajv@^8.8.2, ajv@^8.9.0: version "8.12.0" resolved "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz" integrity sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA== @@ -2536,7 +2577,7 @@ algoliasearch-helper@^3.13.3: dependencies: "@algolia/events" "^4.0.1" -algoliasearch@^4.18.0, algoliasearch@^4.19.1: +algoliasearch@^4.18.0, algoliasearch@^4.19.1, "algoliasearch@>= 3.1 < 6", "algoliasearch@>= 4.9.1 < 6": version "4.24.0" resolved "https://registry.npmjs.org/algoliasearch/-/algoliasearch-4.24.0.tgz" integrity sha512-bf0QV/9jVejssFBmz2HQLxUadxk574t4iwjCKp5E7NBzwKkrDEhKPISIIjAU/p6K5qDx3qoeh4+26zWN1jmw3g== @@ -2623,16 +2664,16 @@ argparse@^2.0.1: resolved "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -array-flatten@1.1.1: - version "1.1.1" - resolved "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" - integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== - array-flatten@^2.1.2: version "2.1.2" resolved "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz" integrity sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ== +array-flatten@1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz" + integrity sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg== + array-union@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz" @@ -2800,7 +2841,7 @@ braces@^3.0.3, braces@~3.0.2: dependencies: fill-range "^7.1.1" -browserslist@^4.0.0, browserslist@^4.18.1, browserslist@^4.21.10, browserslist@^4.21.9, browserslist@^4.22.1, browserslist@^4.23.0, browserslist@^4.23.3: +browserslist@^4.0.0, browserslist@^4.18.1, browserslist@^4.21.10, browserslist@^4.21.9, browserslist@^4.22.1, browserslist@^4.23.0, browserslist@^4.23.3, "browserslist@>= 4.21.0": version "4.24.0" resolved "https://registry.npmjs.org/browserslist/-/browserslist-4.24.0.tgz" integrity sha512-Rmb62sR1Zpjql25eSanFGEhAxcFwfA1K0GuQcLoaJBAcENegrQut3hYdhXFF1obQfiDyqIW/cLM5HSJ/9k884A== @@ -3058,16 +3099,16 @@ color-convert@^2.0.1: dependencies: color-name "~1.1.4" -color-name@1.1.3: - version "1.1.3" - resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" - integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== - color-name@~1.1.4: version "1.1.4" resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz" integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== +color-name@1.1.3: + version "1.1.3" + resolved "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz" + integrity sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw== + colord@^2.9.3: version "2.9.3" resolved "https://registry.npmjs.org/colord/-/colord-2.9.3.tgz" @@ -3438,20 +3479,27 @@ debounce@^1.2.1: resolved "https://registry.npmjs.org/debounce/-/debounce-1.2.1.tgz" integrity sha512-XRRe6Glud4rd/ZGQfiV1ruXSfbvfJedlV9Y6zOlP+2K04vBYiJEte6stfFkCP03aMnY5tsipamumUjL14fofug== -debug@2.6.9, debug@^2.6.0: +debug@^2.6.0: version "2.6.9" resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== dependencies: ms "2.0.0" -debug@4, debug@^4.0.0, debug@^4.1.0, debug@^4.1.1: +debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@4: version "4.3.4" resolved "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== dependencies: ms "2.1.2" +debug@2.6.9: + version "2.6.9" + resolved "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz" + integrity sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA== + dependencies: + ms "2.0.0" + decode-named-character-reference@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz" @@ -3525,16 +3573,16 @@ del@^6.1.1: rimraf "^3.0.2" slash "^3.0.0" -depd@2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" - integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== - depd@~1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz" integrity sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ== +depd@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz" + integrity sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw== + dequal@^2.0.0: version "2.0.3" resolved "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz" @@ -4053,7 +4101,7 @@ feed@^4.2.2: dependencies: xml-js "^1.6.11" -file-loader@^6.2.0: +file-loader@*, file-loader@^6.2.0: version "6.2.0" resolved "https://registry.npmjs.org/file-loader/-/file-loader-6.2.0.tgz" integrity sha512-qo3glqyTa61Ytg4u73GultjHGjdRyig3tG6lPtyX/jOEJvHif9uB0/OCI2Kif6ctF3caQTW2G5gym21oAsI4pw== @@ -4202,7 +4250,7 @@ fs.realpath@^1.0.0: fsevents@~2.3.2: version "2.3.3" - resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" + resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz" integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== function-bind@^1.1.2: @@ -4347,16 +4395,16 @@ got@^12.1.0: p-cancelable "^3.0.0" responselike "^3.0.0" -graceful-fs@4.2.10: - version "4.2.10" - resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz" - integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== - graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0, graceful-fs@^4.2.11, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== +graceful-fs@4.2.10: + version "4.2.10" + resolved "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz" + integrity sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA== + gray-matter@^4.0.3: version "4.0.3" resolved "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz" @@ -4653,6 +4701,16 @@ http-deceiver@^1.2.7: resolved "https://registry.npmjs.org/http-deceiver/-/http-deceiver-1.2.7.tgz" integrity sha512-LmpOGxTfbpgtGVxJrj5k7asXHCgNZp5nLfp+hWc8QQRqtb7fUy6kRY3BO1h9ddF6yIPYUARgxGOwB42DnxIaNw== +http-errors@~1.6.2: + version "1.6.3" + resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz" + integrity sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A== + dependencies: + depd "~1.1.2" + inherits "2.0.3" + setprototypeof "1.1.0" + statuses ">= 1.4.0 < 2" + http-errors@2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz" @@ -4664,16 +4722,6 @@ http-errors@2.0.0: statuses "2.0.1" toidentifier "1.0.1" -http-errors@~1.6.2: - version "1.6.3" - resolved "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz" - integrity sha512-lks+lVC8dgGyh97jxvxeYTWQFvh4uw4yC12gVl63Cg30sjPX4wuGcdkICVXDAESr6OJGjqGA8Iz5mkeN6zlD7A== - dependencies: - depd "~1.1.2" - inherits "2.0.3" - setprototypeof "1.1.0" - statuses ">= 1.4.0 < 2" - http-parser-js@>=0.5.1: version "0.5.8" resolved "https://registry.npmjs.org/http-parser-js/-/http-parser-js-0.5.8.tgz" @@ -4777,7 +4825,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3: +inherits@^2.0.1, inherits@^2.0.3, inherits@~2.0.3, inherits@2, inherits@2.0.4: version "2.0.4" resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -4787,16 +4835,16 @@ inherits@2.0.3: resolved "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz" integrity sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw== -ini@2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz" - integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== - ini@^1.3.4, ini@^1.3.5, ini@~1.3.0: version "1.3.8" resolved "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz" integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== +ini@2.0.0: + version "2.0.0" + resolved "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz" + integrity sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA== + inline-style-parser@0.1.1: version "0.1.1" resolved "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.1.1.tgz" @@ -4819,16 +4867,16 @@ invariant@^2.2.4: dependencies: loose-envify "^1.0.0" -ipaddr.js@1.9.1: - version "1.9.1" - resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz" - integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== - ipaddr.js@^2.0.1: version "2.1.0" resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.1.0.tgz" integrity sha512-LlbxQ7xKzfBusov6UMi4MFpEg0m+mAm9xyNGEduwXMEDuf4WfzB/RZwMVYEd7IKGvh4IUkEXYxtAVu9T3OelJQ== +ipaddr.js@1.9.1: + version "1.9.1" + resolved "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz" + integrity sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g== + is-alphabetical@^2.0.0: version "2.0.1" resolved "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz" @@ -4997,16 +5045,16 @@ is-yarn-global@^0.4.0: resolved "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.4.1.tgz" integrity sha512-/kppl+R+LO5VmhYSEWARUFjodS25D68gvj8W7z0I7OWhUla5xWu8KL6CtB2V0R6yqhnRgbcaREMr4EEM6htLPQ== -isarray@0.0.1: - version "0.0.1" - resolved "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" - integrity sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ== - isarray@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz" integrity sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ== +isarray@0.0.1: + version "0.0.1" + resolved "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" + integrity sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ== + isexe@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz" @@ -5968,7 +6016,7 @@ micromatch@^4.0.2, micromatch@^4.0.4, micromatch@^4.0.5: braces "^3.0.3" picomatch "^2.3.1" -mime-db@1.52.0, "mime-db@>= 1.43.0 < 2": +"mime-db@>= 1.43.0 < 2": version "1.52.0" resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz" integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== @@ -5978,14 +6026,40 @@ mime-db@~1.33.0: resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.33.0.tgz" integrity sha512-BHJ/EKruNIqJf/QahvxwQZXKygOQ256myeN/Ew+THcAa5q+PjyTTMMeNQC4DZw5AwfvelsUrA6B67NKMqXDbzQ== -mime-types@2.1.18, mime-types@~2.1.17: +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.27: + version "2.1.35" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + +mime-types@^2.1.31: + version "2.1.35" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + +mime-types@~2.1.17, mime-types@2.1.18: version "2.1.18" resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.18.tgz" integrity sha512-lc/aahn+t4/SWV/qcmumYjymLsWfN3ELhpmVuUFjgsORruuZPVSwAQryq+HHGvO/SI2KVX26bx+En+zhM8g8hQ== dependencies: mime-db "~1.33.0" -mime-types@^2.1.27, mime-types@^2.1.31, mime-types@~2.1.24, mime-types@~2.1.34: +mime-types@~2.1.24: + version "2.1.35" + resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + +mime-types@~2.1.34: version "2.1.35" resolved "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz" integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== @@ -6024,7 +6098,7 @@ minimalistic-assert@^1.0.0: resolved "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz" integrity sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A== -minimatch@3.1.2, minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1: +minimatch@^3.0.4, minimatch@^3.0.5, minimatch@^3.1.1, minimatch@3.1.2: version "3.1.2" resolved "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz" integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw== @@ -6395,6 +6469,13 @@ path-parse@^1.0.7: resolved "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz" integrity sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw== +path-to-regexp@^1.7.0: + version "1.9.0" + resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.9.0.tgz" + integrity sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g== + dependencies: + isarray "0.0.1" + path-to-regexp@0.1.10: version "0.1.10" resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.10.tgz" @@ -6405,13 +6486,6 @@ path-to-regexp@3.3.0: resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-3.3.0.tgz" integrity sha512-qyCH421YQPS2WFDxDjftfc1ZR5WKQzVzqsp4n9M2kQhVOo/ByahFoUNJfl58kOcEGfQ//7weFTDhm+ss8Ecxgw== -path-to-regexp@^1.7.0: - version "1.9.0" - resolved "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.9.0.tgz" - integrity sha512-xIp7/apCFJuUHdDLWe8O1HIkb0kQrOMb/0u6FXQjemHn/ii5LrIzU6bdECnsiTF/GjZkMEKg1xdiZwNqDYlZ6g== - dependencies: - isarray "0.0.1" - path-type@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz" @@ -6721,7 +6795,7 @@ postcss-zindex@^6.0.2: resolved "https://registry.npmjs.org/postcss-zindex/-/postcss-zindex-6.0.2.tgz" integrity sha512-5BxW9l1evPB/4ZIc+2GobEBoKC+h8gPGCMi+jxsYvd2x0mjq7wazk6DrP71pStqxE9Foxh5TVnonbWpFZzXaYg== -postcss@^8.4.21, postcss@^8.4.24, postcss@^8.4.26, postcss@^8.4.38: +"postcss@^7.0.0 || ^8.0.1", postcss@^8.0.9, postcss@^8.1.0, postcss@^8.2.2, postcss@^8.4.21, postcss@^8.4.23, postcss@^8.4.24, postcss@^8.4.26, postcss@^8.4.31, postcss@^8.4.38: version "8.4.47" resolved "https://registry.npmjs.org/postcss/-/postcss-8.4.47.tgz" integrity sha512-56rxCq7G/XfB4EkXq9Egn5GCqugWvDFjafDOThIdMBsI15iqPqR5r15TfSr1YPYeEI19YeaXMCbY6u88Y76GLQ== @@ -6839,16 +6913,21 @@ randombytes@^2.1.0: dependencies: safe-buffer "^5.1.0" -range-parser@1.2.0: - version "1.2.0" - resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz" - integrity sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A== +range-parser@^1.2.1: + version "1.2.1" + resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz" + integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== -range-parser@^1.2.1, range-parser@~1.2.1: +range-parser@~1.2.1: version "1.2.1" resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz" integrity sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg== +range-parser@1.2.0: + version "1.2.0" + resolved "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz" + integrity sha512-kA5WQoNVo4t9lNx2kQNFCxKeBl5IbbSNBl1M/tLkw9WCn+hxNBAW5Qh8gdhs63CJnhjJ2zQWFoqPJP2sK1AV5A== + raw-body@2.5.2: version "2.5.2" resolved "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz" @@ -6899,7 +6978,7 @@ react-dev-utils@^12.0.1: strip-ansi "^6.0.1" text-table "^0.2.0" -react-dom@^18.0.0: +react-dom@*, "react-dom@^16.6.0 || ^17.0.0 || ^18.0.0", react-dom@^18.0.0, "react-dom@>= 16.8.0 < 19.0.0": version "18.2.0" resolved "https://registry.npmjs.org/react-dom/-/react-dom-18.2.0.tgz" integrity sha512-6IMTriUmvsjHUjNtEDudZfuDQUoWXVxKHhlEGSk81n4YFS+r/Kl99wXiwlVXtPBtJenozv2P+hxDsw9eA7Xo6g== @@ -6945,7 +7024,7 @@ react-loadable-ssr-addon-v5-slorber@^1.0.1: dependencies: "@babel/runtime" "^7.10.3" -"react-loadable@npm:@docusaurus/react-loadable@5.5.2": +react-loadable@*, "react-loadable@npm:@docusaurus/react-loadable@5.5.2": version "5.5.2" resolved "https://registry.npmjs.org/@docusaurus/react-loadable/-/react-loadable-5.5.2.tgz" integrity sha512-A3dYjdBGuy0IGT+wyLIGIKLRE+sAk1iNk0f1HjNDysO7u8lhL4N3VEm+FAubmJbAztn94F7MxBTPmnixbiyFdQ== @@ -6980,7 +7059,7 @@ react-router-dom@^5.3.4: tiny-invariant "^1.0.2" tiny-warning "^1.0.0" -react-router@5.3.4, react-router@^5.3.4: +react-router@^5.3.4, react-router@>=5, react-router@5.3.4: version "5.3.4" resolved "https://registry.npmjs.org/react-router/-/react-router-5.3.4.tgz" integrity sha512-Ys9K+ppnJah3QuaRiLxk+jDWOR1MekYQrlytiXxC1RyfbdsZkS5pvKAzCCr031xHixZwpnsYNT5xysdFHQaYsA== @@ -6995,7 +7074,7 @@ react-router@5.3.4, react-router@^5.3.4: tiny-invariant "^1.0.2" tiny-warning "^1.0.0" -react@^18.0.0: +react@*, "react@^16.13.1 || ^17.0.0 || ^18.0.0", "react@^16.6.0 || ^17.0.0 || ^18.0.0", react@^18.0.0, react@^18.2.0, "react@>= 16.8.0 < 19.0.0", react@>=15, react@>=16, react@>=16.0.0: version "18.2.0" resolved "https://registry.npmjs.org/react/-/react-18.2.0.tgz" integrity sha512-/3IjMdb2L9QbBdWiW5e3P2/npwMBaU9mHCSCUzNln0ZCYbcfTsGbTJrU/kGemdH2IWmB2ioZ+zkxtmq6g09fGQ== @@ -7347,15 +7426,20 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" -safe-buffer@5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: +safe-buffer@^5.1.0, safe-buffer@>=5.1.0, safe-buffer@~5.2.0, safe-buffer@5.2.1: + version "5.2.1" + resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + +safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safe-buffer@5.2.1, safe-buffer@>=5.1.0, safe-buffer@^5.1.0, safe-buffer@~5.2.0: - version "5.2.1" - resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz" - integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== +safe-buffer@5.1.2: + version "5.1.2" + resolved "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz" + integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== "safer-buffer@>= 2.1.2 < 3": version "2.1.2" @@ -7374,16 +7458,25 @@ scheduler@^0.23.0: dependencies: loose-envify "^1.1.0" -schema-utils@2.7.0: - version "2.7.0" - resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz" - integrity sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A== +schema-utils@^3.0.0: + version "3.3.0" + resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz" + integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== dependencies: - "@types/json-schema" "^7.0.4" - ajv "^6.12.2" - ajv-keywords "^3.4.1" + "@types/json-schema" "^7.0.8" + ajv "^6.12.5" + ajv-keywords "^3.5.2" + +schema-utils@^3.1.1: + version "3.3.0" + resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz" + integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== + dependencies: + "@types/json-schema" "^7.0.8" + ajv "^6.12.5" + ajv-keywords "^3.5.2" -schema-utils@^3.0.0, schema-utils@^3.1.1, schema-utils@^3.2.0: +schema-utils@^3.2.0: version "3.3.0" resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-3.3.0.tgz" integrity sha512-pN/yOAvcC+5rQ5nERGuwrjLlYvLTbCibnZ1I7B1LaiAz9BRBlE9GMgE/eqV30P7aJQUf7Ddimy/RsbYO/GrVGg== @@ -7402,6 +7495,20 @@ schema-utils@^4.0.0, schema-utils@^4.0.1: ajv-formats "^2.1.1" ajv-keywords "^5.1.0" +schema-utils@2.7.0: + version "2.7.0" + resolved "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz" + integrity sha512-0ilKFI6QQF5nxDZLFn2dMjvc4hjg/Wkg7rHd3jK6/A4a1Hl9VFdQWvgB1UMGoU94pad1P/8N7fMcEnLnSiju8A== + dependencies: + "@types/json-schema" "^7.0.4" + ajv "^6.12.2" + ajv-keywords "^3.4.1" + +"search-insights@>= 1 < 3": + version "2.17.2" + resolved "https://registry.npmjs.org/search-insights/-/search-insights-2.17.2.tgz" + integrity sha512-zFNpOpUO+tY2D85KrxJ+aqwnIfdEGi06UH2+xEb+Bp9Mwznmauqc9djbnBibJO5mpfUPPa8st6Sx65+vbeO45g== + section-matter@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz" @@ -7655,7 +7762,7 @@ source-map-support@~0.5.20: buffer-from "^1.0.0" source-map "^0.6.0" -source-map@^0.6.0, source-map@~0.6.0: +source-map@^0.6.0: version "0.6.1" resolved "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz" integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== @@ -7665,6 +7772,11 @@ source-map@^0.7.0: resolved "https://registry.npmjs.org/source-map/-/source-map-0.7.4.tgz" integrity sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA== +source-map@~0.6.0: + version "0.6.1" + resolved "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz" + integrity sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g== + space-separated-tokens@^2.0.0: version "2.0.2" resolved "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz" @@ -7703,22 +7815,45 @@ srcset@^4.0.0: resolved "https://registry.npmjs.org/srcset/-/srcset-4.0.0.tgz" integrity sha512-wvLeHgcVHKO8Sc/H/5lkGreJQVeYMm9rlmt8PuR1xE31rIuXhuzznUUqAt8MqLhB3MqJdFzlNAfpcWnxiFUcPw== -statuses@2.0.1: - version "2.0.1" - resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz" - integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== - "statuses@>= 1.4.0 < 2": version "1.5.0" resolved "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz" integrity sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA== +statuses@2.0.1: + version "2.0.1" + resolved "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz" + integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== + std-env@^3.0.1: version "3.5.0" resolved "https://registry.npmjs.org/std-env/-/std-env-3.5.0.tgz" integrity sha512-JGUEaALvL0Mf6JCfYnJOTcobY+Nc7sG/TemDRBqCA0wEr4DER7zDchaaixTlmOxAjG1uRJmX82EQcxwTQTkqVA== -string-width@^4.1.0, string-width@^4.2.0: +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== + dependencies: + safe-buffer "~5.2.0" + +string_decoder@~1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" + integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== + dependencies: + safe-buffer "~5.1.0" + +string-width@^4.1.0: + version "4.2.3" + resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +string-width@^4.2.0: version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -7736,20 +7871,6 @@ string-width@^5.0.1, string-width@^5.1.2: emoji-regex "^9.2.2" strip-ansi "^7.0.1" -string_decoder@^1.1.1: - version "1.3.0" - resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz" - integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== - dependencies: - safe-buffer "~5.2.0" - -string_decoder@~1.1.1: - version "1.1.1" - resolved "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz" - integrity sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg== - dependencies: - safe-buffer "~5.1.0" - stringify-entities@^4.0.0: version "4.0.4" resolved "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz" @@ -7980,6 +8101,11 @@ typedarray-to-buffer@^3.1.5: dependencies: is-typedarray "^1.0.0" +"typescript@>= 2.7", typescript@>=4.9.5: + version "5.2.2" + resolved "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz" + integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== + undici-types@~5.26.4: version "5.26.5" resolved "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz" @@ -8083,7 +8209,7 @@ universalify@^2.0.0: resolved "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz" integrity sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw== -unpipe@1.0.0, unpipe@~1.0.0: +unpipe@~1.0.0, unpipe@1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz" integrity sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ== @@ -8291,7 +8417,7 @@ webpack-sources@^3.2.3: resolved "https://registry.npmjs.org/webpack-sources/-/webpack-sources-3.2.3.tgz" integrity sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w== -webpack@^5.88.1: +"webpack@^4.0.0 || ^5.0.0", "webpack@^4.37.0 || ^5.0.0", webpack@^5.0.0, webpack@^5.1.0, webpack@^5.20.0, webpack@^5.88.1, "webpack@>= 4", "webpack@>=4.41.1 || 5.x", webpack@>=5, "webpack@3 || 4 || 5": version "5.95.0" resolved "https://registry.npmjs.org/webpack/-/webpack-5.95.0.tgz" integrity sha512-2t3XstrKULz41MNMBF+cJ97TyHdyQ8HCt//pqErqDvNjU9YQBnZxIHa11VXsi7F3mb5/aO2tuDxdeTPdU7xu9Q== @@ -8330,7 +8456,7 @@ webpackbar@^5.0.2: pretty-time "^1.1.0" std-env "^3.0.1" -websocket-driver@>=0.5.1, websocket-driver@^0.7.4: +websocket-driver@^0.7.4, websocket-driver@>=0.5.1: version "0.7.4" resolved "https://registry.npmjs.org/websocket-driver/-/websocket-driver-0.7.4.tgz" integrity sha512-b17KeDIQVjvb0ssuSDF2cYXSg2iztliJ4B9WdsuB6J952qCPKmnVq4DyW5motImXHDC1cBT/1UezrJVsKw5zjg== diff --git a/metrics/examples/python/python_ingest_simple/python_ingest_simple.yaml b/metrics/examples/python/python_ingest_simple/python_ingest_simple.yaml index 0bdda7e2..0964a92a 100644 --- a/metrics/examples/python/python_ingest_simple/python_ingest_simple.yaml +++ b/metrics/examples/python/python_ingest_simple/python_ingest_simple.yaml @@ -1,10 +1,10 @@ metric_batch: "python_ingest_simple" table_key: "metrics_python_ingest_simple" alert_methods: "email,slack" -ingest_cron_schedule: "*/5 * * * *" -train_cron_schedule: "*/10 * * * *" -score_cron_schedule: "*/6 * * * *" -alert_cron_schedule: "*/5 * * * *" +ingest_cron_schedule: "*/10 * * * *" +train_cron_schedule: "*/15 * * * *" +score_cron_schedule: "*/11 * * * *" +alert_cron_schedule: "*/12 * * * *" disable_llmalert: False # Enable threshold alerts diff --git a/scripts/README.md b/scripts/README.md index 4b6daf54..cfef6b01 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,21 +8,91 @@ The scripts in this directory provide helpful utilities for common administrativ ## Structure -### `sqlite/` -Contains SQLite-specific utility scripts for users running Anomstack with SQLite as their database backend. +### `kill_long_running_tasks.py` +Dagster utility for terminating or marking as failed any runs that exceed the configured timeout. + +**Features:** +- Uses the same timeout configuration as the timeout sensor +- Gracefully handles unreachable user code servers +- Provides detailed logging of termination actions +- Automatically marks stuck runs as failed when termination isn't possible + +**Usage:** +```bash +cd scripts/ +python kill_long_running_tasks.py +``` ### `posthog_example.py` Runs the PostHog metrics ingest function to ensure your PostHog credentials work. +### `sqlite/` +Contains SQLite-specific utility scripts for users running Anomstack with SQLite as their database backend: + +#### `create_index.py` +Automatically creates performance indexes on common metric columns (`metric_timestamp`, `metric_batch`, `metric_type`) for all tables in the SQLite database. + +**Usage:** +```bash +cd scripts/sqlite/ +python create_index.py +``` + +#### `list_tables.py` +Lists all tables in the SQLite database with their names and details. + +**Usage:** +```bash +cd scripts/sqlite/ +python list_tables.py +``` + +#### `list_indexes.py` +Lists all indexes in the SQLite database with their names and associated tables. + +**Usage:** +```bash +cd scripts/sqlite/ +python list_indexes.py +``` + +#### `qry.py` +Executes the SQL query from `qry.sql` file and returns results as a DataFrame. Useful for running ad-hoc SQL queries against your SQLite database. + +**Usage:** +```bash +cd scripts/sqlite/ +# Edit qry.sql with your query first +python qry.py +``` + +### `utils/` +Contains general utility scripts for system management and maintenance. + +### `utils/reset_docker.sh` +Comprehensive Docker reset utility with multiple cleanup levels: +- **Gentle**: Rebuild containers with fresh images (safest) +- **Medium**: Remove containers and networks, preserve data volumes +- **Nuclear**: Remove all data including volumes and local files +- **Full Nuclear**: Nuclear reset + complete Docker system cleanup + +Can be run interactively or with specific reset levels via Makefile targets. + ## Common Use Cases These scripts are typically used for: - **Database Setup**: Initialize databases and create required tables +- **Database Performance**: Create indexes to improve query performance (`create_index.py`) +- **Database Inspection**: View table and index structures (`list_tables.py`, `list_indexes.py`) - **Data Migration**: Move data between different storage backends +- **Data Analysis**: Run ad-hoc SQL queries for troubleshooting (`qry.py`) - **Maintenance**: Clean up old data, optimize performance +- **Task Management**: Terminate stuck or long-running Dagster jobs (`kill_long_running_tasks.py`) - **Development**: Setup development environments and test data - **Deployment**: Automate deployment tasks and configuration +- **System Reset**: Clean up Docker environments with various levels of data preservation (`reset_docker.sh`) +- **Credential Validation**: Test external service integrations (`posthog_example.py`) ## Usage @@ -36,6 +106,34 @@ cd scripts/ python script_name.py ``` +### Quick Examples + +**Database performance optimization:** +```bash +cd scripts/sqlite/ +python create_index.py # Create performance indexes +``` + +**Database inspection:** +```bash +cd scripts/sqlite/ +python list_tables.py # See all tables +python list_indexes.py # See all indexes +``` + +**Clean up stuck Dagster jobs:** +```bash +cd scripts/ +python kill_long_running_tasks.py +``` + +**Complete Docker environment reset:** +```bash +make reset-interactive # Interactive mode +# or +make reset-nuclear # Nuclear reset +``` + ## Database-Specific Scripts Different database backends may have specific requirements and utilities: diff --git a/scripts/reload_config.py b/scripts/reload_config.py new file mode 100755 index 00000000..0e7928d9 --- /dev/null +++ b/scripts/reload_config.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +""" +Reload Dagster configuration after making changes to YAML files or environment variables. + +This script helps users update configurations on the fly without restarting Docker containers. +""" + +import os +import sys +import json +import time +import requests +from pathlib import Path + +def get_dagster_graphql_url(): + """Get Dagster GraphQL URL from environment or default.""" + host = os.getenv('DAGSTER_HOST', 'localhost') + port = os.getenv('DAGSTER_PORT', '3000') + return f"http://{host}:{port}/graphql" + +def reload_code_location(location_name="anomstack_code"): + """Reload a specific code location in Dagster.""" + url = get_dagster_graphql_url() + + # GraphQL mutation to reload code location + mutation = """ + mutation ReloadRepositoryLocation($locationName: String!) { + reloadRepositoryLocation(repositoryLocationName: $locationName) { + __typename + ... on WorkspaceLocationEntry { + name + locationOrLoadError { + __typename + ... on RepositoryLocation { + name + repositories { + name + } + } + ... on PythonError { + message + stack + } + } + } + ... on ReloadNotSupported { + message + } + ... on RepositoryLocationNotFound { + message + } + } + } + """ + + variables = {"locationName": location_name} + + try: + print(f"πŸ”„ Attempting to reload code location '{location_name}'...") + response = requests.post( + url, + json={"query": mutation, "variables": variables}, + timeout=30 + ) + + if response.status_code != 200: + print(f"❌ HTTP Error {response.status_code}: {response.text}") + return False + + data = response.json() + + if "errors" in data: + print(f"❌ GraphQL Errors: {data['errors']}") + return False + + result = data["data"]["reloadRepositoryLocation"] + + if result["__typename"] == "WorkspaceLocationEntry": + print(f"βœ… Successfully reloaded code location '{location_name}'") + + # Show loaded repositories + location_data = result["locationOrLoadError"] + if location_data["__typename"] == "RepositoryLocation": + repos = location_data["repositories"] + print(f"πŸ“¦ Loaded {len(repos)} repositories:") + for repo in repos: + print(f" - {repo['name']}") + else: + print(f"⚠️ Location loaded but with errors: {location_data}") + + return True + + elif result["__typename"] == "ReloadNotSupported": + print(f"❌ Reload not supported: {result['message']}") + return False + + elif result["__typename"] == "RepositoryLocationNotFound": + print(f"❌ Location not found: {result['message']}") + return False + + else: + print(f"❌ Unexpected result type: {result}") + return False + + except requests.exceptions.ConnectionError: + print("❌ Cannot connect to Dagster. Is the webserver running on http://localhost:3000?") + return False + except requests.exceptions.Timeout: + print("❌ Request timed out. Dagster may be busy.") + return False + except Exception as e: + print(f"❌ Unexpected error: {e}") + return False + +def check_dagster_health(): + """Check if Dagster is accessible.""" + url = get_dagster_graphql_url().replace("/graphql", "/server_info") + try: + response = requests.get(url, timeout=5) + if response.status_code == 200: + print(f"βœ… Dagster webserver is accessible at {url}") + return True + else: + print(f"❌ Dagster webserver returned {response.status_code}") + return False + except Exception as e: + print(f"❌ Cannot reach Dagster webserver: {e}") + return False + +def validate_config(): + """Validate that configuration files are accessible.""" + metrics_dir = Path("./metrics") + + if not metrics_dir.exists(): + print(f"❌ Metrics directory not found at {metrics_dir.absolute()}") + return False + + defaults_file = metrics_dir / "defaults" / "defaults.yaml" + if not defaults_file.exists(): + print(f"❌ Defaults file not found at {defaults_file.absolute()}") + return False + + print(f"βœ… Configuration files accessible at {metrics_dir.absolute()}") + return True + +def main(): + """Main function to reload configuration.""" + print("πŸ”§ Anomstack Configuration Reloader") + print("=" * 50) + + # Check prerequisites + if not validate_config(): + print("\nπŸ’‘ Ensure you're running this script from the anomstack root directory.") + sys.exit(1) + + if not check_dagster_health(): + print("\nπŸ’‘ Ensure Docker containers are running:") + print(" docker compose up -d") + sys.exit(1) + + # Reload configuration + print("\nπŸ”„ Reloading configuration...") + success = reload_code_location("anomstack_code") + + if success: + print("\nπŸŽ‰ Configuration reload complete!") + print("\nπŸ“ Your changes should now be active:") + print(" β€’ YAML configuration updates") + print(" β€’ Environment variable changes (if containers restarted)") + print(" β€’ New metric batches") + print(" β€’ Modified SQL queries or Python functions") + print("\n🌐 Check the Dagster UI: http://localhost:3000") + else: + print("\nπŸ’₯ Configuration reload failed!") + print("\nπŸ”§ Try these troubleshooting steps:") + print(" 1. Check Docker containers: docker compose ps") + print(" 2. Check Dagster logs: docker compose logs anomstack_code") + print(" 3. Restart containers if needed: docker compose restart") + + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/utils/README.md b/scripts/utils/README.md new file mode 100644 index 00000000..0355ac08 --- /dev/null +++ b/scripts/utils/README.md @@ -0,0 +1,158 @@ +# Utilities Directory + +This directory contains general utility scripts for system management and maintenance. + +## Scripts + +### `reset_docker.sh` + +Comprehensive Docker reset utility for Anomstack with multiple cleanup levels and safety checks. + +#### Features + +- πŸ”„ **Interactive Mode**: Guided menu with clear options +- πŸ“Š **Data Size Display**: Shows current disk usage before cleanup +- ⚠️ **Safety Confirmations**: Multiple confirmations for destructive operations +- 🎨 **Colorful Output**: Clear visual feedback with emojis and colors +- πŸ›‘οΈ **Fallback Handling**: Works even if Makefile targets fail + +#### Reset Levels + +##### 1. Gentle Reset (`gentle`) +**Safest option** - Rebuilds containers with fresh images +- βœ… **Preserves**: All data, volumes, local files +- πŸ”„ **Rebuilds**: Docker images from scratch +- 🎯 **Use when**: You want fresh containers but keep all data + +```bash +# Via Makefile +make reset-gentle + +# Direct script usage +./scripts/utils/reset_docker.sh gentle +``` + +##### 2. Medium Reset (`medium`) +**Balanced option** - Removes containers but preserves data +- βœ… **Preserves**: Docker volumes, local data files +- πŸ—‘οΈ **Removes**: Containers, networks +- 🎯 **Use when**: Container issues but want to keep data + +```bash +# Via Makefile +make reset-medium + +# Direct script usage +./scripts/utils/reset_docker.sh medium +``` + +##### 3. Nuclear Reset (`nuclear`) +**Destructive option** - Removes all data and containers +- ⚠️ **Removes**: Docker volumes, all local data (63GB+ in your case) +- πŸ—‘οΈ **Cleans**: tmp/, storage/, history/, logs/, telemetry/ +- 🎯 **Use when**: You want a completely fresh start + +```bash +# Via Makefile +make reset-nuclear + +# Direct script usage +./scripts/utils/reset_docker.sh nuclear +``` + +##### 4. Full Nuclear Reset (`full-nuclear`) +**Maximum cleanup** - Nuclear reset + full Docker system cleanup +- ⚠️ **Removes**: Everything from nuclear reset +- πŸ—‘οΈ **Plus**: All unused Docker images, networks, build cache +- πŸ’Ύ **Frees**: Maximum possible disk space +- 🎯 **Use when**: You want absolute maximum cleanup + +```bash +# Via Makefile +make reset-full-nuclear + +# Direct script usage +./scripts/utils/reset_docker.sh full-nuclear +``` + +#### Interactive Mode + +Run without arguments for a guided experience: + +```bash +# Via Makefile (recommended) +make reset-interactive + +# Direct script usage +./scripts/utils/reset_docker.sh +``` + +**Example Interactive Session:** +``` +🐳 Anomstack Docker Reset Tool + +ℹ️ Select reset level: +1) πŸ”„ Gentle - Rebuild containers (safest) +2) 🧹 Medium - Remove containers, keep data +3) ☒️ Nuclear - Remove all data and containers +4) πŸ’₯ Full Nuclear - Nuclear + full Docker cleanup +5) ❌ Cancel + +Choose option [1-5]: +``` + +#### What's Always Preserved + +Regardless of reset level, these are **always preserved**: +- πŸ’Ύ Source code (`anomstack/`, `dashboard/`, `metrics/`) +- βš™οΈ Configuration files (`.env`, `docker-compose.yaml`, etc.) +- πŸ› οΈ Git repository and history +- πŸ“ Documentation and README files + +#### Safety Features + +- **Double Confirmation**: Destructive operations require two confirmations +- **Data Size Display**: Shows exactly how much data will be deleted +- **Detailed Warnings**: Clear indication of what will be lost +- **Graceful Fallback**: Works even if Makefile targets are broken +- **Error Handling**: Continues operation even if individual steps fail + +#### Usage Tips + +1. **Start Conservative**: Begin with `gentle` reset if unsure +2. **Check Data Size**: Script shows current usage before cleanup +3. **Use Interactive Mode**: Provides the best guidance for choosing reset level +4. **Free Disk Space**: `full-nuclear` provides maximum space recovery +5. **Fresh Installation Feel**: `nuclear` or `full-nuclear` gives you a brand new setup + +#### Examples + +**Quick gentle reset:** +```bash +make reset-gentle +``` + +**Interactive guided reset:** +```bash +make reset-interactive +``` + +**Nuclear reset for maximum cleanup (your 63GB case):** +```bash +make reset-nuclear +``` + +**Maximum possible disk space recovery:** +```bash +make reset-full-nuclear +``` + +#### Script Output + +The script provides colorful, informative output: +- πŸ”΅ **Info**: General information and progress +- 🟒 **Success**: Completed operations +- 🟑 **Warning**: Important notices and confirmations +- πŸ”΄ **Error**: Critical warnings about data loss + +This makes it easy to understand what's happening at each step of the reset process. \ No newline at end of file diff --git a/scripts/utils/cleanup_dagster_storage.sh b/scripts/utils/cleanup_dagster_storage.sh new file mode 100755 index 00000000..11bc5fdc --- /dev/null +++ b/scripts/utils/cleanup_dagster_storage.sh @@ -0,0 +1,485 @@ +#!/bin/bash +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Function to print colored output +print_info() { echo -e "${BLUE}ℹ️ $1${NC}"; } +print_success() { echo -e "${GREEN}βœ… $1${NC}"; } +print_warning() { echo -e "${YELLOW}⚠️ $1${NC}"; } +print_error() { echo -e "${RED}❌ $1${NC}"; } + +# Function to get user confirmation +confirm() { + local message="$1" + local default="${2:-n}" + + if [[ "$default" == "y" ]]; then + prompt="[Y/n]" + else + prompt="[y/N]" + fi + + read -p "$(echo -e "${YELLOW}$message $prompt${NC} ")" -n 1 -r + echo + + if [[ "$default" == "y" ]]; then + [[ $REPLY =~ ^[Nn]$ ]] && return 1 || return 0 + else + [[ $REPLY =~ ^[Yy]$ ]] && return 0 || return 1 + fi +} + +# Function to analyze current storage +analyze_storage() { + print_info "πŸ“Š Analyzing Dagster storage usage..." + + cd "$PROJECT_ROOT" + + # Count run directories + run_dirs=0 + if [ -d "tmp" ]; then + run_dirs=$(find tmp/ -maxdepth 1 -type d -name "*-*-*-*-*" 2>/dev/null | wc -l | tr -d ' ') + fi + + # Calculate sizes + tmp_size="0B" + storage_size="0B" + history_size="0B" + + [ -d "tmp" ] && tmp_size=$(du -sh tmp/ 2>/dev/null | cut -f1 || echo "0B") + [ -d "dagster_home/storage" ] && storage_size=$(du -sh dagster_home/storage/ 2>/dev/null | cut -f1 || echo "0B") + [ -d "dagster_home/history" ] && history_size=$(du -sh dagster_home/history/ 2>/dev/null | cut -f1 || echo "0B") + + print_info "Current Dagster storage usage:" + echo -e " πŸ—‚οΈ Run directories: ${run_dirs}" + echo -e " πŸ“ tmp/ directory: ${tmp_size}" + echo -e " πŸ—„οΈ dagster_home/storage: ${storage_size}" + echo -e " πŸ“š dagster_home/history: ${history_size}" + + # Estimate potential savings + if [ "$run_dirs" -gt 1000 ]; then + print_warning "🚨 EXCESSIVE RUN DIRECTORIES DETECTED!" + print_warning "You have ${run_dirs} run directories - this will cause disk space issues!" + echo + fi +} + +# Function to show cleanup options +show_cleanup_options() { + print_info "🧹 Cleanup Options Available:" + echo + echo "1. πŸ”§ Minimal Cleanup - Remove old compute logs only" + echo " β€’ Cleans compute logs older than 7 days" + echo " β€’ Preserves all run metadata and artifacts" + echo " β€’ Safe for production use" + echo + echo "2. 🧹 Standard Cleanup - Remove old runs and logs" + echo " β€’ Removes runs older than 30 days" + echo " β€’ Cleans associated compute logs and artifacts" + echo " β€’ Keeps recent runs for debugging" + echo + echo "3. πŸ”₯ Aggressive Cleanup - Remove most old data" + echo " β€’ Removes runs older than 7 days" + echo " β€’ Aggressive log cleanup" + echo " β€’ Only keeps very recent data" + echo + echo "4. ☒️ Nuclear Cleanup - Remove almost everything" + echo " β€’ Removes all but the last 24 hours of runs" + echo " β€’ Clears most storage directories" + echo " β€’ Use only if disk space is critical" + echo + echo "5. πŸ› οΈ CLI-based cleanup using Dagster commands" + echo " β€’ Uses built-in 'dagster run wipe' commands" + echo " β€’ Most thorough database cleanup" + echo " β€’ Recommended for severe cases" +} + +# Cleanup functions +cleanup_minimal() { + print_info "πŸ”§ Performing minimal cleanup..." + + cd "$PROJECT_ROOT" + + # Clean old compute logs (older than 7 days) + if [ -d "tmp" ]; then + print_info "Cleaning compute logs older than 7 days..." + find tmp/ -name "*.log" -mtime +7 -delete 2>/dev/null || true + find tmp/ -name "compute_logs" -type d -exec find {} -name "*.log" -mtime +7 -delete \; 2>/dev/null || true + fi + + # Clean old log files in dagster_home + if [ -d "dagster_home/logs" ]; then + find dagster_home/logs/ -name "*.log" -mtime +7 -delete 2>/dev/null || true + fi + + print_success "Minimal cleanup completed" +} + +cleanup_standard() { + print_info "🧹 Performing standard cleanup..." + + cd "$PROJECT_ROOT" + + if [ -d "tmp" ]; then + # Remove run directories older than 30 days + print_info "Removing run directories older than 30 days..." + find tmp/ -maxdepth 1 -type d -name "*-*-*-*-*" -mtime +30 -exec rm -rf {} \; 2>/dev/null || true + + # Clean compute logs older than 14 days + find tmp/ -name "*.log" -mtime +14 -delete 2>/dev/null || true + find tmp/ -name "compute_logs" -type d -exec find {} -name "*.log" -mtime +14 -delete \; 2>/dev/null || true + fi + + # Clean old storage files + if [ -d "dagster_home/storage" ]; then + find dagster_home/storage/ -type f -mtime +30 -delete 2>/dev/null || true + fi + + print_success "Standard cleanup completed" +} + +cleanup_aggressive() { + print_info "πŸ”₯ Performing aggressive cleanup..." + + cd "$PROJECT_ROOT" + + if [ -d "tmp" ]; then + # Count total directories to clean for progress tracking + print_info "πŸ“Š Counting directories to clean..." + total_dirs=$(find tmp/ -maxdepth 1 -type d -name "*-*-*-*-*" -mtime +7 2>/dev/null | wc -l | tr -d ' ') + + if [ "$total_dirs" -gt 0 ]; then + print_info "πŸ—‚οΈ Found $total_dirs run directories older than 7 days to clean" + print_info "⏱️ Starting cleanup (this may take several minutes)..." + + # Enhanced removal with progress tracking + count=0 + batch_size=100 + start_time=$(date +%s) + + # Use find with -print0 and process in batches for better performance + find tmp/ -maxdepth 1 -type d -name "*-*-*-*-*" -mtime +7 -print0 2>/dev/null | \ + while IFS= read -r -d '' dir; do + # Remove directory + rm -rf "$dir" 2>/dev/null || true + + # Increment counter + count=$((count + 1)) + + # Show progress every batch_size directories + if [ $((count % batch_size)) -eq 0 ] || [ "$count" -eq "$total_dirs" ]; then + # Calculate progress percentage + progress=$((count * 100 / total_dirs)) + + # Calculate elapsed time and estimate remaining + current_time=$(date +%s) + elapsed=$((current_time - start_time)) + + if [ "$count" -gt 0 ] && [ "$elapsed" -gt 0 ]; then + rate=$((count * 60 / elapsed)) # dirs per minute + remaining_dirs=$((total_dirs - count)) + + if [ "$rate" -gt 0 ]; then + eta_minutes=$((remaining_dirs / rate)) + print_info "πŸ—‚οΈ Progress: $count/$total_dirs (${progress}%) | Rate: ${rate}/min | ETA: ${eta_minutes}min" + else + print_info "πŸ—‚οΈ Progress: $count/$total_dirs (${progress}%) | Elapsed: ${elapsed}s" + fi + else + print_info "πŸ—‚οΈ Progress: $count/$total_dirs (${progress}%)" + fi + + # Show current directory being processed (truncated for readability) + dir_name=$(basename "$dir") + if [ ${#dir_name} -gt 50 ]; then + dir_display="${dir_name:0:25}...${dir_name: -20}" + else + dir_display="$dir_name" + fi + print_info "πŸ“ Processing: $dir_display" + fi + done + + print_success "βœ… Removed $total_dirs run directories" + else + print_info "πŸŽ‰ No run directories older than 7 days found" + fi + + # Clean all old logs with progress + print_info "🧹 Cleaning old log files..." + log_count=$(find tmp/ -name "*.log" -mtime +3 2>/dev/null | wc -l | tr -d ' ') + if [ "$log_count" -gt 0 ]; then + print_info "πŸ“„ Found $log_count log files to clean" + find tmp/ -name "*.log" -mtime +3 -delete 2>/dev/null || true + print_success "βœ… Cleaned $log_count log files" + fi + + # Clean compute logs with progress + compute_log_count=$(find tmp/ -name "compute_logs" -type d -exec find {} -name "*.log" -mtime +3 \; 2>/dev/null | wc -l | tr -d ' ') + if [ "$compute_log_count" -gt 0 ]; then + print_info "πŸ’Ύ Found $compute_log_count compute log files to clean" + find tmp/ -name "compute_logs" -type d -exec find {} -name "*.log" -mtime +3 -delete \; 2>/dev/null || true + print_success "βœ… Cleaned $compute_log_count compute log files" + fi + fi + + # Aggressive storage cleanup with progress + if [ -d "dagster_home/storage" ]; then + print_info "πŸ—„οΈ Cleaning dagster_home/storage..." + storage_files=$(find dagster_home/storage/ -type f -mtime +7 2>/dev/null | wc -l | tr -d ' ') + if [ "$storage_files" -gt 0 ]; then + print_info "πŸ“¦ Found $storage_files storage files to clean" + find dagster_home/storage/ -type f -mtime +7 -delete 2>/dev/null || true + print_success "βœ… Cleaned $storage_files storage files" + fi + fi + + # Clean old history with progress + if [ -d "dagster_home/history" ]; then + print_info "πŸ“š Cleaning dagster_home/history..." + history_files=$(find dagster_home/history/ -type f -mtime +7 2>/dev/null | wc -l | tr -d ' ') + if [ "$history_files" -gt 0 ]; then + print_info "πŸ“‹ Found $history_files history files to clean" + find dagster_home/history/ -type f -mtime +7 -delete 2>/dev/null || true + print_success "βœ… Cleaned $history_files history files" + fi + fi + + # Final cleanup summary + print_info "πŸ“Š Final storage analysis..." + final_dirs=$(find tmp/ -maxdepth 1 -type d 2>/dev/null | wc -l | tr -d ' ') + final_size=$(du -sh tmp/ 2>/dev/null | cut -f1 || echo "0B") + + print_success "πŸ”₯ Aggressive cleanup completed!" + print_info "πŸ“ˆ Remaining directories: $final_dirs" + print_info "πŸ’Ύ Remaining tmp/ size: $final_size" +} + +cleanup_nuclear() { + print_info "☒️ Performing nuclear cleanup..." + + cd "$PROJECT_ROOT" + + if [ -d "tmp" ]; then + # Remove run directories older than 1 day + print_info "Removing run directories older than 24 hours..." + find tmp/ -maxdepth 1 -type d -name "*-*-*-*-*" -mtime +1 -exec rm -rf {} \; 2>/dev/null || true + + # Remove all logs older than 1 day + find tmp/ -name "*.log" -mtime +1 -delete 2>/dev/null || true + fi + + # Nuclear storage cleanup + if [ -d "dagster_home/storage" ]; then + find dagster_home/storage/ -type f -mtime +1 -delete 2>/dev/null || true + fi + + if [ -d "dagster_home/history" ]; then + find dagster_home/history/ -type f -mtime +1 -delete 2>/dev/null || true + fi + + # Clean up empty directories + find tmp/ -type d -empty -delete 2>/dev/null || true + find dagster_home/storage/ -type d -empty -delete 2>/dev/null || true + find dagster_home/history/ -type d -empty -delete 2>/dev/null || true + + print_success "Nuclear cleanup completed" +} + +cleanup_cli_based() { + print_info "πŸ› οΈ Performing CLI-based cleanup using Dagster commands..." + + cd "$PROJECT_ROOT" + + print_warning "This will use Dagster's built-in cleanup commands" + print_warning "This requires a running Dagster instance" + + if confirm "Do you want to wipe all run history from the database?"; then + print_info "Wiping run history..." + # Note: This requires DAGSTER_HOME to be set correctly + export DAGSTER_HOME="$PROJECT_ROOT/dagster_home" + + # Try to wipe runs (requires instance to be accessible) + if command -v dagster >/dev/null 2>&1; then + dagster run wipe --yes 2>/dev/null && print_success "Run history wiped" || print_warning "Could not wipe runs (instance may not be running)" + else + print_warning "Dagster CLI not available" + fi + + # Clean local storage manually as backup + cleanup_aggressive + else + print_info "CLI cleanup cancelled" + fi +} + +# Show current configuration status +show_config_status() { + print_info "πŸ”§ Current Dagster Configuration Status" + + cd "$PROJECT_ROOT" + + # Check if retention policies are configured + if grep -q "retention:" dagster_docker.yaml 2>/dev/null; then + print_success "βœ… Retention policies are configured in dagster_docker.yaml" + else + print_warning "⚠️ No retention policies found in dagster_docker.yaml" + print_info " Add retention policies to prevent storage buildup" + fi + + # Check concurrent runs setting + if grep -q "max_concurrent_runs: 10" dagster_docker.yaml 2>/dev/null; then + print_success "βœ… Concurrent runs limited to 10 (good for storage)" + elif grep -q "max_concurrent_runs:" dagster_docker.yaml 2>/dev/null; then + concurrent_runs=$(grep "max_concurrent_runs:" dagster_docker.yaml | head -1 | awk '{print $2}') + if [ "$concurrent_runs" -gt 15 ]; then + print_warning "⚠️ High concurrent runs ($concurrent_runs) may cause storage issues" + else + print_info "ℹ️ Concurrent runs set to $concurrent_runs" + fi + fi + + # Check run monitoring + if grep -q "run_monitoring:" dagster_docker.yaml 2>/dev/null; then + print_success "βœ… Run monitoring is configured" + else + print_warning "⚠️ Run monitoring not configured - may miss stuck runs" + fi +} + +# Main menu +main_menu() { + while true; do + clear + print_info "πŸ—‚οΈ Dagster Storage Cleanup Tool" + echo + + analyze_storage + echo + + show_cleanup_options + echo + + echo "Additional Options:" + echo "6. πŸ”§ Show Dagster configuration status" + echo "7. πŸ“Š Re-analyze storage (refresh)" + echo "8. ❌ Exit" + echo + + read -p "Choose cleanup option [1-8]: " -n 1 -r choice + echo + + case $choice in + 1) + cleanup_minimal + ;; + 2) + if confirm "Proceed with standard cleanup? (removes data older than 30 days)"; then + cleanup_standard + fi + ;; + 3) + if confirm "Proceed with aggressive cleanup? (removes data older than 7 days)"; then + cleanup_aggressive + fi + ;; + 4) + print_error "⚠️ NUCLEAR CLEANUP WARNING" + print_error "This will remove almost all run data (except last 24 hours)!" + if confirm "Are you absolutely sure?" "n"; then + if confirm "Really proceed with nuclear cleanup?" "n"; then + cleanup_nuclear + fi + fi + ;; + 5) + cleanup_cli_based + ;; + 6) + show_config_status + ;; + 7) + continue # Will re-analyze at top of loop + ;; + 8) + print_info "Exiting cleanup tool" + exit 0 + ;; + *) + print_error "Invalid option: $choice" + ;; + esac + + echo + print_info "Press any key to continue..." + read -n 1 -s + done +} + +# Help function +usage() { + echo "Usage: $0 [minimal|standard|aggressive|nuclear|cli|status|menu]" + echo + echo "Options:" + echo " minimal - Clean old compute logs only" + echo " standard - Remove runs older than 30 days" + echo " aggressive - Remove runs older than 7 days" + echo " nuclear - Remove almost all data" + echo " cli - Use Dagster CLI cleanup commands" + echo " status - Show Dagster configuration status" + echo " menu - Interactive menu (default)" + echo +} + +# Main execution +case "${1:-menu}" in + "minimal") + analyze_storage + cleanup_minimal + ;; + "standard") + analyze_storage + if confirm "Proceed with standard cleanup?"; then + cleanup_standard + fi + ;; + "aggressive") + analyze_storage + if confirm "Proceed with aggressive cleanup?"; then + cleanup_aggressive + fi + ;; + "nuclear") + analyze_storage + if confirm "Proceed with nuclear cleanup?" "n"; then + cleanup_nuclear + fi + ;; + "cli") + cleanup_cli_based + ;; + "status") + show_config_status + ;; + "menu"|"") + main_menu + ;; + "-h"|"--help") + usage + ;; + *) + print_error "Invalid option: $1" + usage + exit 1 + ;; +esac \ No newline at end of file diff --git a/scripts/utils/reset_docker.sh b/scripts/utils/reset_docker.sh new file mode 100755 index 00000000..c548578d --- /dev/null +++ b/scripts/utils/reset_docker.sh @@ -0,0 +1,283 @@ +#!/bin/bash +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Function to print colored output +print_info() { echo -e "${BLUE}ℹ️ $1${NC}"; } +print_success() { echo -e "${GREEN}βœ… $1${NC}"; } +print_warning() { echo -e "${YELLOW}⚠️ $1${NC}"; } +print_error() { echo -e "${RED}❌ $1${NC}"; } + +# Function to get user confirmation +confirm() { + local message="$1" + local default="${2:-n}" + + if [[ "$default" == "y" ]]; then + prompt="[Y/n]" + else + prompt="[y/N]" + fi + + read -p "$(echo -e "${YELLOW}$message $prompt${NC} ")" -n 1 -r + echo + + if [[ "$default" == "y" ]]; then + [[ $REPLY =~ ^[Nn]$ ]] && return 1 || return 0 + else + [[ $REPLY =~ ^[Yy]$ ]] && return 0 || return 1 + fi +} + +# Function to show data sizes +show_data_sizes() { + print_info "Current data usage:" + + if [ -d "$PROJECT_ROOT/tmp" ] && [ "$(ls -A "$PROJECT_ROOT/tmp" 2>/dev/null)" ]; then + echo -e " πŸ“ tmp/ directory: $(du -sh "$PROJECT_ROOT/tmp" 2>/dev/null | cut -f1 || echo "empty")" + fi + + if [ -d "$PROJECT_ROOT/dagster_home/storage" ] && [ "$(ls -A "$PROJECT_ROOT/dagster_home/storage" 2>/dev/null)" ]; then + echo -e " πŸ—„οΈ dagster_home/storage: $(du -sh "$PROJECT_ROOT/dagster_home/storage" 2>/dev/null | cut -f1 || echo "empty")" + fi + + if [ -d "$PROJECT_ROOT/dagster_home/history" ] && [ "$(ls -A "$PROJECT_ROOT/dagster_home/history" 2>/dev/null)" ]; then + echo -e " πŸ“š dagster_home/history: $(du -sh "$PROJECT_ROOT/dagster_home/history" 2>/dev/null | cut -f1 || echo "empty")" + fi + + # Show Docker volumes + volumes=$(docker volume ls -q | grep -E "anomstack" 2>/dev/null || echo "") + if [ -n "$volumes" ]; then + print_info "Docker volumes:" + echo "$volumes" | while read -r volume; do + echo -e " 🐳 $volume" + done + fi + echo +} + +# Function to show what will be preserved +show_preserved() { + print_success "What will be PRESERVED:" + echo -e " πŸ’Ύ Source code (anomstack/, dashboard/, metrics/)" + echo -e " βš™οΈ Configuration files (.env, docker-compose files)" + echo -e " πŸ› οΈ Git repository and history" + echo +} + +# Reset level functions +gentle_reset() { + print_info "πŸ”„ GENTLE RESET: Rebuilding containers with fresh images" + show_preserved + + if confirm "Proceed with gentle reset?"; then + cd "$PROJECT_ROOT" + print_info "Stopping containers..." + make docker-down + + print_info "Building fresh images..." + make docker-dev-build --no-cache + + print_info "Starting containers..." + make docker-dev + + print_success "Gentle reset complete! πŸŽ‰" + else + print_info "Reset cancelled." + fi +} + +medium_reset() { + print_info "🧹 MEDIUM RESET: Remove containers and networks, keep data volumes" + + show_data_sizes + show_preserved + print_success "Docker volumes will be PRESERVED (your data is safe)" + echo + + if confirm "Proceed with medium reset?"; then + cd "$PROJECT_ROOT" + print_info "Removing containers and networks..." + make docker-rm || docker-compose down --remove-orphans + + print_info "Building fresh images..." + make docker-dev-build + + print_info "Starting fresh containers..." + make docker-dev + + print_success "Medium reset complete! πŸŽ‰" + else + print_info "Reset cancelled." + fi +} + +nuclear_reset() { + print_info "☒️ NUCLEAR RESET: Remove containers, volumes, and local data" + + show_data_sizes + show_preserved + print_error "⚠️ ALL DATA WILL BE LOST:" + echo -e " πŸ—‘οΈ All Docker volumes (metrics, database)" + echo -e " πŸ—‘οΈ All local run data (tmp/, storage/, history/)" + echo -e " πŸ—‘οΈ All Dagster run history and logs" + echo + + if confirm "Are you absolutely sure you want to delete ALL data?" "n"; then + if confirm "This cannot be undone. Really proceed?" "n"; then + cd "$PROJECT_ROOT" + + print_info "Stopping and removing containers with volumes..." + make docker-prune || { + docker-compose down -v --remove-orphans + docker system prune -f + } + + print_info "Removing local data directories..." + [ -d "tmp" ] && rm -rf tmp/* && print_success "Cleared tmp/ directory" + [ -d "dagster_home/storage" ] && rm -rf dagster_home/storage/* && print_success "Cleared dagster storage" + [ -d "dagster_home/history" ] && rm -rf dagster_home/history/* && print_success "Cleared dagster history" + [ -d "dagster_home/.logs_queue" ] && rm -rf dagster_home/.logs_queue/* && print_success "Cleared logs queue" + [ -d "dagster_home/logs" ] && rm -rf dagster_home/logs/* && print_success "Cleared logs" + [ -d "dagster_home/.nux" ] && rm -rf dagster_home/.nux/* && print_success "Cleared nux cache" + [ -d "dagster_home/.telemetry" ] && rm -rf dagster_home/.telemetry/* && print_success "Cleared telemetry" + + print_info "Building fresh images..." + make docker-dev-build + + print_info "Starting completely fresh setup..." + make docker-dev + + print_success "Nuclear reset complete! Everything is fresh and clean πŸŽ‰" + print_info "Your setup is now like a brand new installation" + else + print_info "Reset cancelled." + fi + else + print_info "Reset cancelled." + fi +} + +full_nuclear_reset() { + print_info "πŸ’₯ FULL NUCLEAR RESET: Everything + Docker system cleanup" + + show_data_sizes + show_preserved + print_error "⚠️ EVERYTHING DOCKER WILL BE CLEANED:" + echo -e " πŸ—‘οΈ All data (same as nuclear reset)" + echo -e " πŸ—‘οΈ ALL unused Docker images, networks, build cache" + echo -e " πŸ—‘οΈ Docker system prune (frees maximum disk space)" + echo + + if confirm "Are you absolutely sure? This is the most destructive option!" "n"; then + if confirm "This will clean up ALL Docker resources. Really proceed?" "n"; then + cd "$PROJECT_ROOT" + + # Do nuclear reset first + print_info "Performing nuclear data cleanup..." + make docker-prune || { + docker-compose down -v --remove-orphans + docker system prune -f + } + + print_info "Removing local data directories..." + [ -d "tmp" ] && rm -rf tmp/* && print_success "Cleared tmp/ directory" + [ -d "dagster_home/storage" ] && rm -rf dagster_home/storage/* && print_success "Cleared dagster storage" + [ -d "dagster_home/history" ] && rm -rf dagster_home/history/* && print_success "Cleared dagster history" + [ -d "dagster_home/.logs_queue" ] && rm -rf dagster_home/.logs_queue/* && print_success "Cleared logs queue" + [ -d "dagster_home/logs" ] && rm -rf dagster_home/logs/* && print_success "Cleared logs" + + print_info "Performing full Docker system cleanup..." + docker system prune -a -f + + print_info "Building completely fresh images..." + make docker-dev-build + + print_info "Starting pristine setup..." + make docker-dev + + print_success "Full nuclear reset complete! Maximum cleanup achieved πŸŽ‰" + print_info "You've reclaimed maximum disk space and have a pristine setup" + else + print_info "Reset cancelled." + fi + else + print_info "Reset cancelled." + fi +} + +# Main script +usage() { + echo "Usage: $0 [gentle|medium|nuclear|full-nuclear]" + echo + echo "Reset levels:" + echo " gentle - Rebuild containers with fresh images (safest)" + echo " medium - Remove containers, keep data volumes" + echo " nuclear - Remove everything including local data" + echo " full-nuclear - Nuclear + full Docker system cleanup" + echo + echo "If no argument provided, interactive mode will guide you." +} + +main() { + cd "$PROJECT_ROOT" + + print_info "🐳 Anomstack Docker Reset Tool" + echo + + case "${1:-}" in + "gentle") + gentle_reset + ;; + "medium") + medium_reset + ;; + "nuclear") + nuclear_reset + ;; + "full-nuclear") + full_nuclear_reset + ;; + "-h"|"--help") + usage + ;; + "") + # Interactive mode + print_info "Select reset level:" + echo "1) πŸ”„ Gentle - Rebuild containers (safest)" + echo "2) 🧹 Medium - Remove containers, keep data" + echo "3) ☒️ Nuclear - Remove all data and containers" + echo "4) πŸ’₯ Full Nuclear - Nuclear + full Docker cleanup" + echo "5) ❌ Cancel" + echo + + read -p "Choose option [1-5]: " -n 1 -r choice + echo + + case $choice in + 1) gentle_reset ;; + 2) medium_reset ;; + 3) nuclear_reset ;; + 4) full_nuclear_reset ;; + 5|*) print_info "Cancelled." ;; + esac + ;; + *) + print_error "Invalid option: $1" + usage + exit 1 + ;; + esac +} + +main "$@" \ No newline at end of file