improve: Enhance fly.io deployment cache busting reliability

- Add stronger cache busting in deploy_fly.sh with timestamp+random values
- Make Dockerfile actually use CACHEBUST argument to invalidate Docker layers
- Improve Makefile fresh deployment targets to clear builder cache
- Add explicit dockerfile paths to avoid build confusion

These changes ensure --force-rebuild actually forces a complete rebuild,
preventing cached layers from causing deployment issues like we experienced
with the Dagster config fix.

Author: andrewm4894

Makefile

@@ -216,18 +216,24 @@ fly-deploy-development:
 fly-deploy-demo-fresh:
 	@echo "🧹 Cleaning local Docker cache to ensure fresh build..."
 	docker system prune -f --filter "until=1h"
+	@echo "🧹 Cleaning Docker builder cache..."
+	docker builder prune -f 2>/dev/null || true
 	./scripts/deployment/deploy_fly.sh --profile demo --force-rebuild
 
 # deploy with fresh build (clears local Docker cache first) - production profile
 fly-deploy-production-fresh:
 	@echo "🧹 Cleaning local Docker cache to ensure fresh build..."
 	docker system prune -f --filter "until=1h"
+	@echo "🧹 Cleaning Docker builder cache..."
+	docker builder prune -f 2>/dev/null || true
 	./scripts/deployment/deploy_fly.sh --profile production --force-rebuild
 
 # deploy with fresh build (clears local Docker cache first) - development profile
 fly-deploy-development-fresh:
 	@echo "🧹 Cleaning local Docker cache to ensure fresh build..."
 	docker system prune -f --filter "until=1h"
+	@echo "🧹 Cleaning Docker builder cache..."
+	docker builder prune -f 2>/dev/null || true
 	./scripts/deployment/deploy_fly.sh --profile development --force-rebuild
 
 # test fly.io build locally before deploying (helps catch issues early)

docker/Dockerfile.fly

@@ -3,6 +3,9 @@ FROM python:3.12-slim
 # Cache busting argument (set during build to force fresh layers)
 ARG CACHEBUST=1
 
+# Use CACHEBUST to invalidate cache when needed (this layer changes when CACHEBUST changes)
+RUN echo "Cache bust: $CACHEBUST" > /tmp/cachebust
+
 # Install system dependencies including nginx
 RUN apt-get update && apt-get install -y --no-install-recommends \
     git \

scripts/deployment/deploy_fly.sh

@@ -235,10 +235,23 @@ rm fly.toml.bak
 echo "🚀 Deploying application..."
 
 if [[ "$FORCE_REBUILD" == "true" ]]; then
+    # Generate unique cache busting value with timestamp + random
+    CACHEBUST_VALUE="$(date +%s)-$(openssl rand -hex 4 2>/dev/null || echo $RANDOM)"
     echo "🔄 Force rebuild enabled - using aggressive cache busting..."
-    fly deploy --no-cache --build-arg CACHEBUST="$(date +%s)" -a "$APP_NAME"
+    echo "🎯 Cache bust value: $CACHEBUST_VALUE"
+    
+    # Use multiple cache busting strategies:
+    # 1. --no-cache: Skip Docker layer cache
+    # 2. CACHEBUST build arg: Force rebuild of layers that use it  
+    # 3. --dockerfile: Explicit dockerfile path to avoid confusion
+    fly deploy \
+        --no-cache \
+        --build-arg CACHEBUST="$CACHEBUST_VALUE" \
+        --dockerfile docker/Dockerfile.fly \
+        -a "$APP_NAME"
 else
-    fly deploy --no-cache -a "$APP_NAME"
+    echo "⚡ Standard deployment (with caching)..."
+    fly deploy --dockerfile docker/Dockerfile.fly -a "$APP_NAME"
 fi
 
 # Show the status