Add table_key to metric configuration files and create scripts for indexing and listing tables

andrewm4894 · andrewm4894 · commit 6cec2a03c8df · 2025-02-23T20:47:06.000+01:00
diff --git a/metrics/examples/coindesk/coindesk.yaml b/metrics/examples/coindesk/coindesk.yaml
@@ -1,4 +1,5 @@
 metric_batch: "coindesk"
+table_key: "metrics_coindesk"
 ingest_cron_schedule: "*/5 * * * *"
 train_cron_schedule: "*/180 * * * *"
 score_cron_schedule: "*/10 * * * *"
diff --git a/metrics/examples/github/github.yaml b/metrics/examples/github/github.yaml
@@ -1,4 +1,5 @@
 metric_batch: "github"
+table_key: "metrics_github"
 ingest_cron_schedule: "*/15 * * * *"
 train_cron_schedule: "*/180 * * * *"
 score_cron_schedule: "*/30 * * * *"
diff --git a/metrics/examples/hackernews/hn_top_stories_scores.yaml b/metrics/examples/hackernews/hn_top_stories_scores.yaml
@@ -1,4 +1,5 @@
 metric_batch: "hn_top_stories_scores"
+table_key: "metrics_hackernews"
 ingest_cron_schedule: "45 6 * * *"
 train_cron_schedule: "15 8 * * *"
 score_cron_schedule: "15 7 * * *"
diff --git a/metrics/examples/netdata/netdata.yaml b/metrics/examples/netdata/netdata.yaml
@@ -1,4 +1,5 @@
 metric_batch: "netdata"
+table_key: "metrics_netdata"
 ingest_cron_schedule: "*/10 * * * *"
 train_cron_schedule: "*/180 * * * *"
 score_cron_schedule: "*/40 * * * *"
diff --git a/metrics/examples/netdata_httpcheck/netdata_httpcheck.yaml b/metrics/examples/netdata_httpcheck/netdata_httpcheck.yaml
@@ -1,4 +1,5 @@
 metric_batch: "netdata_httpcheck"
+table_key: "metrics_netdata_httpcheck"
 ingest_cron_schedule: "*/10 * * * *"
 train_cron_schedule: "*/180 * * * *"
 score_cron_schedule: "*/40 * * * *"
diff --git a/metrics/examples/prometheus/prometheus.yaml b/metrics/examples/prometheus/prometheus.yaml
@@ -1,4 +1,5 @@
 metric_batch: "prometheus"
+table_key: "metrics_prometheus"
 ingest_cron_schedule: "*/5 * * * *"
 train_cron_schedule: "*/180 * * * *"
 score_cron_schedule: "*/10 * * * *"
diff --git a/metrics/examples/python/python_ingest_simple/python_ingest_simple.yaml b/metrics/examples/python/python_ingest_simple/python_ingest_simple.yaml
@@ -1,4 +1,5 @@
 metric_batch: "python_ingest_simple"
+table_key: "metrics_python_ingest_simple"
 alert_methods: "email,slack"
 ingest_cron_schedule: "*/2 * * * *"
 train_cron_schedule: "*/4 * * * *"
diff --git a/metrics/examples/tomtom/tomtom.yaml b/metrics/examples/tomtom/tomtom.yaml
@@ -1,4 +1,5 @@
 metric_batch: "tomtom"
+table_key: "metrics_tomtom"
 ingest_cron_schedule: "45 10 * * *"
 train_cron_schedule: "15 11 * * *"
 score_cron_schedule: "15 10 * * *"
diff --git a/metrics/examples/weather/weather.yaml b/metrics/examples/weather/weather.yaml
@@ -1,4 +1,5 @@
 metric_batch: "weather"
+table_key: "metrics_weather"
 ingest_cron_schedule: "45 14 * * *"
 train_cron_schedule: "15 15 * * *"
 score_cron_schedule: "15 14 * * *"
diff --git a/metrics/examples/yfinance/yfinance.yaml b/metrics/examples/yfinance/yfinance.yaml
@@ -1,4 +1,5 @@
 metric_batch: "yfinance"
+table_key: "metrics_yfinance"
 ingest_cron_schedule: "45 18 * * *"
 train_cron_schedule: "15 20 * * *"
 score_cron_schedule: "15 19 * * *"
diff --git a/scripts/sqlite/create_index.py b/scripts/sqlite/create_index.py
@@ -0,0 +1,41 @@
+from anomstack.external.sqlite import run_sql_sqlite
+
+
+def add_indexes_to_metrics():
+    # Get the list of tables
+    tables_query = "SELECT name FROM sqlite_master WHERE type='table';"
+    tables = run_sql_sqlite(tables_query)
+    
+    if not tables:
+        print("No tables found.")
+        return
+
+    for table_row in tables:
+        table = table_row[0]  # Extract table name from row tuple
+        
+        # Check if the table contains 'metric_timestamp', 'metric_batch', or 'metric_type'
+        columns_query = f"PRAGMA table_info({table});"
+        columns = run_sql_sqlite(columns_query)
+        
+        column_names = [col[1] for col in columns]  # Extract column names
+
+        # Create index on metric_timestamp if it exists
+        if "metric_timestamp" in column_names:
+            index_query = f"CREATE INDEX IF NOT EXISTS idx_{table}_metric_timestamp ON {table} (metric_timestamp);"
+            run_sql_sqlite(index_query)
+            print(f"Index created for metric_timestamp in table: {table}")
+
+        # Create index on metric_batch if it exists
+        if "metric_batch" in column_names:
+            index_query = f"CREATE INDEX IF NOT EXISTS idx_{table}_metric_batch ON {table} (metric_batch);"
+            run_sql_sqlite(index_query)
+            print(f"Index created for metric_batch in table: {table}")
+
+        # Create index on metric_type if it exists
+        if "metric_type" in column_names:
+            index_query = f"CREATE INDEX IF NOT EXISTS idx_{table}_metric_type ON {table} (metric_type);"
+            run_sql_sqlite(index_query)
+            print(f"Index created for metric_type in table: {table}")
+
+# Run the function
+add_indexes_to_metrics()
diff --git a/scripts/sqlite/list_tables.py b/scripts/sqlite/list_tables.py
@@ -0,0 +1,24 @@
+from anomstack.external.sqlite.sqlite import run_sql_sqlite
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+def list_all_tables():
+    # Query to get all table names from sqlite_master
+    tables_query = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;"
+    
+    # Execute the query
+    tables = run_sql_sqlite(tables_query)
+    
+    if not tables:
+        print("No tables found.")
+        return
+
+    # Print the list of tables
+    print("Tables in the database:")
+    for table in tables:
+        print(f"- {table[0]}")  # Extract and print table name
+
+# Run the function
+list_all_tables()

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`metric_batch: "coindesk"`
	`2`	`+table_key: "metrics_coindesk"`
`2`	`3`	`ingest_cron_schedule: "/5 * * *"`
`3`	`4`	`train_cron_schedule: "/180 * * *"`
`4`	`5`	`score_cron_schedule: "/10 * * *"`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`metric_batch: "github"`
	`2`	`+table_key: "metrics_github"`
`2`	`3`	`ingest_cron_schedule: "/15 * * *"`
`3`	`4`	`train_cron_schedule: "/180 * * *"`
`4`	`5`	`score_cron_schedule: "/30 * * *"`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`metric_batch: "hn_top_stories_scores"`
	`2`	`+table_key: "metrics_hackernews"`
`2`	`3`	`ingest_cron_schedule: "45 6 * * *"`
`3`	`4`	`train_cron_schedule: "15 8 * * *"`
`4`	`5`	`score_cron_schedule: "15 7 * * *"`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`metric_batch: "netdata"`
	`2`	`+table_key: "metrics_netdata"`
`2`	`3`	`ingest_cron_schedule: "/10 * * *"`
`3`	`4`	`train_cron_schedule: "/180 * * *"`
`4`	`5`	`score_cron_schedule: "/40 * * *"`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`metric_batch: "netdata_httpcheck"`
	`2`	`+table_key: "metrics_netdata_httpcheck"`
`2`	`3`	`ingest_cron_schedule: "/10 * * *"`
`3`	`4`	`train_cron_schedule: "/180 * * *"`
`4`	`5`	`score_cron_schedule: "/40 * * *"`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`metric_batch: "prometheus"`
	`2`	`+table_key: "metrics_prometheus"`
`2`	`3`	`ingest_cron_schedule: "/5 * * *"`
`3`	`4`	`train_cron_schedule: "/180 * * *"`
`4`	`5`	`score_cron_schedule: "/10 * * *"`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`metric_batch: "python_ingest_simple"`
	`2`	`+table_key: "metrics_python_ingest_simple"`
`2`	`3`	`alert_methods: "email,slack"`
`3`	`4`	`ingest_cron_schedule: "/2 * * *"`
`4`	`5`	`train_cron_schedule: "/4 * * *"`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`metric_batch: "tomtom"`
	`2`	`+table_key: "metrics_tomtom"`
`2`	`3`	`ingest_cron_schedule: "45 10 * * *"`
`3`	`4`	`train_cron_schedule: "15 11 * * *"`
`4`	`5`	`score_cron_schedule: "15 10 * * *"`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`metric_batch: "weather"`
	`2`	`+table_key: "metrics_weather"`
`2`	`3`	`ingest_cron_schedule: "45 14 * * *"`
`3`	`4`	`train_cron_schedule: "15 15 * * *"`
`4`	`5`	`score_cron_schedule: "15 14 * * *"`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`metric_batch: "yfinance"`
	`2`	`+table_key: "metrics_yfinance"`
`2`	`3`	`ingest_cron_schedule: "45 18 * * *"`
`3`	`4`	`train_cron_schedule: "15 20 * * *"`
`4`	`5`	`score_cron_schedule: "15 19 * * *"`