Merge pull request #120 from andrewm4894/simplify-llm

Simplify llm

Author: andrewm4894

anomstack/alerts/asciiart.py

@@ -528,15 +528,21 @@ def make_alert_message(
     metric_timestamp_to = (
         df_alert_metric["metric_timestamp"].max().strftime("%Y-%m-%d %H:%M")
     )
-    labels = (
-        np.where(df_alert_metric["metric_alert"] == 1, anomaly_symbol, normal_symbol)
-        + (df_alert_metric[score_col].round(2) * 100).astype("int").astype("str")
-        + "% "
-    )
-    data = zip(labels, x)
     graph_title = f"{metric_name} ({metric_timestamp_from} to {metric_timestamp_to})"
     message = ""
     if ascii_graph:
+        labels = (
+            np.where(
+                df_alert_metric["metric_alert"] == 1,
+                anomaly_symbol,
+                normal_symbol
+            )
+            + (df_alert_metric[score_col].round(2) * 100)
+            .astype("int")
+            .astype("str")
+            + "% "
+        )
+        data = zip(labels, x)
         graph = Pyasciigraph(
             titlebar=" ", graphsymbol=graph_symbol, float_format=alert_float_format
         ).graph(graph_title, data)

anomstack/jobs/llmalert.py

@@ -58,7 +58,7 @@ def noop():
     alert_methods = spec["alert_methods"]
     llmalert_recent_n = spec["llmalert_recent_n"]
     llmalert_smooth_n = spec["llmalert_smooth_n"]
-    llmalert_metric_rounding = spec.get("llmalert_metric_rounding", 4)
+    llmalert_metric_rounding = spec.get("llmalert_metric_rounding", -1)
 
     @job(
         name=f"{metric_batch}_llmalert_job",
@@ -77,7 +77,7 @@ def get_llmalert_data() -> pd.DataFrame:
                 pd.DataFrame: A pandas DataFrame containing the data for the LLM Alert.
             """
 
-            df = read_sql(render("plot_sql", spec), db)
+            df = read_sql(render("llmalert_sql", spec), db)
 
             return df
 
@@ -105,11 +105,6 @@ def llmalert(context, df: pd.DataFrame) -> None:
                     .sort_values(by="metric_timestamp", ascending=True)
                     .reset_index(drop=True)
                 )
-                df_metric["metric_alert"] = df_metric["metric_alert"].fillna(0)
-                df_metric["metric_score"] = df_metric["metric_score"].fillna(0)
-                df_metric["metric_score_smooth"] = df_metric[
-                    "metric_score_smooth"
-                ].fillna(0)
                 df_metric = df_metric.dropna()
                 df_metric["metric_timestamp"] = pd.to_datetime(
                     df_metric["metric_timestamp"]
@@ -130,8 +125,13 @@ def llmalert(context, df: pd.DataFrame) -> None:
                 df_prompt = (
                     df_metric[["metric_timestamp", "metric_value", "metric_recency"]]
                     .dropna()
-                    .round(llmalert_metric_rounding)
                 )
+                df_prompt["metric_timestamp"] = df_metric[
+                    "metric_timestamp"
+                ].dt.strftime("%Y-%m-%d %H:%M:%S")
+                df_prompt = df_prompt.set_index("metric_timestamp")
+                if llmalert_metric_rounding >= 0:
+                    df_prompt = df_prompt.round(llmalert_metric_rounding)
 
                 # logger.debug(f"df_prompt: \n{df_prompt}")
 
@@ -172,6 +172,7 @@ def llmalert(context, df: pd.DataFrame) -> None:
                             "metric_timestamp": metric_timestamp_max,
                             "alert_type": "llm",
                         },
+                        score_col="metric_score"
                     )
 
         llmalert(get_llmalert_data())

metrics/defaults/defaults.yaml

@@ -61,8 +61,9 @@ change_detect_last_n: 1 # number of last n observations to detect changes on.
 # llmalert params
 ############################################
 llmalert_recent_n: 5 # only llmalert on recent n so as to avoid continually alerting.
-llmalert_smooth_n: 3 # smooth metric value prior to sending to llm.
-llmalert_metric_rounding: 4 # round metric values to this number of decimal places.
+llmalert_smooth_n: 0 # smooth metric value prior to sending to llm.
+llmalert_metric_rounding: -1 # round metric values to this number of decimal places.
+llmalert_metric_timestamp_max_days_ago: 1 # don't alert on metrics older than this.
 
 ############################################
 # schedules
@@ -100,6 +101,9 @@ change_sql: >
 # default templated plot sql
 plot_sql: >
   {% include "./defaults/sql/plot.sql" %}
+# default templated llmalert sql
+llmalert_sql: >
+  {% include "./defaults/sql/llmalert.sql" %}
 # default templated dashboard sql
 dashboard_sql: >
   {% include "./defaults/sql/dashboard.sql" %}

metrics/defaults/python/prompt.py

@@ -10,26 +10,19 @@ def make_prompt(df, llmalert_recent_n) -> str:
         str: A prompt for the user to check if there is an anomaly in the time series data.
     """
 
-    from tabulate import tabulate
-
-    text_representation = tabulate(
-        df.reset_index(), headers="keys", tablefmt="pipe", showindex=False
-    )
+    text_representation = df.to_markdown()
 
     prompt = f"""
-    You are a seasoned time series expert who has worked with time series data for many years and are very acomplished at spotting and explaining anomalies in time series data.
-
-    Can you help me check if there is an anomaly in this time series data for this metric?
+    Can you help me check if there is an anomaly in the below time series data?
 
-    I am solely interested in looking at the last {llmalert_recent_n} observations (when metric_recency=recent) and if it looks like the more recent data may be anomalous or if it looks not all that much different from the rest of the data (metric_recency=baseline).
+    I am solely interested in looking at the last {llmalert_recent_n} observations (when metric_recency=recent) and if it looks like the more recent data may be anomalous in comparison to rest of the data (when metric_recency=baseline).
 
     Here are some questions to think about:
 
     - Is there anything unusual about the last {llmalert_recent_n} recent values of the metric in the df DataFrame?
     - Are there any anomalies or outliers in the recent {llmalert_recent_n} observations of metric in df?
     - Can you identify any patterns or trends in the recent {llmalert_recent_n} values of the metric in df that could be indicative of an anomaly?
     - How does the distribution of the recent {llmalert_recent_n} values of the metric in df compare to the distribution of the entire dataset?
-    - Are there any changes in the mean, median, or standard deviation of the metric in the recent {llmalert_recent_n} observations that could be indicative of an anomaly?
     - Is there a sudden increase or decrease in the metric in the recent {llmalert_recent_n} observations?
     - Is there a change in the slope of the metric trend line in the recent {llmalert_recent_n} observations?
     - Are there any spikes or dips in the metric in the recent {llmalert_recent_n} observations?
@@ -44,10 +37,6 @@ def make_prompt(df, llmalert_recent_n) -> str:
     - Focus only on how the most recent {llmalert_recent_n} observations and if they look anomalous or not in reference to the earlier baseline data.
     - The data comes from a pandas dataframe.
 
-    Here is the data (ordered in ascending order, so from oldest to newest (top to bottom)):
-
-    {text_representation}
-
     I need a yes or no answer as to if you think the recent data looks anomalous or not.
 
     Please also provide a description on why the metric looks anomalous if you think it does.
@@ -57,6 +46,10 @@ def make_prompt(df, llmalert_recent_n) -> str:
     Please think step by step and provide a description, along with evidence, of your thought process as you go through the data.
 
     Think globally too like a human would if they were eyeballing the data.
+
+    Here is the data (ordered in ascending order, so from oldest to newest (top to bottom)):
+
+    {text_representation}
     """
 
     return prompt

metrics/defaults/sql/llmalert.sql

@@ -0,0 +1,98 @@
+/*
+Template for generating the input data for the llmalert job.
+*/
+
+with
+
+metric_value_data as
+(
+select distinct
+  metric_timestamp,
+  metric_batch,
+  metric_name,
+  avg(metric_value) AS metric_value
+from
+  {{ table_key }}
+where
+  metric_batch = '{{ metric_batch }}'
+  and
+  metric_type = 'metric'
+  and
+  date(metric_timestamp) >= date('now', '-{{ llmalert_metric_timestamp_max_days_ago }} day')
+group by metric_timestamp, metric_batch, metric_name
+),
+
+metric_score_data as
+(
+select distinct
+  metric_timestamp,
+  metric_batch,
+  metric_name,
+  avg(metric_value) AS metric_score
+from
+  {{ table_key }}
+where
+  metric_batch = '{{ metric_batch }}'
+  and
+  metric_type = 'score'
+  and
+  date(metric_timestamp) >= date('now', '-{{ llmalert_metric_timestamp_max_days_ago }} day')
+group by metric_timestamp, metric_batch, metric_name
+),
+
+metric_alert_data as
+(
+select distinct
+  metric_timestamp,
+  metric_batch,
+  metric_name,
+  avg(metric_value) AS metric_alert
+from
+  {{ table_key }}
+where
+  metric_batch = '{{ metric_batch }}'
+  and
+  metric_type = 'alert'
+  and
+  date(metric_timestamp) >= date('now', '-{{ llmalert_metric_timestamp_max_days_ago }} day')
+group by metric_timestamp, metric_batch, metric_name
+),
+
+metric_value_recency_ranked as
+(
+select
+  metric_timestamp,
+  metric_batch,
+  metric_name,
+  metric_value,
+  row_number() over (partition by metric_name order by metric_timestamp desc) as metric_value_recency_rank
+from
+  metric_value_data
+)
+
+select
+  m.metric_timestamp,
+  m.metric_batch,
+  m.metric_name,
+  m.metric_value,
+  ifnull(s.metric_score,0) as metric_score,
+  ifnull(a.metric_alert,0) as metric_alert
+from
+  metric_value_recency_ranked m
+left join
+  metric_score_data s
+on
+  m.metric_timestamp = s.metric_timestamp
+  and
+  m.metric_batch = s.metric_batch
+  and
+  m.metric_name = s.metric_name
+left join
+  metric_alert_data a
+on
+  m.metric_timestamp = a.metric_timestamp
+  and
+  m.metric_batch = a.metric_batch
+  and
+  m.metric_name = a.metric_name
+;

requirements.compile

@@ -25,4 +25,3 @@ slack_sdk
 snowflake-connector-python[pandas]
 sqlglot
 streamlit
-tabulate

requirements.txt

@@ -506,9 +506,7 @@ streamlit==1.38.0
 structlog==24.4.0
     # via dagster
 tabulate==0.9.0
-    # via
-    #   -r requirements.compile
-    #   dagster
+    # via dagster
 tenacity==8.5.0
     # via
     #   plotly