Skip to content

Commit 3ffee45

Browse files
andrewm4894andrewm4894
authored
Merge pull request #111 from andrewm4894/make-python-random
Make python random
2 parents f9c5624 + 8efad50 commit 3ffee45

File tree

6 files changed

+167
-17
lines changed

6 files changed

+167
-17
lines changed

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ Supported storage for your trained models:
8787
</tbody>
8888
</table>
8989

90-
Supported ways to receive alerts:
90+
Supported ways to receive [alerts](#alerts):
9191

9292
<table>
9393
<thead>

‎anomstack/jobs/plot.py‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
MetadataValue,
1515
ScheduleDefinition,
1616
asset,
17+
get_dagster_logger,
1718
job,
1819
op,
1920
)
@@ -39,6 +40,8 @@ def build_plot_job(spec: dict) -> JobDefinition:
3940
JobDefinition: The plot job definition.
4041
"""
4142

43+
logger = get_dagster_logger()
44+
4245
if spec.get("disable_plot"):
4346

4447
@job(
@@ -95,14 +98,20 @@ def make_plot(context, df: pd.DataFrame) -> None:
9598
df (pd.DataFrame): The plot data.
9699
"""
97100

98-
fig = make_batch_plot(df)
101+
if len(df) > 0:
102+
103+
fig = make_batch_plot(df)
104+
105+
buffer = BytesIO()
106+
fig.savefig(buffer, format="png")
107+
image_data = base64.b64encode(buffer.getvalue())
108+
md_content = f"![img](data:image/png;base64,{image_data.decode()})"
109+
110+
context.add_output_metadata({"plot": MetadataValue.md(md_content)})
99111

100-
buffer = BytesIO()
101-
fig.savefig(buffer, format="png")
102-
image_data = base64.b64encode(buffer.getvalue())
103-
md_content = f"![img](data:image/png;base64,{image_data.decode()})"
112+
else:
104113

105-
context.add_output_metadata({"plot": MetadataValue.md(md_content)})
114+
logger.info("no data to plot")
106115

107116
make_plot(get_plot_data())
108117

‎metrics/examples/README.md‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,113 @@
11
# Examples
22

33
Some example metric batch sub folders. For example you might have one per source or subject, or whatever makes most sense to you really.
4+
5+
- [bigquery](bigquery/): Example of a BigQuery metric batch.
6+
- [eirgrid](eirgrid/): Example of a metric batch that uses a custom python `ingest_fn` parameter to just use python to create an `ingest()` function that returns a pandas df.
7+
- [example_jinja](example_jinja/): Example of a metric batch that uses Jinja templating.
8+
- [example_simple](example_simple/): Example of a simple metric batch.
9+
- [example_sql_file](example_sql_file/): Example of a metric batch that uses a SQL file.
10+
- [freq](freq/): Example of a metric batch that uses the `freq` parameter.
11+
- [gsod](gsod/): Example of a metric batch that uses GSOD data from BigQuery.
12+
- [gtrends](gtrends/): Example of a metric batch that uses Google Trends data from BigQuery.
13+
- [hackernews](hackernews/): Example of a metric batch that uses the Hacker News API.
14+
- [netdata](netdata/): Example of a metric batch that uses the Netdata API.
15+
- [netdata_httpcheck](netdata_httpcheck/): Example of a metric batch that uses the Netdata API to check the status of a website.
16+
- [python](python/): Example of a metric batch that uses a custom python `ingest_fn` parameter to just use python to create an `ingest()` function that returns a pandas df.
17+
- [s3](s3/): Example of a metric batch that uses S3.
18+
- [sales](sales/): Example of a metric batch that uses a SQL file.
19+
- [snowflake](snowflake/): Example of a metric batch that uses Snowflake.
20+
- [tomtom](tomtom/): Example of a metric batch that uses the TomTom API.
21+
- [users](users/): Example of a metric batch that uses a SQL file.
22+
- [weather](weather/): Example of a metric batch that uses Open Meteo data.
23+
- [weather_forecast](weather_forecast/): Example of a metric batch that uses weather forecast data from Snowflake.
24+
- [yfinance](yfinance/): Example of a metric batch that uses the Yahoo Finance API.
25+
26+
27+
```
28+
.
29+
├── README.md
30+
├── bigquery
31+
│ ├── README.md
32+
│ └── bigquery_example_simple
33+
│ ├── README.md
34+
│ └── bigquery_example_simple.yaml
35+
├── eirgrid
36+
│ ├── README.md
37+
│ ├── eirgrid.py
38+
│ └── eirgrid.yaml
39+
├── example_jinja
40+
│ ├── README.md
41+
│ └── example_jinja.yaml
42+
├── example_simple
43+
│ ├── README.md
44+
│ └── example_simple.yaml
45+
├── example_sql_file
46+
│ ├── README.md
47+
│ ├── example_sql_file.sql
48+
│ └── example_sql_file.yaml
49+
├── freq
50+
│ ├── README.md
51+
│ └── freq.yaml
52+
├── gsod
53+
│ ├── README.md
54+
│ ├── gsod.sql
55+
│ └── gsod.yaml
56+
├── gtrends
57+
│ ├── README.md
58+
│ ├── gtrends.sql
59+
│ └── gtrends.yaml
60+
├── hackernews
61+
│ ├── README.md
62+
│ ├── hn_top_stories_scores.py
63+
│ └── hn_top_stories_scores.yaml
64+
├── netdata
65+
│ ├── README.md
66+
│ ├── netdata.py
67+
│ └── netdata.yaml
68+
├── netdata_httpcheck
69+
│ ├── netdata_httpcheck.py
70+
│ └── netdata_httpcheck.yaml
71+
├── python
72+
│ ├── README.md
73+
│ └── python_ingest_simple
74+
│ ├── README.md
75+
│ ├── ingest.py
76+
│ └── python_ingest_simple.yaml
77+
├── s3
78+
│ ├── README.md
79+
│ └── s3_example_simple
80+
│ ├── README.md
81+
│ └── s3_example_simple.yaml
82+
├── sales
83+
│ ├── README.md
84+
│ ├── sales.sql
85+
│ └── sales.yaml
86+
├── snowflake
87+
│ ├── README.md
88+
│ └── snowflake_example_simple
89+
│ ├── README.md
90+
│ └── snowflake_example_simple.yaml
91+
├── tomtom
92+
│ ├── README.md
93+
│ ├── tomtom.py
94+
│ └── tomtom.yaml
95+
├── users
96+
│ ├── README.md
97+
│ ├── users.sql
98+
│ └── users.yaml
99+
├── weather
100+
│ ├── README.md
101+
│ ├── ingest_weather.py
102+
│ └── weather.yaml
103+
├── weather_forecast
104+
│ ├── README.md
105+
│ ├── weather_forecast.sql
106+
│ └── weather_forecast.yaml
107+
└── yfinance
108+
├── README.md
109+
├── yfinance.py
110+
└── yfinance.yaml
111+
112+
25 directories, 58 files
113+
```

‎metrics/examples/python/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
# Python Examples
22

3-
Examples using a custom python `ingest_fn` paramater to just use python to create an `ingest()` function that returns a pandas df.
3+
Examples using a custom python `ingest_fn` parameter to just use python to create an `ingest()` function that returns a pandas df.

‎metrics/examples/python/python_ingest_simple/ingest.py‎

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,52 @@
11
def ingest():
2+
"""
3+
Generate random metrics data with occasional anomalies (spikes, drops,
4+
or plateaus).
5+
"""
6+
27
import random
38
import time
49

510
import pandas as pd
611

7-
# generate random metrics
8-
metric1_value = random.uniform(0, 10)
9-
metric2_value = random.uniform(0, 10)
12+
# Define the number of metrics
13+
metrics = ["metric1", "metric2", "metric3", "metric4", "metric5"]
14+
metric_values = []
1015

11-
# get current timestamp
16+
# Get current timestamp
1217
current_timestamp = int(time.time())
1318

14-
# make df
19+
# Different anomaly types
20+
anomaly_types = ["spike", "drop", "plateau"]
21+
22+
# Generate random metrics and introduce occasional anomalies (1% chance)
23+
anomaly_chance = random.random()
24+
anomaly_type = (random.choice(anomaly_types)
25+
if anomaly_chance <= 0.01 else None)
26+
27+
for metric in metrics:
28+
if anomaly_type == "spike":
29+
# Generate a spike (e.g., a value significantly higher than normal)
30+
metric_value = random.uniform(15, 30)
31+
elif anomaly_type == "drop":
32+
# Generate a drop (e.g., a negative or significantly low value)
33+
metric_value = random.uniform(-10, -1)
34+
elif anomaly_type == "plateau":
35+
# Generate the same value for all metrics (e.g., a plateau)
36+
plateau_value = random.uniform(5, 6)
37+
metric_values = [plateau_value] * len(metrics)
38+
break # Exit the loop early since all metrics have the same value
39+
else:
40+
# Generate a normal value
41+
metric_value = random.uniform(0, 10)
42+
43+
metric_values.append(metric_value)
44+
45+
# Create a DataFrame with the metric data
1546
data = {
16-
"metric_name": ["metric1", "metric2"],
17-
"metric_value": [metric1_value, metric2_value],
18-
"metric_timestamp": [current_timestamp, current_timestamp],
47+
"metric_name": metrics,
48+
"metric_value": metric_values,
49+
"metric_timestamp": [current_timestamp] * len(metrics),
1950
}
2051
df = pd.DataFrame(data)
2152

‎tests/test_main.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22

33
import pytest
44

5-
from anomstack.main import ingest_jobs, ingest_schedules, jobs, schedules, sensors
5+
from anomstack.main import ingest_jobs, ingest_schedules, jobs, schedules
66

77
logger = logging.getLogger(__name__)
88

0 commit comments

Comments
 (0)