Skip to content

Commit 6f49838

Browse files
rfdearbornrfdearborn
authored
Merge pull request rfdearborn#21 from rfdearborn/rfdearborn/add_tests
Initialize tests
2 parents 357b478 + 2a71b67 commit 6f49838

File tree

8 files changed

+354
-8
lines changed

8 files changed

+354
-8
lines changed

‎.github/workflows/test.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: Run Tests
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest
12+
env:
13+
DATABASE_NAME: 'mock_database_name'
14+
DATABASE_PARENT_ID: 'mock_database_parent_id'
15+
NOTION_TOKEN: 'mock_notion_token'
16+
steps:
17+
- uses: actions/checkout@v2
18+
- name: Set up Python 3.7
19+
uses: actions/setup-python@v2
20+
with:
21+
python-version: '3.7'
22+
- name: Install dependencies
23+
run: |
24+
pip install requests
25+
pip install mock
26+
- name: Run unit tests
27+
run: |
28+
python -m unittest tests/test_unit.py
29+
- name: Run integration tests
30+
run: |
31+
python -m unittest tests/test_integration.py

‎.gitignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,3 @@
1-
.vscode
1+
.vscode
2+
__pycache__
3+
/tests/__pycache__

‎README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,5 +69,4 @@ jobs:
6969
7070
## Todo
7171
72-
- Tests
7372
- Visualize models graph

‎dbt_docs_to_notion.py‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,10 @@ def get_owner(data, catalog_nodes, model_name):
5858
return get_paths_or_empty(catalog_nodes, [[model_name, 'metadata', 'owner']], '')
5959

6060

61-
def main():
62-
model_records_to_write = sys.argv[1:] # 'all' or list of model names
61+
def main(argv=None):
62+
if argv is None:
63+
argv = sys.argv
64+
model_records_to_write = argv[1:] # 'all' or list of model names
6365
print(f'Model records to write: {model_records_to_write}')
6466

6567
###### load nodes from dbt docs ######
@@ -313,15 +315,15 @@ def main():
313315
"children": columns_table_children_obj
314316
}
315317
},
316-
# Raw SQL
318+
# Raw Code
317319
{
318320
"object": "block",
319321
"type": "heading_1",
320322
"heading_1": {
321323
"rich_text": [
322324
{
323325
"type": "text",
324-
"text": { "content": "Raw SQL" }
326+
"text": { "content": "Raw Code" }
325327
}
326328
]
327329
}
@@ -341,15 +343,15 @@ def main():
341343
"language": "sql"
342344
}
343345
},
344-
# Compiled SQL
346+
# Compiled Code
345347
{
346348
"object": "block",
347349
"type": "heading_1",
348350
"heading_1": {
349351
"rich_text": [
350352
{
351353
"type": "text",
352-
"text": { "content": "Compiled SQL" }
354+
"text": { "content": "Compiled Code" }
353355
}
354356
]
355357
}
@@ -509,5 +511,6 @@ def main():
509511
json=record_obj
510512
)
511513

514+
512515
if __name__ == '__main__':
513516
main()

‎tests/__init__.py‎

Whitespace-only changes.

‎tests/mock_data.py‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# Mock Data for dbt and Notion API
2+
import os
3+
4+
# Mock dbt Data
5+
DBT_MOCK_CATALOG = {
6+
"nodes": {
7+
"model.test.model_1": {
8+
"columns": {
9+
"column_1": {
10+
"type": "TEXT"
11+
},
12+
"column_2": {
13+
"type": "TEXT"
14+
},
15+
},
16+
"metadata": {
17+
"owner": "[email protected]"
18+
},
19+
"stats": {
20+
"row_count": {
21+
"value": 1,
22+
},
23+
"bytes": {
24+
"value": 1000000,
25+
},
26+
},
27+
},
28+
},
29+
}
30+
31+
DBT_MOCK_MANIFEST = {
32+
"nodes": {
33+
"model.test.model_1": {
34+
"resource_type": "model",
35+
"columns": {
36+
"column_1": {
37+
"description": "Description for column 1"
38+
},
39+
"column_2": {
40+
"description": "Description for column 2"
41+
},
42+
},
43+
"raw_code": "SELECT 1",
44+
"compiled_code": "SELECT 1",
45+
"name": "model_1",
46+
"description": "Description for model 1",
47+
"relation_name": "model.test.model_1",
48+
"depends_on": ["model.test.model_2"],
49+
"tags": ["tag1", "tag2"],
50+
},
51+
},
52+
}
53+
54+
# Mock Notion API Responses
55+
NOTION_MOCK_EXISTENT_CHILD_PAGE_QUERY = {
56+
"results": [
57+
{
58+
"id": "mock_child_id",
59+
"child_database": {
60+
"title": os.environ['DATABASE_NAME'],
61+
},
62+
},
63+
],
64+
}
65+
66+
NOTION_MOCK_EXISTENT_DATABASE_RECORDS_QUERY = {
67+
"results": [
68+
{
69+
"id": "mock_record_id",
70+
},
71+
],
72+
}
73+
74+
NOTION_MOCK_NONEXISTENT_QUERY = {
75+
"results": [],
76+
}
77+
78+
NOTION_MOCK_DATABASE_CREATE = {
79+
"id": "mock_database_id",
80+
}
81+
82+
NOTION_MOCK_RECORD_CREATE = {
83+
"id": "mock_record_id",
84+
}

‎tests/test_integration.py‎

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
import json
2+
import os
3+
import unittest
4+
from unittest.mock import patch, Mock
5+
6+
from dbt_docs_to_notion import main
7+
from tests.mock_data import (
8+
DBT_MOCK_MANIFEST,
9+
DBT_MOCK_CATALOG,
10+
NOTION_MOCK_EXISTENT_CHILD_PAGE_QUERY,
11+
NOTION_MOCK_EXISTENT_DATABASE_RECORDS_QUERY,
12+
NOTION_MOCK_NONEXISTENT_QUERY,
13+
NOTION_MOCK_DATABASE_CREATE,
14+
NOTION_MOCK_RECORD_CREATE,
15+
)
16+
17+
18+
class TestDbtDocsToNotionIntegration(unittest.TestCase):
19+
20+
def setUp(self):
21+
patch('dbt_docs_to_notion.json.load').start().side_effect = [DBT_MOCK_MANIFEST, DBT_MOCK_CATALOG]
22+
patch('dbt_docs_to_notion.open', new_callable=unittest.mock.mock_open, read_data="data").start()
23+
self.comparison_catalog = DBT_MOCK_CATALOG['nodes']['model.test.model_1']
24+
self.comparison_manifest = DBT_MOCK_MANIFEST['nodes']['model.test.model_1']
25+
self.recorded_requests = []
26+
27+
def tearDown(self):
28+
patch.stopall()
29+
30+
def _verify_database_obj(self, database_obj):
31+
title = database_obj['title'][0]
32+
self.assertEqual(title['type'], 'text')
33+
self.assertEqual(title['text']['content'], os.environ['DATABASE_NAME'])
34+
parent = database_obj['parent']
35+
self.assertEqual(parent['type'], 'page_id')
36+
self.assertEqual(parent['page_id'], os.environ['DATABASE_PARENT_ID'])
37+
properties = database_obj['properties']
38+
self.assertEqual(properties['Name'], {'title': {}})
39+
self.assertEqual(properties['Description'], {'rich_text': {}})
40+
self.assertEqual(properties['Owner'], {'rich_text': {}})
41+
self.assertEqual(properties['Relation'], {'rich_text': {}})
42+
self.assertEqual(
43+
properties['Approx Rows'],
44+
{'number': {'format': 'number_with_commas'}}
45+
)
46+
self.assertEqual(
47+
properties['Approx GB'],
48+
{'number': {'format': 'number_with_commas'}}
49+
)
50+
self.assertEqual(properties['Depends On'], {'rich_text': {}})
51+
self.assertEqual(properties['Tags'], {'rich_text': {}})
52+
53+
def _verify_record_obj(self, record_obj):
54+
parent = record_obj['parent']
55+
self.assertEqual(parent['database_id'], NOTION_MOCK_DATABASE_CREATE['id'])
56+
properties = record_obj['properties']
57+
self.assertEqual(properties['Name']['title'][0]['text']['content'], self.comparison_manifest['name'])
58+
self.assertEqual(properties['Description']['rich_text'][0]['text']['content'], self.comparison_manifest['description'])
59+
self.assertEqual(properties['Owner']['rich_text'][0]['text']['content'], self.comparison_catalog['metadata']['owner'])
60+
self.assertEqual(properties['Relation']['rich_text'][0]['text']['content'], self.comparison_manifest['relation_name'])
61+
self.assertEqual(properties['Approx Rows']['number'], self.comparison_catalog['stats']['row_count']['value'])
62+
self.assertEqual(properties['Approx GB']['number'], self.comparison_catalog['stats']['bytes']['value']/1e9)
63+
self.assertEqual(properties['Depends On']['rich_text'][0]['text']['content'], json.dumps(self.comparison_manifest['depends_on']))
64+
self.assertEqual(properties['Tags']['rich_text'][0]['text']['content'], json.dumps(self.comparison_manifest['tags']))
65+
66+
def _verify_record_children_obj(self, record_children_obj):
67+
toc_child_block = record_children_obj[0]
68+
self.assertEqual(toc_child_block['object'], 'block')
69+
self.assertEqual(toc_child_block['type'], 'table_of_contents')
70+
columns_header_child_block = record_children_obj[1]
71+
self.assertEqual(columns_header_child_block['object'], 'block')
72+
self.assertEqual(columns_header_child_block['type'], 'heading_1')
73+
self.assertEqual(columns_header_child_block['heading_1']['rich_text'][0]['text']['content'], 'Columns')
74+
columns_child_block = record_children_obj[2]
75+
self.assertEqual(columns_child_block['object'], 'block')
76+
self.assertEqual(columns_child_block['type'], 'table')
77+
self.assertEqual(columns_child_block['table']['table_width'], 3)
78+
self.assertEqual(columns_child_block['table']['has_column_header'], True)
79+
self.assertEqual(columns_child_block['table']['has_row_header'], False)
80+
columns_table_children_obj = columns_child_block['table']['children']
81+
columns_table_header_row = columns_table_children_obj[0]
82+
self.assertEqual(columns_table_header_row['type'], 'table_row')
83+
self.assertEqual(columns_table_header_row['table_row']['cells'][0][0]['plain_text'], 'Column')
84+
self.assertEqual(columns_table_header_row['table_row']['cells'][1][0]['plain_text'], 'Type')
85+
self.assertEqual(columns_table_header_row['table_row']['cells'][2][0]['plain_text'], 'Description')
86+
columns_table_row = columns_table_children_obj[1]
87+
self.assertEqual(columns_table_row['type'], 'table_row')
88+
self.assertEqual(columns_table_row['table_row']['cells'][0][0]['plain_text'], list(self.comparison_catalog['columns'].keys())[0])
89+
self.assertEqual(columns_table_row['table_row']['cells'][1][0]['plain_text'], list(self.comparison_catalog['columns'].values())[0]['type'])
90+
self.assertEqual(columns_table_row['table_row']['cells'][2][0]['plain_text'], list(self.comparison_manifest['columns'].values())[0]['description'])
91+
raw_code_header_child_block = record_children_obj[3]
92+
self.assertEqual(raw_code_header_child_block['object'], 'block')
93+
self.assertEqual(raw_code_header_child_block['type'], 'heading_1')
94+
self.assertEqual(raw_code_header_child_block['heading_1']['rich_text'][0]['text']['content'], 'Raw Code')
95+
raw_code_child_block = record_children_obj[4]
96+
self.assertEqual(raw_code_child_block['object'], 'block')
97+
self.assertEqual(raw_code_child_block['type'], 'code')
98+
self.assertEqual(raw_code_child_block['code']['language'], 'sql')
99+
self.assertEqual(raw_code_child_block['code']['rich_text'][0]['text']['content'], self.comparison_manifest['raw_code'])
100+
compiled_code_header_child_block = record_children_obj[5]
101+
self.assertEqual(compiled_code_header_child_block['object'], 'block')
102+
self.assertEqual(compiled_code_header_child_block['type'], 'heading_1')
103+
self.assertEqual(compiled_code_header_child_block['heading_1']['rich_text'][0]['text']['content'], 'Compiled Code')
104+
compiled_code_child_block = record_children_obj[6]
105+
self.assertEqual(compiled_code_child_block['object'], 'block')
106+
self.assertEqual(compiled_code_child_block['type'], 'code')
107+
self.assertEqual(compiled_code_child_block['code']['language'], 'sql')
108+
self.assertEqual(compiled_code_child_block['code']['rich_text'][0]['text']['content'], self.comparison_manifest['compiled_code'])
109+
110+
@patch('dbt_docs_to_notion.make_request')
111+
def test_create_new_database(self, mock_make_request):
112+
def _mocked_make_request(endpoint, querystring, method, **request_kwargs):
113+
self.recorded_requests.append((endpoint, method))
114+
if endpoint == 'blocks/' and method == 'GET':
115+
return NOTION_MOCK_NONEXISTENT_QUERY
116+
elif endpoint == 'databases/' and querystring == '' and method == 'POST':
117+
database_obj = request_kwargs['json']
118+
self._verify_database_obj(database_obj)
119+
return NOTION_MOCK_DATABASE_CREATE
120+
elif endpoint == 'databases/' and '/query' in querystring and method == 'POST':
121+
return NOTION_MOCK_NONEXISTENT_QUERY
122+
elif endpoint == 'pages/' and method == 'POST':
123+
record_obj = request_kwargs['json']
124+
self._verify_record_obj(record_obj)
125+
record_children_obj = request_kwargs['json']['children']
126+
self._verify_record_children_obj(record_children_obj)
127+
return NOTION_MOCK_RECORD_CREATE
128+
mock_make_request.side_effect = _mocked_make_request
129+
130+
main(argv=[None, 'all'])
131+
132+
self.assertEqual(
133+
self.recorded_requests,
134+
[
135+
('blocks/', 'GET'),
136+
('databases/', 'POST'),
137+
('databases/', 'POST'),
138+
('pages/', 'POST'),
139+
]
140+
)
141+
142+
@patch('dbt_docs_to_notion.make_request')
143+
def test_update_existing_database(self, mock_make_request):
144+
def _mocked_make_request(endpoint, querystring, method, **request_kwargs):
145+
self.recorded_requests.append((endpoint, method))
146+
if endpoint == 'blocks/' and method == 'GET':
147+
return NOTION_MOCK_EXISTENT_CHILD_PAGE_QUERY
148+
elif endpoint == 'databases/' and '/query' in querystring and method == 'POST':
149+
return NOTION_MOCK_EXISTENT_DATABASE_RECORDS_QUERY
150+
elif endpoint == 'pages/' and method == 'PATCH':
151+
record_obj = request_kwargs['json']
152+
self._verify_record_obj(record_obj)
153+
return {} # response is thrown away
154+
elif endpoint == 'blocks/' and method == 'DELETE':
155+
return {} # response is thrown away
156+
elif endpoint == 'blocks/' and method == 'PATCH':
157+
record_children_obj = request_kwargs['json']['children']
158+
self._verify_record_children_obj(record_children_obj)
159+
return {} # response is thrown away
160+
mock_make_request.side_effect = _mocked_make_request
161+
162+
main(argv=[None, 'all'])
163+
164+
self.assertEqual(
165+
self.recorded_requests,
166+
[
167+
('blocks/', 'GET'),
168+
('databases/', 'POST'),
169+
('pages/mock_record_id', 'PATCH'),
170+
('blocks/', 'GET'),
171+
('blocks/', 'DELETE'),
172+
('blocks/', 'PATCH'),
173+
]
174+
)
175+
176+
177+
if __name__ == '__main__':
178+
unittest.main()

0 commit comments

Comments
 (0)