added dotenv to requirements

Author: dominiquegarmier

notion_graph/parser.py

@@ -7,6 +7,7 @@
 import time
 from collections import defaultdict
 from functools import wraps
+from pprint import pprint
 from typing import Any
 from typing import Awaitable
 from typing import Callable
@@ -65,7 +66,7 @@ def inner(_self: Parser, *args: Any, **kwargs: Any) -> None:
 class TaskQueue(asyncio.Queue[Task]):
     done: asyncio.Event
 
-    def __init__(self, num_worker: int = 16):
+    def __init__(self, num_worker: int = 64):
         super().__init__()
         self.num_worker = num_worker
         self.done = asyncio.Event()
@@ -163,6 +164,7 @@ class Parser:
     task_queue = TaskQueue()
     graph: Graph
     parsed_blocks: set[str]
+    parsed_blocks_lock = asyncio.Lock()
 
     root_id: str
     max_workers: int
@@ -187,11 +189,6 @@ def parse(self) -> Graph:
 
     @task
     async def parse_page(self, page_id: str) -> None:
-        if page_id in self.parsed_blocks:
-            return
-        else:
-            self.parsed_blocks.add(page_id)
-
         def _parse_title(page_dict: dict[str, Any]) -> str:
             try:
                 icon = cast(str, page_dict['icon']['emoji'])
@@ -239,9 +236,17 @@ async def parse_database(self, database_id: str) -> None:
     async def parse_children(self, block_id: str, page_id: str) -> None:
         children = (await get_children(block_id=block_id))['results']
         for block_dict in children:
-            self._parse_block(block_dict=block_dict, block_id=block_id, page_id=page_id)
+            await self._parse_block(
+                block_dict=block_dict, block_id=block_dict['id'], page_id=page_id
+            )
+
+    async def _parse_block(self, block_dict: dict, block_id: str, page_id: str) -> None:
+        async with self.parsed_blocks_lock:
+            if block_id in self.parsed_blocks:
+                return
+            else:
+                self.parsed_blocks.add(block_id)
 
-    def _parse_block(self, block_dict: dict, block_id: str, page_id: str) -> None:
         def _get_relations(
             block_dict: dict, block_id: str, page_id: str
         ) -> list[Relation]:
@@ -286,11 +291,12 @@ def _get_relations(
             return
 
         elif block_dict['type'] == 'child_database':
+            # logger.warning('child_database not implemented')
             return
 
         else:
             if block_dict['has_children']:
-                self.parsed_blocks.add(block_id)
+                self.parse_children(block_id=block_id, page_id=page_id)
 
             relations = _get_relations(block_dict, block_id, page_id)
             for relation in relations:

notion_graph/templates/index.html

@@ -10,7 +10,7 @@
 
       let selfLoopLinks = {};
       let sameNodesLinks = {};
-      const curvatureMinMax = 0.5;
+      const curvatureMinMax = 0.25;
 
       // 1. assign each link a nodePairId that combines their source and target independent of the links direction
       // 2. group links together that share the same two nodes or are self-loops
@@ -61,6 +61,7 @@
           .onNodeClick(node => window.open(`${node.url}`, '_blank'))
           .linkColor(() => 'rgba(255,255,255,0.2)')
           .linkCurvature('curvature')
+          .d3AlphaDecay(0.01)
           .graphData(gData);
         });
     </script>

requirements.txt

@@ -1,2 +1,3 @@
 aiohttp
 flask
+python-dotenv

t.py

@@ -9,7 +9,7 @@
 
 
 async def amain():
-    p = await get_children(block_id='453776be35854f74b098f7f6529ae33a')
+    p = await get_children(block_id='2465521b79364efba808e29cde228616')
     pprint(p)