neo4j-labs
diff --git a/‎backend/score.py‎
Lines changed: 3 additions & 5 deletions b/‎backend/score.py‎
Lines changed: 3 additions & 5 deletions
diff --git a/‎backend/src/main.py‎
Lines changed: 8 additions & 5 deletions b/‎backend/src/main.py‎
Lines changed: 8 additions & 5 deletions
diff --git a/‎backend/test_integrationqa.py‎
Lines changed: 143 additions & 133 deletions b/‎backend/test_integrationqa.py‎
Lines changed: 143 additions & 133 deletions
@@ -231,7 +231,7 @@ async def extract_knowledge_graph_from_file(
                 logging.info(f'Deleted File Path: {merged_file_path} and Deleted File Name : {file_name}')
                 delete_uploaded_local_file(merged_file_path,file_name)
         json_obj = {'message':message,'error_message':error_message, 'file_name': file_name,'status':'Failed','db_url':uri,'failed_count':1, 'source_type': source_type, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc))}
-        logger.log_struct(json_obj)
+        logger.log_struct(json_obj, "ERROR")
         logging.exception(f'File Failed in extraction: {json_obj}')
         return create_api_response('Failed', message=message + error_message[:100], error=error_message, file_name = file_name)
     finally:
@@ -265,11 +265,10 @@ async def post_processing(uri=Form(), userName=Form(), password=Form(), database
     try:
         graph = create_graph_database_connection(uri, userName, password, database)
         tasks = set(map(str.strip, json.loads(tasks)))
-
+        
         if "materialize_text_chunk_similarities" in tasks:
             await asyncio.to_thread(update_graph, graph)
             json_obj = {'api_name': 'post_processing/update_similarity_graph', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))}
-            logger.log_struct(json_obj)
             logging.info(f'Updated KNN Graph')
 
         if "enable_hybrid_search_and_fulltext_search_in_bloom" in tasks:
@@ -280,11 +279,10 @@ async def post_processing(uri=Form(), userName=Form(), password=Form(), database
         if os.environ.get('ENTITY_EMBEDDING','False').upper()=="TRUE" and "materialize_entity_similarities" in tasks:
             await asyncio.to_thread(create_entity_embedding, graph)
             json_obj = {'api_name': 'post_processing/create_entity_embedding', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))}
-            logger.log_struct(json_obj)
             logging.info(f'Entity Embeddings created')
 
         if "enable_communities" in tasks:
-            model = "openai_gpt_4o"
+            model = "openai-gpt-4o"
             await asyncio.to_thread(create_communities, uri, userName, password, database,model)
             josn_obj = {'api_name': 'post_processing/create_communities', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))}
             logger.log_struct(josn_obj)
 
@@ -376,8 +376,6 @@ async def processing_source(uri, userName, password, database, model, file_name,
       # selected_chunks = []
       is_cancelled_status = False
       job_status = "Completed"
-      node_count = 0
-      rel_count = 0
       for i in range(0, len(chunkId_chunkDoc_list), update_graph_chunk_processed):
         select_chunks_upto = i+update_graph_chunk_processed
         logging.info(f'Selected Chunks upto: {select_chunks_upto}')
@@ -406,9 +404,14 @@ async def processing_source(uri, userName, password, database, model, file_name,
           obj_source_node.file_name = file_name
           obj_source_node.updated_at = end_time
           obj_source_node.processing_time = processed_time
-          obj_source_node.node_count = node_count
-          obj_source_node.processed_chunk = select_chunks_upto
-          obj_source_node.relationship_count = rel_count
+          obj_source_node.processed_chunk = select_chunks_upto+select_chunks_with_retry
+          if retry_condition == START_FROM_BEGINNING:
+            result = graph.query(QUERY_TO_GET_NODES_AND_RELATIONS_OF_A_DOCUMENT, params={"filename":file_name})
+            obj_source_node.node_count = result[0]['nodes']
+            obj_source_node.relationship_count = result[0]['rels']
+          else:  
+            obj_source_node.node_count = node_count
+            obj_source_node.relationship_count = rel_count
           graphDb_data_Access.update_source_node(obj_source_node)
 
       result = graphDb_data_Access.get_current_status_document_node(file_name)
 
@@ -5,7 +5,6 @@
 import pandas as pd
 from datetime import datetime as dt
 from dotenv import load_dotenv
-
 from score import *
 from src.main import *
 from src.QA_integration import QA_RAG
@@ -26,17 +25,28 @@
 graph = create_graph_database_connection(URI,USERNAME,PASSWORD,DATABASE)
 
 def create_source_node_local(graph, model, file_name):
-    """Creates a source node for a local file."""
-    source_node = sourceNode()
-    source_node.file_name = file_name
-    source_node.file_type = 'pdf'
-    source_node.file_size = '1087'
-    source_node.file_source = 'local file'
-    source_node.model = model
-    source_node.created_at = dt.now()
-    graphDB_data_Access = graphDBdataAccess(graph)
-    graphDB_data_Access.create_source_node(source_node)
-    return source_node
+   """Creates a source node for a local file."""
+   source_node = sourceNode()
+   source_node.file_name = file_name
+   source_node.file_type = 'pdf'
+   source_node.file_size = '1087'
+   source_node.file_source = 'local file'
+   source_node.model = model
+   source_node.created_at = dt.now()
+   graphDB_data_Access = graphDBdataAccess(graph)
+   graphDB_data_Access.create_source_node(source_node)
+   return source_node
+
+def delete_extracted_files(file_path):
+    """Delete the extracted files once extraction process is completed"""
+    try:
+        if os.path.exists(file_path):
+            os.remove(file_path)
+            logging.info(f"Deleted file:{file_path}")
+        else:
+            logging.warning(f"File not found for deletion: {file_path}")
+    except Exception as e:
+        logging.error(f"Failed to delete file {file_path}. Error: {e}")
 
 def delete_extracted_files(file_path):
     """Delete the extracted files once extraction process is completed"""
@@ -50,19 +60,19 @@ def delete_extracted_files(file_path):
         logging.error(f"Failed to delete file {file_path}. Error: {e}")
 
 def test_graph_from_file_local(model_name):
-    """Test graph creation from a local file."""
-    file_name = 'About Amazon.pdf'
-    shutil.copyfile('/workspaces/llm-graph-builder/backend/files/About Amazon.pdf',
-                    os.path.join(MERGED_DIR, file_name))
+   """Test graph creation from a local file."""
+   file_name = 'About Amazon.pdf'
+   shutil.copyfile('/workspaces/llm-graph-builder/backend/files/About Amazon.pdf',
+                   os.path.join(MERGED_DIR, file_name))
 
-    create_source_node_local(graph, model_name, file_name)
-    merged_file_path = os.path.join(MERGED_DIR, file_name)
+   create_source_node_local(graph, model_name, file_name)
+   merged_file_path = os.path.join(MERGED_DIR, file_name)
 
-    local_file_result = extract_graph_from_file_local_file(
-        URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', ''
-    )
-    logging.info("Local file processing complete")
-    print(local_file_result)
+   local_file_result = extract_graph_from_file_local_file(
+       URI, USERNAME, PASSWORD, DATABASE, model_name, merged_file_path, file_name, '', '',None
+   )
+   logging.info("Local file processing complete")
+   print(local_file_result)
 
 #    try:
 #        assert local_file_result['status'] == 'Completed'
@@ -75,128 +85,128 @@ def test_graph_from_file_local(model_name):
     # Delete the file after processing
    delete_extracted_files(merged_file_path)
 
-    return local_file_result
+   return local_file_result
 
 def test_graph_from_wikipedia(model_name):
-    """Test graph creation from a Wikipedia page."""
-    wiki_query = 'https://en.wikipedia.org/wiki/Ram_Mandir'
-    source_type = 'Wikipedia'
-    file_name = "Ram_Mandir"
-    create_source_node_graph_url_wikipedia(graph, model_name, wiki_query, source_type)
-
-    wiki_result = extract_graph_from_file_Wikipedia(URI, USERNAME, PASSWORD, DATABASE, model_name, file_name, 1, 'en', '', '')
-    logging.info("Wikipedia test done")
-    print(wiki_result)
-
-    try:
-        assert wiki_result['status'] == 'Completed'
-        assert wiki_result['nodeCount'] > 0
-        assert wiki_result['relationshipCount'] > 0
-        print("Success")
-    except AssertionError as e:
-        print("Fail: ", e)
-
-    return wiki_result
+   # try:
+       """Test graph creation from a Wikipedia page."""
+       wiki_query = 'https://en.wikipedia.org/wiki/Ram_Mandir'
+       source_type = 'Wikipedia'
+       file_name = "Ram_Mandir"
+       create_source_node_graph_url_wikipedia(graph, model_name, wiki_query, source_type)
+
+       wiki_result = extract_graph_from_file_Wikipedia(URI, USERNAME, PASSWORD, DATABASE, model_name, file_name, 'en',file_name, '', '',None)
+       logging.info("Wikipedia test done")
+       print(wiki_result)
+       try:
+           assert wiki_result['status'] == 'Completed'
+           assert wiki_result['nodeCount'] > 0
+           assert wiki_result['relationshipCount'] > 0
+           print("Success")
+       except AssertionError as e:
+           print("Fail: ", e)
+  
+       return wiki_result
+   # except Exception as ex:
+   #     print(ex)
 
 def test_graph_website(model_name):
-    """Test graph creation from a Website page."""
-     #graph, model, source_url, source_type
-    source_url = 'https://www.amazon.com/'
-    source_type = 'web-url'
-    create_source_node_graph_web_url(graph, model_name, source_url, source_type)
-
-    weburl_result = extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, '', '')
-    logging.info("WebUrl test done")
-    print(weburl_result)
-
-    try:
-        assert weburl_result['status'] == 'Completed'
-        assert weburl_result['nodeCount'] > 0
-        assert weburl_result['relationshipCount'] > 0
-        print("Success")
-    except AssertionError as e:
-        print("Fail: ", e)
-    return weburl_result
-
+   """Test graph creation from a Website page."""
+    #graph, model, source_url, source_type
+   source_url = 'https://www.amazon.com/'
+   source_type = 'web-url'
+   file_name = []
+   create_source_node_graph_web_url(graph, model_name, source_url, source_type)
+
+   weburl_result = extract_graph_from_web_page(URI, USERNAME, PASSWORD, DATABASE, model_name, source_url,file_name, '', '',None)
+   logging.info("WebUrl test done")
+   print(weburl_result)
+
+   try:
+       assert weburl_result['status'] == 'Completed'
+       assert weburl_result['nodeCount'] > 0
+       assert weburl_result['relationshipCount'] > 0
+       print("Success")
+   except AssertionError as e:
+       print("Fail: ", e)
+   return weburl_result
 
 def test_graph_from_youtube_video(model_name):
-    """Test graph creation from a YouTube video."""
-    source_url = 'https://www.youtube.com/watch?v=T-qy-zPWgqA'
-    source_type = 'youtube'
-
-    create_source_node_graph_url_youtube(graph, model_name, source_url, source_type)
-    youtube_result = extract_graph_from_file_youtube(
-        URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, '', ''
-    )
-    logging.info("YouTube Video test done")
-    print(youtube_result)
-
-    try:
-        assert youtube_result['status'] == 'Completed'
-        assert youtube_result['nodeCount'] > 1
-        assert youtube_result['relationshipCount'] > 1
-        print("Success")
-    except AssertionError as e:
-        print("Failed: ", e)
-
-    return youtube_result
+   """Test graph creation from a YouTube video."""
+   source_url = 'https://www.youtube.com/watch?v=T-qy-zPWgqA'
+   source_type = 'youtube'
+   create_source_node_graph_url_youtube(graph, model_name, source_url, source_type)
+   youtube_result = extract_graph_from_file_youtube(
+       URI, USERNAME, PASSWORD, DATABASE, model_name, source_url, '', ''
+   )
+   logging.info("YouTube Video test done")
+   print(youtube_result)
+
+   try:
+       assert youtube_result['status'] == 'Completed'
+       assert youtube_result['nodeCount'] > 1
+       assert youtube_result['relationshipCount'] > 1
+       print("Success")
+   except AssertionError as e:
+       print("Failed: ", e)
+
+   return youtube_result
 
 def test_chatbot_qna(model_name, mode='vector'):
-    """Test chatbot QnA functionality for different modes."""
-    QA_n_RAG = QA_RAG(graph, model_name, 'Tell me about amazon', '[]', 1, mode)
-    print(QA_n_RAG)
-    print(len(QA_n_RAG['message']))
-
-    try:
-        assert len(QA_n_RAG['message']) > 20
-        return QA_n_RAG
-        print("Success")
-    except AssertionError as e:
-        print("Failed ", e)
-        return QA_n_RAG
-    
+   """Test chatbot QnA functionality for different modes."""
+   QA_n_RAG = QA_RAG(graph, model_name, 'Tell me about amazon', '[]', 1, mode)
+   print(QA_n_RAG)
+   print(len(QA_n_RAG['message']))
+
+
+   try:
+       assert len(QA_n_RAG['message']) > 20
+       return QA_n_RAG
+       print("Success")
+   except AssertionError as e:
+       print("Failed ", e)
+       return QA_n_RAG
+  
 #Get Test disconnected_nodes list
 def disconected_nodes():
-    #graph = create_graph_database_connection(uri, userName, password, database)
-    graphDb_data_Access = graphDBdataAccess(graph)
-    nodes_list, total_nodes = graphDb_data_Access.list_unconnected_nodes()
-    print(nodes_list[0]["e"]["elementId"])
-    status = "False"
-
-    if total_nodes['total']>0:
-        status = "True"
-    else:
-        status = "False"
-
-    return nodes_list[0]["e"]["elementId"], status
-    
+   #graph = create_graph_database_connection(uri, userName, password, database)
+   graphDb_data_Access = graphDBdataAccess(graph)
+   nodes_list, total_nodes = graphDb_data_Access.list_unconnected_nodes()
+   print(nodes_list[0]["e"]["elementId"])
+   status = "False"
+   if total_nodes['total']>0:
+       status = "True"
+   else:
+       status = "False"
+   return nodes_list[0]["e"]["elementId"], status
+  
 #Test Delete delete_disconnected_nodes list
 def delete_disconected_nodes(lst_element_id):
-    print(f'disconnect elementid list {lst_element_id}')
-    #graph = create_graph_database_connection(uri, userName, password, database)
-    graphDb_data_Access = graphDBdataAccess(graph)
-    result = graphDb_data_Access.delete_unconnected_nodes(json.dumps(lst_element_id))
-    print(f'delete disconnect api result {result}')
-    if not result:
-        return "True"
-    else:
-        return "False"
+   print(f'disconnect elementid list {lst_element_id}')
+   #graph = create_graph_database_connection(uri, userName, password, database)
+   graphDb_data_Access = graphDBdataAccess(graph)
+   result = graphDb_data_Access.delete_unconnected_nodes(json.dumps(lst_element_id))
+   print(f'delete disconnect api result {result}')
+   if not result:
+       return "True"
+   else:
+       return "False"
 
 #Test Get Duplicate_nodes
 def get_duplicate_nodes():
-        #graph = create_graph_database_connection(uri, userName, password, database)
-        graphDb_data_Access = graphDBdataAccess(graph)
-        nodes_list, total_nodes = graphDb_data_Access.get_duplicate_nodes_list()
-        if total_nodes['total']>0:
-            return "True"
-        else:
-            return "False"
-        
+       #graph = create_graph_database_connection(uri, userName, password, database)
+       graphDb_data_Access = graphDBdataAccess(graph)
+       nodes_list, total_nodes = graphDb_data_Access.get_duplicate_nodes_list()
+       if total_nodes['total']>0:
+           return "True"
+       else:
+           return "False"
+      
 #Test populate_graph_schema
 def test_populate_graph_schema_from_text(model):
-    result_schema = populate_graph_schema_from_text('When Amazon was founded in 1993 by creator Jeff Benzos, it was mostly an online bookstore. Initially Amazon’s growth was very slow, not turning a profit until over 7 years after its founding. This was thanks to the great momentum provided by the dot-com bubble.', model, True)
-    print(result_schema)
-    return result_schema
+   result_schema = populate_graph_schema_from_text('When Amazon was founded in 1993 by creator Jeff Benzos, it was mostly an online bookstore. Initially Amazon’s growth was very slow, not turning a profit until over 7 years after its founding. This was thanks to the great momentum provided by the dot-com bubble.', model, True)
+   print(result_schema)
+   return result_schema
 
 
 def run_tests():
@@ -239,10 +249,10 @@ def run_tests():
    # df['test_populate_graph_schema_from_text'] = schma
    df.to_csv(f"Integration_TestResult_{dt.now().strftime('%Y%m%d_%H%M%S')}.csv", index=False)
 
-    # Save error details to CSV
-    df_errors = pd.DataFrame(error_list, columns=['Model', 'Error'])
-    df_errors['execution_date'] = dt.today().strftime('%Y-%m-%d')
-    df_errors.to_csv(f"Error_details_{dt.now().strftime('%Y%m%d_%H%M%S')}.csv", index=False)
+   # Save error details to CSV
+   df_errors = pd.DataFrame(error_list, columns=['Model', 'Error'])
+   df_errors['execution_date'] = dt.today().strftime('%Y-%m-%d')
+   df_errors.to_csv(f"Error_details_{dt.now().strftime('%Y%m%d_%H%M%S')}.csv", index=False)
 
 if __name__ == "__main__":
-    run_tests()
+   run_tests()