fix: add empty string case for language metadata (#4062)

Add an empty string edge case for when the element text field is None or
not a string.

most of the diff is `make tidy`

Author: shreyanid

CHANGELOG.md

@@ -1,11 +1,12 @@
-## 0.18.9-dev0
+## 0.18.9
 
 ### Enhancements
 
 ### Features
 - **Convert elements to markdown for output** Added function to convert elements to markdown format for easy viewing.
 
 ### Fixes
+- *Language detection nit** Handle empty text
 
 ## 0.18.8
 

test_unstructured/file_utils/test_filetype.py

@@ -992,7 +992,6 @@ def test_json_content_type_is_disambiguated_for_ndjson():
 
 
 def test_office_files_when_document_archive_has_non_standard_prefix():
-
     predicted_type = detect_filetype(
         file_path=input_path("file_type/test_document_from_office365.docx")
     )

test_unstructured/metrics/test_evaluate.py

@@ -203,7 +203,6 @@ def test_process_document_returns_the_correct_amount_of_values(
 def test_TextExtractionMetricsCalculator_process_document_returns_the_correct_doctype(
     mock_dependencies, calculator_class, output_dirname, source_dirname, path, kwargs
 ):
-
     output_dir = Path(TESTING_FILE_DIR) / output_dirname
     source_dir = Path(TESTING_FILE_DIR) / source_dirname
     mock_calculate_accuracy = mock_dependencies["mock_calculate_accuracy"]
@@ -241,7 +240,6 @@ def test_TextExtractionMetricsCalculator_process_document_returns_the_correct_do
 def test_TableStructureMetricsCalculator_process_document_returns_the_correct_doctype(
     mock_dependencies, calculator_class, output_dirname, source_dirname, path, kwargs
 ):
-
     output_dir = Path(TESTING_FILE_DIR) / output_dirname
     source_dir = Path(TESTING_FILE_DIR) / source_dirname
     calculator = calculator_class(documents_dir=output_dir, ground_truths_dir=source_dir, **kwargs)
@@ -278,7 +276,6 @@ def test_TableStructureMetricsCalculator_process_document_returns_the_correct_do
 def test_ElementTypeMetricsCalculator_process_document_returns_the_correct_doctype(
     mock_dependencies, calculator_class, output_dirname, source_dirname, path, kwargs
 ):
-
     output_dir = Path(TESTING_FILE_DIR) / output_dirname
     source_dir = Path(TESTING_FILE_DIR) / source_dirname
     calculator = calculator_class(documents_dir=output_dir, ground_truths_dir=source_dir, **kwargs)

test_unstructured/partition/common/test_common.py

@@ -331,7 +331,6 @@ def test_normalize_layout_element_bulleted_list():
 
 
 class MockRunOutput:
-
     def __init__(self, returncode, stdout, stderr):
         self.returncode = returncode
         self.stdout = stdout

test_unstructured/partition/common/test_metadata.py

@@ -63,7 +63,6 @@ def but_it_returns_None_when_there_is_no_file_at_that_path(self, tmp_path: pathl
 
 
 class Describe_set_element_hierarchy:
-
     def it_applies_default_ruleset(self):
         elements = [
             Title(element_id="0", text="Title0"),

test_unstructured/partition/pdf_image/test_pdf.py

@@ -1600,7 +1600,6 @@ def _test(result):
 
 
 def test_partition_pdf_with_specified_ocr_agents(mock_ocr_get_instance, mocker):
-
     pdf.partition_pdf(
         filename=example_doc_path("pdf/layout-parser-paper-with-table.pdf"),
         strategy=PartitionStrategy.HI_RES,

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.18.9-dev0"  # pragma: no cover
+__version__ = "0.18.9"  # pragma: no cover

unstructured/metrics/evaluate.py

@@ -780,7 +780,6 @@ def _generate_dataframes(self, rows) -> tuple[pd.DataFrame, pd.DataFrame]:
 
 
 class ObjectDetectionPerClassMetricsCalculator(ObjectDetectionMetricsCalculatorBase):
-
     def __post_init__(self):
         super().__post_init__()
         self.per_class_metric_names: list[str] | None = None

unstructured/metrics/table/table_eval.py

@@ -288,11 +288,13 @@ def process_file(self) -> TableEvaluation:
                 cutoff=self.cutoff,
             )
 
-            table_detection_recall, table_detection_precision, table_detection_f1 = (
-                calculate_table_detection_metrics(
-                    matched_indices=matched_indices,
-                    ground_truth_tables_number=len(ground_truth_table_data),
-                )
+            (
+                table_detection_recall,
+                table_detection_precision,
+                table_detection_f1,
+            ) = calculate_table_detection_metrics(
+                matched_indices=matched_indices,
+                ground_truth_tables_number=len(ground_truth_table_data),
             )
 
             evaluation = TableEvaluation(

unstructured/partition/common/lang.py

@@ -487,7 +487,7 @@ def apply_lang_metadata(
     if not isinstance(elements, list):
         elements = list(elements)
 
-    full_text = " ".join(e.text for e in elements if hasattr(e, "text"))
+    full_text = " ".join(str(e.text) for e in elements if hasattr(e, "text") and e.text)
     detected_languages = detect_languages(text=full_text, languages=languages)
     if (
         detected_languages is not None
@@ -501,7 +501,8 @@ def apply_lang_metadata(
     else:
         for e in elements:
             if hasattr(e, "text"):
-                e.metadata.languages = detect_languages(e.text)
+                text_value = str(e.text) if e.text is not None else ""
+                e.metadata.languages = detect_languages(text_value)
                 yield e
             else:
                 yield e