fix: check ole storage content to differentiate filetypes (#3581)

      ### Summary

Updates the file detection logic for OLE files to check the storage
content of the file to more reliable differentiate between DOC, PPT, XLS
and MSG files. This corrects a bug that caused file type detection to be
incorrect in cases where the `filetype` library guessed and incorrect
MIME type, such as `'application/vnd.ms-excel'` for a `.msg` file.

As part of this work, the `"msg"` extra was removed because the
`python-oxmsg` package is now a base dependency.

### Testing

Using a test `.msg` file that returns `'application/vnd.ms-excel'` from
`filetype.guess_mime`.

```python
from unstructured.file_utils.filetype import detect_filetype

filename = "test-file.msg"
detect_filetype(filename=filename) # result should be FileType.MSG
```

Author: MthwRobinson

.github/workflows/ci.yml

@@ -215,7 +215,7 @@ jobs:
     strategy:
       matrix:
         python-version: ["3.10"]
-        extra: ["csv", "docx", "odt", "markdown", "pypandoc", "msg", "pdf-image", "pptx", "xlsx"]
+        extra: ["csv", "docx", "odt", "markdown", "pypandoc", "pdf-image", "pptx", "xlsx"]
     runs-on: ubuntu-latest
     env:
       NLTK_DATA: ${{ github.workspace }}/nltk_data

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.15.9-dev1
+## 0.15.9
 
 ### Enhancements
 
@@ -8,6 +8,7 @@
 
 ### Fixes
 
+* **Check storage contents for OLE file type detection** Updates `detect_filetype` to check the content of OLE files to more reliable differentiate DOC, PPT, XLS, and MSG files. As part of this, the `"msg"` extra was removed because the `python-oxmsg` package is now a base dependency.
 * **Fix disk space leaks and Windows errors when accessing file.name on a NamedTemporaryFile** Uses of `NamedTemporaryFile(..., delete=False)` and/or uses of `file.name` of NamedTemporaryFiles have been replaced with TemporaryFileDirectory to avoid a known issue: https://docs.python.org/3/library/tempfile.html#tempfile.NamedTemporaryFile
 
 ## 0.15.8

Makefile

@@ -83,10 +83,6 @@ install-pypandoc:
 install-markdown:
 	python3 -m pip install -r requirements/extra-markdown.txt
 
-.PHONY: install-msg
-install-msg:
-	python3 -m pip install -r requirements/extra-msg.txt
-
 .PHONY: install-pdf-image
 install-pdf-image:
 	python3 -m pip install -r requirements/extra-pdf-image.txt
@@ -100,7 +96,7 @@ install-xlsx:
 	python3 -m pip install -r requirements/extra-xlsx.txt
 
 .PHONY: install-all-docs
-install-all-docs: install-base install-csv install-docx install-epub install-odt install-pypandoc install-markdown install-msg install-pdf-image install-pptx install-xlsx
+install-all-docs: install-base install-csv install-docx install-epub install-odt install-pypandoc install-markdown install-pdf-image install-pptx install-xlsx
 
 .PHONY: install-all-ingest
 install-all-ingest:
@@ -343,12 +339,6 @@ test-extra-epub:
 test-extra-markdown:
 	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/test_md.py
 
-.PHONY: test-extra-msg
-test-extra-msg:
-	# NOTE(scanny): exclude attachment test because partitioning attachments requires other extras
-	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/test_msg.py \
-          -k "not test_partition_msg_can_process_attachments"
-
 .PHONY: test-extra-odt
 test-extra-odt:
 	PYTHONPATH=. CI=$(CI) pytest test_unstructured/partition/test_odt.py

requirements/base.in

@@ -21,3 +21,4 @@ unstructured-client
 wrapt
 tqdm
 psutil
+python-oxmsg

requirements/base.txt

@@ -10,7 +10,7 @@ backoff==2.2.1
     # via -r ./base.in
 beautifulsoup4==4.12.3
     # via -r ./base.in
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   httpcore
     #   httpx
@@ -23,7 +23,9 @@ charset-normalizer==3.3.2
     #   requests
     #   unstructured-client
 click==8.1.7
-    # via nltk
+    # via
+    #   nltk
+    #   python-oxmsg
 dataclasses-json==0.6.7
     # via
     #   -r ./base.in
@@ -70,6 +72,8 @@ nltk==3.9.1
     # via -r ./base.in
 numpy==1.26.4
     # via -r ./base.in
+olefile==0.47
+    # via python-oxmsg
 orderly-set==5.2.2
     # via deepdiff
 packaging==24.1
@@ -86,6 +90,8 @@ python-iso639==2024.4.27
     # via -r ./base.in
 python-magic==0.4.27
     # via -r ./base.in
+python-oxmsg==0.0.1
+    # via -r ./base.in
 rapidfuzz==3.9.6
     # via -r ./base.in
 regex==2024.7.24
@@ -120,6 +126,7 @@ typing-extensions==4.12.2
     #   anyio
     #   emoji
     #   pypdf
+    #   python-oxmsg
     #   typing-inspect
     #   unstructured-client
 typing-inspect==0.9.0
@@ -128,7 +135,7 @@ typing-inspect==0.9.0
     #   unstructured-client
 unstructured-client==0.25.5
     # via -r ./base.in
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ././deps/constraints.txt
     #   requests

requirements/dev.txt

@@ -34,7 +34,7 @@ bleach==6.1.0
     # via nbconvert
 build==1.2.1
     # via pip-tools
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./base.txt
     #   -c ./test.txt
@@ -130,7 +130,7 @@ jsonschema[format-nongpl]==3.2.0
     #   jupyter-events
     #   jupyterlab-server
     #   nbformat
-jupyter==1.1.0
+jupyter==1.1.1
     # via -r ./dev.in
 jupyter-client==7.4.9
     # via
@@ -370,7 +370,7 @@ typing-extensions==4.12.2
     #   -c ./test.txt
     #   anyio
     #   ipython
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ././deps/constraints.txt
     #   -c ./base.txt

requirements/extra-msg.in

@@ -10,7 +10,7 @@ anyio==4.4.0
     #   httpx
 astor==0.8.1
     # via paddlepaddle
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./base.txt
     #   httpcore
@@ -170,7 +170,7 @@ typing-extensions==4.12.2
     #   paddlepaddle
 unstructured-paddleocr==2.8.1.0
     # via -r ./extra-paddleocr.in
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ././deps/constraints.txt
     #   -c ./base.txt

requirements/extra-msg.txt

@@ -8,7 +8,7 @@ antlr4-python3-runtime==4.9.3
     # via omegaconf
 cachetools==5.5.0
     # via google-auth
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   -c ./base.txt
     #   requests
@@ -279,7 +279,7 @@ unstructured-inference==0.7.36
     # via -r ./extra-pdf-image.in
 unstructured-pytesseract==0.3.13
     # via -r ./extra-pdf-image.in
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -c ././deps/constraints.txt
     #   -c ./base.txt