tests: add integration tests for s3

Author: quitrk

integration/main.py

@@ -0,0 +1,19 @@
+import asyncio
+
+from .s3 import run as s3_run
+
+async def main():
+    success = True
+
+    try:
+        # this will test integration with aioboto3
+        await s3_run()
+    except Exception as e:
+        print(f'Error: {e}')
+        success = False
+
+    if not success:
+        raise Exception('Integration tests failed')
+
+
+asyncio.run(main())

integration/s3.py

@@ -0,0 +1,34 @@
+import os
+
+async def run():
+    from skynet.modules.ttt.s3 import S3
+
+    s3 = S3()
+
+    file = './test_upload.txt'
+    # create a test file
+    with open(file, 'w') as f:
+        f.write('This is a test file for S3 integration.')
+
+    # # # Test upload
+    await s3.upload_file(file)
+
+    # # delete the local test file
+    os.remove(file)
+
+    # # # Test download
+    await s3.download_file(file)
+
+    # # # Check if the file exists after download
+    if not os.path.exists(file):
+        raise Exception('Downloaded file does not exist')
+
+    # # # Test delete
+    await s3.delete_file(file)
+    os.remove(file)
+
+    # # # Check if the file is deleted from S3
+    await s3.download_file(file)
+
+    if os.path.exists(file):
+        raise Exception('File was not deleted from S3')

poetry.lock

@@ -26,7 +26,7 @@ optional = true
 vllm = "0.7.3"
 
 [tool.poetry.dependencies]
-aioboto3 = "14.3.0"
+aioboto3 = "15.0.0"
 aiofiles = "24.1.0"
 aiohttp = "3.11.18"
 async-lru = "2.0.5"

pyproject.toml

@@ -1,5 +1,5 @@
-aioboto3==14.3.0 ; python_version == "3.11"
-aiobotocore==2.22.0 ; python_version == "3.11"
+aioboto3==15.0.0 ; python_version == "3.11"
+aiobotocore==2.23.0 ; python_version == "3.11"
 aiofiles==24.1.0 ; python_version == "3.11"
 aiohappyeyeballs==2.6.1 ; python_version == "3.11"
 aiohttp==3.11.18 ; python_version == "3.11"
@@ -15,8 +15,8 @@ attrs==25.3.0 ; python_version == "3.11"
 av==12.3.0 ; python_version == "3.11"
 beautifulsoup4==4.13.4 ; python_version == "3.11"
 blake3==1.0.5 ; python_version == "3.11"
-boto3==1.37.3 ; python_version == "3.11"
-botocore==1.37.3 ; python_version == "3.11"
+boto3==1.38.27 ; python_version == "3.11"
+botocore==1.38.27 ; python_version == "3.11"
 certifi==2025.4.26 ; python_version == "3.11"
 cffi==1.17.1 ; python_version == "3.11"
 charset-normalizer==3.4.2 ; python_version == "3.11"
@@ -159,7 +159,7 @@ requests==2.32.3 ; python_version == "3.11"
 rich-toolkit==0.14.6 ; python_version == "3.11"
 rich==14.0.0 ; python_version == "3.11"
 rpds-py==0.25.1 ; python_version == "3.11"
-s3transfer==0.11.3 ; python_version == "3.11"
+s3transfer==0.13.0 ; python_version == "3.11"
 safetensors==0.5.3 ; python_version == "3.11"
 scikit-learn==1.6.1 ; python_version == "3.11"
 scipy==1.15.3 ; python_version == "3.11"

requirements-vllm.txt

@@ -1,5 +1,5 @@
-aioboto3==14.3.0 ; python_version == "3.11"
-aiobotocore==2.22.0 ; python_version == "3.11"
+aioboto3==15.0.0 ; python_version == "3.11"
+aiobotocore==2.23.0 ; python_version == "3.11"
 aiofiles==24.1.0 ; python_version == "3.11"
 aiohappyeyeballs==2.6.1 ; python_version == "3.11"
 aiohttp==3.11.18 ; python_version == "3.11"
@@ -12,8 +12,8 @@ async-timeout==5.0.1 ; python_version == "3.11" and python_full_version < "3.11.
 attrs==25.3.0 ; python_version == "3.11"
 av==12.3.0 ; python_version == "3.11"
 beautifulsoup4==4.13.4 ; python_version == "3.11"
-boto3==1.37.3 ; python_version == "3.11"
-botocore==1.37.3 ; python_version == "3.11"
+boto3==1.38.27 ; python_version == "3.11"
+botocore==1.38.27 ; python_version == "3.11"
 certifi==2025.4.26 ; python_version == "3.11"
 cffi==1.17.1 ; python_version == "3.11"
 charset-normalizer==3.4.2 ; python_version == "3.11"
@@ -112,7 +112,7 @@ redis==6.1.0 ; python_version == "3.11"
 regex==2024.11.6 ; python_version == "3.11"
 requests-toolbelt==1.0.0 ; python_version == "3.11"
 requests==2.32.3 ; python_version == "3.11"
-s3transfer==0.11.3 ; python_version == "3.11"
+s3transfer==0.13.0 ; python_version == "3.11"
 safetensors==0.5.3 ; python_version == "3.11"
 scikit-learn==1.6.1 ; python_version == "3.11"
 scipy==1.15.3 ; python_version == "3.11"

requirements.txt

@@ -1,3 +1,4 @@
+import aiofiles.os
 from botocore.config import Config
 
 from skynet.env import (
@@ -32,6 +33,7 @@ async def download_file(self, filename):
                     log.info(f'Downloaded file from S3: {filename}')
         except Exception as e:
             log.error(f'Failed to download file {filename} from S3: {e}')
+            await aiofiles.os.remove(filename)
 
     async def upload_file(self, filename):
         try: