Use the openfoodfacts skd to download the product images

Author: rolandgeider

wger/nutrition/sync.py

@@ -27,10 +27,14 @@
 
 # Third Party
 import requests
-from openfoodfacts.images import (
-    AWS_S3_BASE_URL,
-    generate_image_path,
+from openfoodfacts import (
+    API,
+    APIVersion,
+    Country,
+    Environment,
+    Flavor,
 )
+from openfoodfacts.images import download_image
 from tqdm import tqdm
 
 # wger
@@ -55,6 +59,7 @@
 from wger.utils.requests import (
     get_paginated,
     wger_headers,
+    wger_user_agent,
 )
 from wger.utils.url import make_uri
 
@@ -135,55 +140,47 @@ def fetch_image_from_off(ingredient: Ingredient):
     - https://openfoodfacts.github.io/openfoodfacts-server/api/how-to-download-images/
     - https://openfoodfacts.github.io/openfoodfacts-server/api/ref-v2/
     """
-    logger.info(f'Trying to fetch image from OFF for {ingredient.name} (UUID: {ingredient.uuid})')
-
-    url = ingredient.source_url + '?fields=images,image_front_url'
-    headers = wger_headers()
-    try:
-        product_data = requests.get(url, headers=headers, timeout=3).json()
-    except requests.JSONDecodeError:
-        logger.warning(f'Could not decode JSON response from {url}')
-        return
-    except requests.ConnectTimeout as e:
-        logger.warning(f'Connection timeout while trying to fetch {url}: {e}')
-        return
-    except requests.ReadTimeout as e:
-        logger.warning(f'Read timeout while trying to fetch {url}: {e}')
-        return
+    logger.info(f'Trying to fetch image from OFF for "{ingredient.name}" ({ingredient.uuid})')
+    off_api = API(
+        user_agent=wger_user_agent(),
+        country=Country.world,
+        flavor=Flavor.off,
+        version=APIVersion.v2,
+        environment=Environment.org,
+    )
+    product_data = off_api.product.get(ingredient.code, fields=['images', 'image_front_url'])
 
-    try:
-        image_url: Optional[str] = product_data['product'].get('image_front_url')
-    except KeyError:
-        logger.info('No "product" key found, exiting...')
+    if product_data is None:
+        logger.info('No product data found for this ingredient')
         return
 
+    image_url: Optional[str] = product_data.get('image_front_url')
     if not image_url:
         logger.info('Product data has no "image_front_url" key')
         return
-    image_data = product_data['product']['images']
+    image_data = product_data['images']
+    if not image_data:
+        logger.info('Product data has no "images" key')
+        return
 
     # Extract the image key from the url:
     # https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg -> "front_en"
-    image_id: str = image_url.rpartition('/')[2].partition('.')[0]
+    image_key: str = image_url.rpartition('/')[2].partition('.')[0]
 
-    # Extract the uploader name
+    # Extract the uploader name and numerical image id
     try:
-        image_id: str = image_data[image_id]['imgid']
+        image_id: str = image_data[image_key]['imgid']
         uploader_name: str = image_data[image_id]['uploader']
     except KeyError as e:
         logger.info('could not load all image information, skipping...', e)
         return
 
-    # Download image from amazon
-    image_s3_url = f'{AWS_S3_BASE_URL}{generate_image_path(ingredient.code, image_id)}'
-    response = requests.get(image_s3_url, headers=headers)
-    if not response.ok:
-        logger.info(f'Could not locate image on AWS! Status code: {response.status_code}')
-        return
+    off_response = download_image(image_url, return_struct=True)
 
     # Save to DB
-    url = (
-        f'https://world.openfoodfacts.org/cgi/product_image.pl?code={ingredient.code}&id={image_id}'
+    url = make_uri(
+        'https://world.openfoodfacts.org/cgi/product_image.pl',
+        query={'code': ingredient.code, 'id': image_id},
     )
     uploader_url = f'https://world.openfoodfacts.org/photographer/{uploader_name}'
     image_data = {
@@ -194,15 +191,15 @@ def fetch_image_from_off(ingredient: Ingredient):
         'license_author_url': uploader_url,
         'license_object_url': url,
         'license_derivative_source_url': '',
-        'size': len(response.content),
+        'size': len(off_response.image_bytes),
     }
     try:
-        Image.from_json(ingredient, response, image_data, generate_uuid=True)
+        Image.from_json(ingredient, off_response.response, image_data, generate_uuid=True)
     # Due to a race condition (e.g. when adding tasks over the search), we might
     # try to save an image to an ingredient that already has one. In that case,
     # just ignore the error
     except IntegrityError:
-        logger.info('Ingredient has already an image, skipping...')
+        logger.debug('Ingredient has already an image, skipping...')
         return
     ingredient.last_image_check = timezone.now()
     ingredient.save()

wger/nutrition/tests/test_tasks.py

@@ -15,10 +15,7 @@
 
 # Standard Library
 import datetime
-from unittest.mock import (
-    ANY,
-    patch,
-)
+from unittest.mock import patch
 
 # Django
 from django.utils import timezone
@@ -37,74 +34,62 @@
 from wger.utils.requests import wger_headers
 
 
-class MockOffResponse:
-    def __init__(self):
-        self.status_code = 200
-        self.ok = True
-        self.content = b'2000'
+mock_off_response = {
+    'image_front_url': 'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',
+    'images': {
+        'front_en': {
+            'imgid': '12345',
+        },
+        '12345': {'uploader': 'Mr Foobar'},
+    },
+}
 
-    # yapf: disable
-    @staticmethod
-    def json():
-        return {
-            "product": {
-                'image_front_url':
-                    'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',
-                'images': {
-                    'front_en': {
-                        'imgid': '12345',
-                    },
-                    '12345': {
-                        'uploader': 'Mr Foobar'
-                    }
-                }
-            },
-        }
-    # yapf: disable
+
+class MockOffImageResponse:
+    image_bytes = [0, 1, 2, 3]
+    response = type('obj', (object,), {'content': b'mock_content'})()
 
 
 class MockWgerApiResponse:
     def __init__(self):
         self.status_code = 200
         self.content = b'2000'
 
-    # yapf: disable
     @staticmethod
     def json():
         return {
-            "count": 1,
-            "next": None,
-            "previous": None,
-            "results": [
+            'count': 1,
+            'next': None,
+            'previous': None,
+            'results': [
                 {
-                    "id": 1,
-                    "uuid": "188324b5-587f-42d7-9abc-d2ca64c73d45",
-                    "ingredient_id": "12345",
-                    "ingredient_uuid": "e9baa8bd-84fc-4756-8d90-5b9739b06cf8",
-                    "image": "http://localhost:8000/media/ingredients/1/188324b5-587f-42d7-9abc-d2ca64c73d45.jpg",
-                    "created": "2023-03-15T23:20:10.969369+01:00",
-                    "last_update": "2023-03-15T23:20:10.969369+01:00",
-                    "size": 20179,
-                    "width": 400,
-                    "height": 166,
-                    "license": 1,
-                    "license_author": "Tester McTest",
-                    "license_author_url": "https://example.com/editors/mcLovin",
-                    "license_title": "",
-                    "license_object_url": "",
-                    "license_derivative_source_url": ""
+                    'id': 1,
+                    'uuid': '188324b5-587f-42d7-9abc-d2ca64c73d45',
+                    'ingredient_id': '12345',
+                    'ingredient_uuid': 'e9baa8bd-84fc-4756-8d90-5b9739b06cf8',
+                    'image': 'http://localhost:8000/media/ingredients/1/188324b5-587f-42d7-9abc-d2ca64c73d45.jpg',
+                    'created': '2023-03-15T23:20:10.969369+01:00',
+                    'last_update': '2023-03-15T23:20:10.969369+01:00',
+                    'size': 20179,
+                    'width': 400,
+                    'height': 166,
+                    'license': 1,
+                    'license_author': 'Tester McTest',
+                    'license_author_url': 'https://example.com/editors/mcLovin',
+                    'license_title': '',
+                    'license_object_url': '',
+                    'license_derivative_source_url': '',
                 }
-            ]
+            ],
         }
-    # yapf: enable
 
 
 class FetchIngredientImageTestCase(WgerTestCase):
     """
     Test fetching an ingredient image
     """
 
-    @patch('requests.get')
+    @patch('openfoodfacts.api.ProductResource.get')
     @patch.object(logger, 'info')
     def test_source(self, mock_logger, mock_request):
         """
@@ -120,7 +105,7 @@ def test_source(self, mock_logger, mock_request):
             mock_request.assert_not_called()
             self.assertEqual(result, None)
 
-    @patch('requests.get')
+    @patch('openfoodfacts.api.ProductResource.get')
     @patch.object(logger, 'info')
     def test_download_off_setting(self, mock_logger, mock_request):
         """
@@ -136,7 +121,7 @@ def test_download_off_setting(self, mock_logger, mock_request):
             mock_request.assert_not_called()
             self.assertEqual(result, None)
 
-    @patch('requests.get', return_value=MockOffResponse())
+    @patch('openfoodfacts.api.ProductResource.get')
     @patch('wger.nutrition.models.Image.from_json')
     @patch.object(logger, 'info')
     def test_last_new_image_date(self, mock_logger, mock_from_json, mock_request):
@@ -160,10 +145,17 @@ def test_last_new_image_date(self, mock_logger, mock_from_json, mock_request):
             mock_request.assert_not_called()
             self.assertEqual(result, None)
 
-    @patch('requests.get', return_value=MockOffResponse())
+    @patch('wger.nutrition.sync.download_image', return_value=MockOffImageResponse)
+    @patch('openfoodfacts.api.ProductResource.get', return_value=mock_off_response)
     @patch('wger.nutrition.models.Image.from_json')
     @patch.object(logger, 'info')
-    def test_last_old_image_date(self, mock_logger, mock_from_json, mock_request):
+    def test_last_old_image_date(
+        self,
+        mock_logger,
+        mock_from_json,
+        mock_request,
+        mock_download_image,
+    ):
         """
         Test that images are fetched if we checked a long time ago
         """
@@ -180,14 +172,27 @@ def test_last_old_image_date(self, mock_logger, mock_from_json, mock_request):
         ):
             result = fetch_ingredient_image(1)
             mock_from_json.assert_called()
+            mock_download_image.assert_called_with(
+                'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',
+                return_struct=True,
+            )
             mock_logger.assert_any_call('Fetching image for ingredient 1')
-            mock_request.assert_called()
+            mock_request.assert_called_with(
+                '1234567890987654321', fields=['images', 'image_front_url']
+            )
             self.assertEqual(result, None)
 
-    @patch('requests.get', return_value=MockOffResponse())
+    @patch('wger.nutrition.sync.download_image', return_value=MockOffImageResponse)
+    @patch('openfoodfacts.api.ProductResource.get', return_value=mock_off_response)
     @patch('wger.nutrition.models.Image.from_json')
     @patch.object(logger, 'info')
-    def test_download_ingredient_off(self, mock_logger, mock_from_json, mock_request):
+    def test_download_ingredient_off(
+        self,
+        mock_logger,
+        mock_from_json,
+        mock_request,
+        mock_download_image,
+    ):
         """
         Test that the image is correctly downloaded
 
@@ -202,23 +207,21 @@ def test_download_ingredient_off(self, mock_logger, mock_from_json, mock_request
         ):
             result = fetch_ingredient_image(1)
 
+            # print(mock_request.mock_calls)
+            mock_request.assert_called_with(
+                '1234567890987654321',
+                fields=['images', 'image_front_url'],
+            )
+            mock_download_image.assert_called_with(
+                'https://images.openfoodfacts.org/images/products/00975957/front_en.5.400.jpg',
+                return_struct=True,
+            )
             mock_logger.assert_any_call('Fetching image for ingredient 1')
             mock_logger.assert_any_call(
-                'Trying to fetch image from OFF for Test ingredient 1 (UUID: '
-                '7908c204-907f-4b1e-ad4e-f482e9769ade)'
+                'Trying to fetch image from OFF for "Test ingredient 1" '
+                '(7908c204-907f-4b1e-ad4e-f482e9769ade)'
             )
             mock_logger.assert_any_call('Image successfully saved')
-
-            # print(mock_request.mock_calls)
-            mock_request.assert_any_call(
-                'https://world.openfoodfacts.org/api/v2/product/5055365635003.json?fields=images,image_front_url',
-                headers=wger_headers(),
-                timeout=ANY,
-            )
-            mock_request.assert_any_call(
-                'https://openfoodfacts-images.s3.eu-west-3.amazonaws.com/data/123/456/789/0987654321/12345.jpg',
-                headers=wger_headers(),
-            )
             mock_from_json.assert_called()
 
             self.assertEqual(result, None)