From e81e99ed0ab5a44dd2ed5c39047eedbad56d2dfb Mon Sep 17 00:00:00 2001 From: Roland Geider Date: Mon, 27 Nov 2023 20:22:03 +0100 Subject: [PATCH] Use a dataclass to save ingredient information --- .../commands/import-off-products.py | 25 +++--- wger/nutrition/models/ingredient.py | 7 +- wger/nutrition/off.py | 80 +++++++++++++------ wger/nutrition/tests/test_off.py | 64 +++++++++------ 4 files changed, 112 insertions(+), 64 deletions(-) diff --git a/wger/nutrition/management/commands/import-off-products.py b/wger/nutrition/management/commands/import-off-products.py index bc263a830..e9475d37b 100644 --- a/wger/nutrition/management/commands/import-off-products.py +++ b/wger/nutrition/management/commands/import-off-products.py @@ -25,7 +25,6 @@ from wger.nutrition.models import Ingredient from wger.nutrition.off import extract_info_from_off - logger = logging.getLogger(__name__) @@ -57,8 +56,8 @@ def add_arguments(self, parser): dest='mode', type=str, help='Script mode, "insert" or "update". Insert will insert the ingredients as new ' - 'entries in the database, while update will try to update them if they are ' - 'already present. Deault: insert' + 'entries in the database, while update will try to update them if they are ' + 'already present. Deault: insert' ) parser.add_argument( '--completeness', @@ -67,7 +66,7 @@ def add_arguments(self, parser): dest='completeness', type=float, help='Completeness threshold for importing the products. Products in OFF have ' - 'completeness score that ranges from 0 to 1.1. Default: 0.7' + 'completeness score that ranges from 0 to 1.1. Default: 0.7' ) def handle(self, **options): @@ -115,17 +114,24 @@ def handle(self, **options): ingredient_data = extract_info_from_off(product, languages[product['lang']]) except KeyError as e: # self.stdout.write(f'--> KeyError while extracting info from OFF: {e}') + # self.stdout.write( + # '***********************************************************************************************') + # self.stdout.write( + # '***********************************************************************************************') + # self.stdout.write( + # '***********************************************************************************************') + # pprint(product) # self.stdout.write(f'--> Product: {product}') counter['skipped'] += 1 continue # Some products have no name or name is too long, skipping - if not ingredient_data['name']: + if not ingredient_data.name: # self.stdout.write('--> Ingredient has no name field') counter['skipped'] += 1 continue - if not ingredient_data['common_name']: + if not ingredient_data.common_name: # self.stdout.write('--> Ingredient has no common name field') counter['skipped'] += 1 continue @@ -133,7 +139,7 @@ def handle(self, **options): # # Add entries as new products if self.mode == Mode.INSERT: - bulk_update_bucket.append(Ingredient(**ingredient_data)) + bulk_update_bucket.append(Ingredient(**ingredient_data.dict())) if len(bulk_update_bucket) > self.bulk_size: try: Ingredient.objects.bulk_create(bulk_update_bucket) @@ -165,7 +171,8 @@ def handle(self, **options): # one. While this might not be the most efficient query (there will always # be a SELECT first), it's ok because this script is run very rarely. obj, created = Ingredient.objects.update_or_create( - code=ingredient_data['code'], defaults=ingredient_data + code=ingredient_data.code, + defaults=ingredient_data.dict(), ) if created: @@ -177,7 +184,7 @@ def handle(self, **options): except Exception as e: self.stdout.write('--> Error while performing update_or_create') - self.stdout.write(e) + self.stdout.write(str(e)) counter['error'] += 1 continue diff --git a/wger/nutrition/models/ingredient.py b/wger/nutrition/models/ingredient.py index 36b312140..c84523b14 100644 --- a/wger/nutrition/models/ingredient.py +++ b/wger/nutrition/models/ingredient.py @@ -65,7 +65,6 @@ # Local from .ingredient_category import IngredientCategory - logger = logging.getLogger(__name__) @@ -481,13 +480,13 @@ def fetch_ingredient_from_off(cls, code: str): except KeyError: return None - if not ingredient_data['name']: + if not ingredient_data.name: return - if not ingredient_data['common_name']: + if not ingredient_data.common_name: return - ingredient = cls(**ingredient_data) + ingredient = cls(**ingredient_data.dict()) ingredient.save() logger.info(f'Ingredient found and saved to local database: {ingredient.uuid}') return ingredient diff --git a/wger/nutrition/off.py b/wger/nutrition/off.py index c033935bc..df713f974 100644 --- a/wger/nutrition/off.py +++ b/wger/nutrition/off.py @@ -13,13 +13,15 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from dataclasses import dataclass, asdict +from typing import Optional + # wger from wger.nutrition.consts import KJ_PER_KCAL from wger.nutrition.models import Source from wger.utils.constants import ODBL_LICENSE_ID from wger.utils.models import AbstractSubmissionModel - OFF_REQUIRED_TOP_LEVEL = [ 'product_name', 'code', @@ -28,12 +30,38 @@ OFF_REQUIRED_NUTRIMENTS = [ 'proteins_100g', 'carbohydrates_100g', - 'sugars_100g', 'fat_100g', 'saturated-fat_100g', ] +@dataclass +class IngredientData: + name: str + language_id: int + energy: float + protein: float + carbohydrates: float + carbohydrates_sugar: float + fat: float + fat_saturated: float + fibres: Optional[float] + sodium: Optional[float] + code: str + source_name: str + source_url: str + common_name: str + brand: str + status: str + license_id: int + license_author: str + license_title: str + license_object_url: str + + def dict(self): + return asdict(self) + + def extract_info_from_off(product_data, language: int): if not all(req in product_data for req in OFF_REQUIRED_TOP_LEVEL): raise KeyError('Missing required top-level key') @@ -63,9 +91,9 @@ def extract_info_from_off(product_data, language: int): code = product_data['code'] protein = product_data['nutriments']['proteins_100g'] carbs = product_data['nutriments']['carbohydrates_100g'] - sugars = product_data['nutriments']['sugars_100g'] + sugars = product_data['nutriments'].get('sugars_100g', 0) fat = product_data['nutriments']['fat_100g'] - saturated = product_data['nutriments']['saturated-fat_100g'] + saturated = product_data['nutriments'].get('saturated-fat_100g', 0) # these are optional sodium = product_data['nutriments'].get('sodium_100g', None) @@ -78,25 +106,25 @@ def extract_info_from_off(product_data, language: int): authors = ', '.join(product_data.get('editors_tags', ['open food facts'])) object_url = f'https://world.openfoodfacts.org/product/{code}/' - return { - 'name': name, - 'language_id': language, - 'energy': energy, - 'protein': protein, - 'carbohydrates': carbs, - 'carbohydrates_sugar': sugars, - 'fat': fat, - 'fat_saturated': saturated, - 'fibres': fibre, - 'sodium': sodium, - 'code': code, - 'source_name': source_name, - 'source_url': source_url, - 'common_name': common_name, - 'brand': brand, - 'status': AbstractSubmissionModel.STATUS_ACCEPTED, - 'license_id': ODBL_LICENSE_ID, - 'license_author': authors, - 'license_title': name, - 'license_object_url': object_url - } + return IngredientData( + name=name, + language_id=language, + energy=energy, + protein=protein, + carbohydrates=carbs, + carbohydrates_sugar=sugars, + fat=fat, + fat_saturated=saturated, + fibres=fibre, + sodium=sodium, + code=code, + source_name=source_name, + source_url=source_url, + common_name=common_name, + brand=brand, + status=AbstractSubmissionModel.STATUS_ACCEPTED, + license_id=ODBL_LICENSE_ID, + license_author=authors, + license_title=name, + license_object_url=object_url + ) diff --git a/wger/nutrition/tests/test_off.py b/wger/nutrition/tests/test_off.py index 07532df99..314511724 100644 --- a/wger/nutrition/tests/test_off.py +++ b/wger/nutrition/tests/test_off.py @@ -17,7 +17,10 @@ from django.test import SimpleTestCase # wger -from wger.nutrition.off import extract_info_from_off +from wger.nutrition.off import ( + extract_info_from_off, + IngredientData +) from wger.utils.constants import ODBL_LICENSE_ID from wger.utils.models import AbstractSubmissionModel @@ -54,30 +57,30 @@ def test_regular_response(self): Test that the function can read the regular case """ result = extract_info_from_off(self.off_data1, 1) - test = { - 'name': 'Foo with chocolate', - 'language_id': 1, - 'energy': 120, - 'protein': 10, - 'carbohydrates': 20, - 'carbohydrates_sugar': 30, - 'fat': 40, - 'fat_saturated': 11, - 'fibres': None, - 'sodium': 5, - 'code': '1234', - 'source_name': 'Open Food Facts', - 'source_url': 'https://world.openfoodfacts.org/api/v2/product/1234.json', - 'common_name': 'Foo with chocolate, 250g package', - 'brand': 'The bar company', - 'status': AbstractSubmissionModel.STATUS_ACCEPTED, - 'license_id': ODBL_LICENSE_ID, - 'license_author': 'open food facts, MrX', - 'license_title': 'Foo with chocolate', - 'license_object_url': 'https://world.openfoodfacts.org/product/1234/' - } + data = IngredientData( + name='Foo with chocolate', + language_id=1, + energy=120, + protein=10, + carbohydrates=20, + carbohydrates_sugar=30, + fat=40, + fat_saturated=11, + fibres=None, + sodium=5, + code='1234', + source_name='Open Food Facts', + source_url='https://world.openfoodfacts.org/api/v2/product/1234.json', + common_name='Foo with chocolate, 250g package', + brand='The bar company', + status=AbstractSubmissionModel.STATUS_ACCEPTED, + license_id=ODBL_LICENSE_ID, + license_author='open food facts, MrX', + license_title='Foo with chocolate', + license_object_url='https://world.openfoodfacts.org/product/1234/' + ) - self.assertDictEqual(result, test) + self.assertEqual(result, data) def test_convert_kj(self): """ @@ -90,7 +93,7 @@ def test_convert_kj(self): result = extract_info_from_off(self.off_data1, 1) # 120 / KJ_PER_KCAL - self.assertAlmostEqual(result['energy'], 28.6806, 3) + self.assertAlmostEqual(result.energy, 28.6806, 3) def test_no_energy(self): """ @@ -99,3 +102,14 @@ def test_no_energy(self): del self.off_data1['nutriments']['energy-kcal_100g'] self.assertRaises(KeyError, extract_info_from_off, self.off_data1, 1) + + def test_no_sugar_or_saturated_fat(self): + """ + No sugar or saturated fat available + """ + del self.off_data1['nutriments']['sugars_100g'] + del self.off_data1['nutriments']['saturated-fat_100g'] + result = extract_info_from_off(self.off_data1, 1) + + self.assertEqual(result.carbohydrates_sugar, 0) + self.assertEqual(result.fat_saturated, 0)