Skip to content

Commit

Permalink
Use a dataclass to save ingredient information
Browse files Browse the repository at this point in the history
  • Loading branch information
@rolandgeider
rolandgeider committed Dec 5, 2023
1 parent 53a8de5 commit e81e99e
Show file tree
Hide file tree
Showing 4 changed files with 112 additions and 64 deletions.
25 changes: 16 additions & 9 deletions wger/nutrition/management/commands/import-off-products.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
from wger.nutrition.models import Ingredient
from wger.nutrition.off import extract_info_from_off


logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -57,8 +56,8 @@ def add_arguments(self, parser):
dest='mode',
type=str,
help='Script mode, "insert" or "update". Insert will insert the ingredients as new '
'entries in the database, while update will try to update them if they are '
'already present. Deault: insert'
'entries in the database, while update will try to update them if they are '
'already present. Deault: insert'
)
parser.add_argument(
'--completeness',
Expand All @@ -67,7 +66,7 @@ def add_arguments(self, parser):
dest='completeness',
type=float,
help='Completeness threshold for importing the products. Products in OFF have '
'completeness score that ranges from 0 to 1.1. Default: 0.7'
'completeness score that ranges from 0 to 1.1. Default: 0.7'
)

def handle(self, **options):
Expand Down Expand Up @@ -115,25 +114,32 @@ def handle(self, **options):
ingredient_data = extract_info_from_off(product, languages[product['lang']])
except KeyError as e:
# self.stdout.write(f'--> KeyError while extracting info from OFF: {e}')
# self.stdout.write(
# '***********************************************************************************************')
# self.stdout.write(
# '***********************************************************************************************')
# self.stdout.write(
# '***********************************************************************************************')
# pprint(product)
# self.stdout.write(f'--> Product: {product}')
counter['skipped'] += 1
continue

# Some products have no name or name is too long, skipping
if not ingredient_data['name']:
if not ingredient_data.name:
# self.stdout.write('--> Ingredient has no name field')
counter['skipped'] += 1
continue

if not ingredient_data['common_name']:
if not ingredient_data.common_name:
# self.stdout.write('--> Ingredient has no common name field')
counter['skipped'] += 1
continue

#
# Add entries as new products
if self.mode == Mode.INSERT:
bulk_update_bucket.append(Ingredient(**ingredient_data))
bulk_update_bucket.append(Ingredient(**ingredient_data.dict()))
if len(bulk_update_bucket) > self.bulk_size:
try:
Ingredient.objects.bulk_create(bulk_update_bucket)
Expand Down Expand Up @@ -165,7 +171,8 @@ def handle(self, **options):
# one. While this might not be the most efficient query (there will always
# be a SELECT first), it's ok because this script is run very rarely.
obj, created = Ingredient.objects.update_or_create(
code=ingredient_data['code'], defaults=ingredient_data
code=ingredient_data.code,
defaults=ingredient_data.dict(),
)

if created:
Expand All @@ -177,7 +184,7 @@ def handle(self, **options):

except Exception as e:
self.stdout.write('--> Error while performing update_or_create')
self.stdout.write(e)
self.stdout.write(str(e))
counter['error'] += 1
continue

Expand Down
7 changes: 3 additions & 4 deletions wger/nutrition/models/ingredient.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,6 @@
# Local
from .ingredient_category import IngredientCategory


logger = logging.getLogger(__name__)


Expand Down Expand Up @@ -481,13 +480,13 @@ def fetch_ingredient_from_off(cls, code: str):
except KeyError:
return None

if not ingredient_data['name']:
if not ingredient_data.name:
return

if not ingredient_data['common_name']:
if not ingredient_data.common_name:
return

ingredient = cls(**ingredient_data)
ingredient = cls(**ingredient_data.dict())
ingredient.save()
logger.info(f'Ingredient found and saved to local database: {ingredient.uuid}')
return ingredient
80 changes: 54 additions & 26 deletions wger/nutrition/off.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from dataclasses import dataclass, asdict
from typing import Optional

# wger
from wger.nutrition.consts import KJ_PER_KCAL
from wger.nutrition.models import Source
from wger.utils.constants import ODBL_LICENSE_ID
from wger.utils.models import AbstractSubmissionModel


OFF_REQUIRED_TOP_LEVEL = [
'product_name',
'code',
Expand All @@ -28,12 +30,38 @@
OFF_REQUIRED_NUTRIMENTS = [
'proteins_100g',
'carbohydrates_100g',
'sugars_100g',
'fat_100g',
'saturated-fat_100g',
]


@dataclass
class IngredientData:
name: str
language_id: int
energy: float
protein: float
carbohydrates: float
carbohydrates_sugar: float
fat: float
fat_saturated: float
fibres: Optional[float]
sodium: Optional[float]
code: str
source_name: str
source_url: str
common_name: str
brand: str
status: str
license_id: int
license_author: str
license_title: str
license_object_url: str

def dict(self):
return asdict(self)


def extract_info_from_off(product_data, language: int):
if not all(req in product_data for req in OFF_REQUIRED_TOP_LEVEL):
raise KeyError('Missing required top-level key')
Expand Down Expand Up @@ -63,9 +91,9 @@ def extract_info_from_off(product_data, language: int):
code = product_data['code']
protein = product_data['nutriments']['proteins_100g']
carbs = product_data['nutriments']['carbohydrates_100g']
sugars = product_data['nutriments']['sugars_100g']
sugars = product_data['nutriments'].get('sugars_100g', 0)
fat = product_data['nutriments']['fat_100g']
saturated = product_data['nutriments']['saturated-fat_100g']
saturated = product_data['nutriments'].get('saturated-fat_100g', 0)

# these are optional
sodium = product_data['nutriments'].get('sodium_100g', None)
Expand All @@ -78,25 +106,25 @@ def extract_info_from_off(product_data, language: int):
authors = ', '.join(product_data.get('editors_tags', ['open food facts']))
object_url = f'https://world.openfoodfacts.org/product/{code}/'

return {
'name': name,
'language_id': language,
'energy': energy,
'protein': protein,
'carbohydrates': carbs,
'carbohydrates_sugar': sugars,
'fat': fat,
'fat_saturated': saturated,
'fibres': fibre,
'sodium': sodium,
'code': code,
'source_name': source_name,
'source_url': source_url,
'common_name': common_name,
'brand': brand,
'status': AbstractSubmissionModel.STATUS_ACCEPTED,
'license_id': ODBL_LICENSE_ID,
'license_author': authors,
'license_title': name,
'license_object_url': object_url
}
return IngredientData(
name=name,
language_id=language,
energy=energy,
protein=protein,
carbohydrates=carbs,
carbohydrates_sugar=sugars,
fat=fat,
fat_saturated=saturated,
fibres=fibre,
sodium=sodium,
code=code,
source_name=source_name,
source_url=source_url,
common_name=common_name,
brand=brand,
status=AbstractSubmissionModel.STATUS_ACCEPTED,
license_id=ODBL_LICENSE_ID,
license_author=authors,
license_title=name,
license_object_url=object_url
)
64 changes: 39 additions & 25 deletions wger/nutrition/tests/test_off.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,10 @@
from django.test import SimpleTestCase

# wger
from wger.nutrition.off import extract_info_from_off
from wger.nutrition.off import (
extract_info_from_off,
IngredientData
)
from wger.utils.constants import ODBL_LICENSE_ID
from wger.utils.models import AbstractSubmissionModel

Expand Down Expand Up @@ -54,30 +57,30 @@ def test_regular_response(self):
Test that the function can read the regular case
"""
result = extract_info_from_off(self.off_data1, 1)
test = {
'name': 'Foo with chocolate',
'language_id': 1,
'energy': 120,
'protein': 10,
'carbohydrates': 20,
'carbohydrates_sugar': 30,
'fat': 40,
'fat_saturated': 11,
'fibres': None,
'sodium': 5,
'code': '1234',
'source_name': 'Open Food Facts',
'source_url': 'https://world.openfoodfacts.org/api/v2/product/1234.json',
'common_name': 'Foo with chocolate, 250g package',
'brand': 'The bar company',
'status': AbstractSubmissionModel.STATUS_ACCEPTED,
'license_id': ODBL_LICENSE_ID,
'license_author': 'open food facts, MrX',
'license_title': 'Foo with chocolate',
'license_object_url': 'https://world.openfoodfacts.org/product/1234/'
}
data = IngredientData(
name='Foo with chocolate',
language_id=1,
energy=120,
protein=10,
carbohydrates=20,
carbohydrates_sugar=30,
fat=40,
fat_saturated=11,
fibres=None,
sodium=5,
code='1234',
source_name='Open Food Facts',
source_url='https://world.openfoodfacts.org/api/v2/product/1234.json',
common_name='Foo with chocolate, 250g package',
brand='The bar company',
status=AbstractSubmissionModel.STATUS_ACCEPTED,
license_id=ODBL_LICENSE_ID,
license_author='open food facts, MrX',
license_title='Foo with chocolate',
license_object_url='https://world.openfoodfacts.org/product/1234/'
)

self.assertDictEqual(result, test)
self.assertEqual(result, data)

def test_convert_kj(self):
"""
Expand All @@ -90,7 +93,7 @@ def test_convert_kj(self):
result = extract_info_from_off(self.off_data1, 1)

# 120 / KJ_PER_KCAL
self.assertAlmostEqual(result['energy'], 28.6806, 3)
self.assertAlmostEqual(result.energy, 28.6806, 3)

def test_no_energy(self):
"""
Expand All @@ -99,3 +102,14 @@ def test_no_energy(self):
del self.off_data1['nutriments']['energy-kcal_100g']

self.assertRaises(KeyError, extract_info_from_off, self.off_data1, 1)

def test_no_sugar_or_saturated_fat(self):
"""
No sugar or saturated fat available
"""
del self.off_data1['nutriments']['sugars_100g']
del self.off_data1['nutriments']['saturated-fat_100g']
result = extract_info_from_off(self.off_data1, 1)

self.assertEqual(result.carbohydrates_sugar, 0)
self.assertEqual(result.fat_saturated, 0)

0 comments on commit e81e99e

Please sign in to comment.