From ad70a7db326e34f2e49a8676fe5e4893cd09763d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 21 May 2020 13:32:24 -0300 Subject: [PATCH 01/11] Tox env with no dependencies --- .github/workflows/main.yml | 34 +++++++++++++++++++++++++++------- tests/requirements.txt | 5 ----- tests/test_utils.py | 25 ------------------------- tox.ini | 29 ++++++++++++++++++++++++++--- 4 files changed, 53 insertions(+), 40 deletions(-) delete mode 100644 tests/requirements.txt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e979653..7f9321a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: [push] jobs: checks: - name: Lint, typing, coverage + name: Flake8, typing, black runs-on: ubuntu-latest steps: @@ -27,12 +27,6 @@ jobs: - name: Black run: tox -e black - - name: Generate coverage report - run: tox -e py - - - name: Upload coverage report - run: bash <(curl -s https://codecov.io/bash) - tests-ubuntu: name: "Test: py${{ matrix.python-version }}, Ubuntu" @@ -55,6 +49,32 @@ jobs: - name: Run tests run: tox -e py + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) -cF tests + + + tests-no-deps: + name: "Test: python3.6, Ubuntu, no deps" + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Set up Python 3.6 + uses: actions/setup-python@v1 + with: + python-version: 3.6 + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e no-deps + + - name: Upload coverage report + run: bash <(curl -s https://codecov.io/bash) -cF tests + + tests-other-os: name: "Test: py3.8, ${{ matrix.os }}" runs-on: "${{ matrix.os }}" diff --git a/tests/requirements.txt b/tests/requirements.txt deleted file mode 100644 index 671a10e..0000000 --- a/tests/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -attrs -dataclasses; python_version == "3.6" -pytest-cov>=2.8 -pytest>=5.4 -scrapy>=2.0 diff --git a/tests/test_utils.py b/tests/test_utils.py index 8c4194a..c620610 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -6,10 +6,6 @@ from tests import AttrsItem, DataClassItem, ScrapyItem, ScrapySubclassedItem -def mocked_import(name, *args, **kwargs): - raise ImportError(name) - - class ItemLikeTestCase(unittest.TestCase): def test_false(self): self.assertFalse(is_item(int)) @@ -50,8 +46,6 @@ def test_false(self): self.assertFalse(is_attrs_instance(sum)) self.assertFalse(is_attrs_instance(1234)) self.assertFalse(is_attrs_instance(object())) - self.assertFalse(is_attrs_instance(ScrapyItem())) - self.assertFalse(is_attrs_instance(ScrapySubclassedItem())) self.assertFalse(is_attrs_instance("a string")) self.assertFalse(is_attrs_instance(b"some bytes")) self.assertFalse(is_attrs_instance({"a": "dict"})) @@ -60,11 +54,6 @@ def test_false(self): self.assertFalse(is_attrs_instance({"a", "set"})) self.assertFalse(is_attrs_instance(AttrsItem)) - @unittest.skipIf(not AttrsItem, "attrs module is not available") - @mock.patch("builtins.__import__", mocked_import) - def test_module_not_available(self): - self.assertFalse(is_attrs_instance(AttrsItem(name="asdf", value=1234))) - @unittest.skipIf(not AttrsItem, "attrs module is not available") def test_true(self): self.assertTrue(is_attrs_instance(AttrsItem())) @@ -77,9 +66,6 @@ def test_false(self): self.assertFalse(is_dataclass_instance(sum)) self.assertFalse(is_dataclass_instance(1234)) self.assertFalse(is_dataclass_instance(object())) - self.assertFalse(is_dataclass_instance(ScrapyItem())) - self.assertFalse(is_dataclass_instance(AttrsItem())) - self.assertFalse(is_dataclass_instance(ScrapySubclassedItem())) self.assertFalse(is_dataclass_instance("a string")) self.assertFalse(is_dataclass_instance(b"some bytes")) self.assertFalse(is_dataclass_instance({"a": "dict"})) @@ -88,11 +74,6 @@ def test_false(self): self.assertFalse(is_dataclass_instance({"a", "set"})) self.assertFalse(is_dataclass_instance(DataClassItem)) - @unittest.skipIf(not DataClassItem, "dataclasses module is not available") - @mock.patch("builtins.__import__", mocked_import) - def test_module_not_available(self): - self.assertFalse(is_dataclass_instance(DataClassItem(name="asdf", value=1234))) - @unittest.skipIf(not DataClassItem, "dataclasses module is not available") def test_true(self): self.assertTrue(is_dataclass_instance(DataClassItem())) @@ -105,7 +86,6 @@ def test_false(self): self.assertFalse(is_scrapy_item(sum)) self.assertFalse(is_scrapy_item(1234)) self.assertFalse(is_scrapy_item(object())) - self.assertFalse(is_scrapy_item(AttrsItem())) self.assertFalse(is_scrapy_item("a string")) self.assertFalse(is_scrapy_item(b"some bytes")) self.assertFalse(is_scrapy_item({"a": "dict"})) @@ -114,11 +94,6 @@ def test_false(self): self.assertFalse(is_scrapy_item({"a", "set"})) self.assertFalse(is_scrapy_item(ScrapySubclassedItem)) - @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") - @mock.patch("builtins.__import__", mocked_import) - def test_module_not_available(self): - self.assertFalse(is_scrapy_item(ScrapySubclassedItem(name="asdf", value=1234))) - @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") def test_true(self): self.assertTrue(is_scrapy_item(ScrapyItem())) diff --git a/tox.ini b/tox.ini index 62f80b4..119127e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,23 +1,46 @@ [tox] -envlist = py35,py36,py37,py38,flake8,typing,black +envlist = py35,py36,py37,py38,no-deps,flake8,typing,black [testenv] deps = - -rtests/requirements.txt + pytest>=5.4 + pytest-cov>=2.8 commands = - pytest --verbose --cov=itemadapter --cov-report=term-missing --cov-report=html --cov-report=xml {posargs: itemadapter tests} + pytest --verbose --cov=itemadapter --cov-report=term-missing --cov-report=html --cov-report=xml --cov-append {posargs: itemadapter tests} [testenv:py35] basepython = python3.5 +deps = + {[testenv]deps} + attrs + scrapy>=2.0 [testenv:py36] basepython = python3.6 +deps = + {[testenv]deps} + attrs + dataclasses + scrapy>=2.0 [testenv:py37] basepython = python3.7 +deps = + {[testenv]deps} + attrs + scrapy>=2.0 [testenv:py38] basepython = python3.8 +deps = + {[testenv]deps} + attrs + scrapy>=2.0 + +[testenv:no-deps] +deps = + {[testenv]deps} +basepython = python3.6 [testenv:flake8] basepython = python3.8 From 37e109f35641117d1c45c7928d1f690bfe94e2b4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 2 Jun 2020 14:43:57 -0300 Subject: [PATCH 02/11] Rename tox envs --- .github/workflows/main.yml | 2 +- tox.ini | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f9321a..4396b5e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,7 +47,7 @@ jobs: run: pip install tox - name: Run tests - run: tox -e py + run: tox -e py${{ matrix.python-version }} - name: Upload coverage report run: bash <(curl -s https://codecov.io/bash) -cF tests diff --git a/tox.ini b/tox.ini index cf19fdd..c9275bb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = flake8,typing,black,py35,py36,py37,py38,no-deps +envlist = flake8,typing,black,py3.5,py3.6,py3.7,py3.8,no-deps [testenv] deps = @@ -8,14 +8,14 @@ deps = commands = pytest --verbose --cov=itemadapter --cov-report=term-missing --cov-report=html --cov-report=xml --cov-append {posargs: itemadapter tests} -[testenv:py35] +[testenv:py3.5] basepython = python3.5 deps = {[testenv]deps} attrs scrapy>=2.0 -[testenv:py36] +[testenv:py3.6] basepython = python3.6 deps = {[testenv]deps} @@ -23,14 +23,14 @@ deps = dataclasses scrapy>=2.0 -[testenv:py37] +[testenv:py3.7] basepython = python3.7 deps = {[testenv]deps} attrs scrapy>=2.0 -[testenv:py38] +[testenv:py3.8] basepython = python3.8 deps = {[testenv]deps} From ee65dee019e668e89b57dc8066e5917f9c0dbc00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 11:43:13 +0200 Subject: [PATCH 03/11] Improve the handling of no-extra-dependencies test runs --- .github/workflows/main.yml | 8 +-- tests/__init__.py | 121 ++++++++++++++++++++++++++++++++----- tests/test_adapter.py | 30 ++++++--- tests/test_utils.py | 43 ++++++++----- tox.ini | 44 +++----------- 5 files changed, 168 insertions(+), 78 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4396b5e..aebb77d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,13 +47,13 @@ jobs: run: pip install tox - name: Run tests - run: tox -e py${{ matrix.python-version }} + run: tox -e py - name: Upload coverage report run: bash <(curl -s https://codecov.io/bash) -cF tests - tests-no-deps: + tests-no-extra-deps: name: "Test: python3.6, Ubuntu, no deps" runs-on: ubuntu-latest @@ -63,13 +63,13 @@ jobs: - name: Set up Python 3.6 uses: actions/setup-python@v1 with: - python-version: 3.6 + python-version: [3.5, 3.6, 3.8] - name: Install tox run: pip install tox - name: Run tests - run: tox -e no-deps + run: tox -e no-extra-deps - name: Upload coverage report run: bash <(curl -s https://codecov.io/bash) -cF tests diff --git a/tests/__init__.py b/tests/__init__.py index 988abdd..bd342e0 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,13 +1,20 @@ +import os +import sys +from unittest import skipIf, TestCase as _TestCase + + try: import attr except ImportError: AttrsItem = None else: - - @attr.s - class AttrsItem: - name = attr.ib(default=None, metadata={"serializer": str}) - value = attr.ib(default=None, metadata={"serializer": int}) + if os.environ.get('ITEMADAPTER_NO_EXTRA_DEPS'): + AttrsItem = None + else: + @attr.s + class AttrsItem: + name = attr.ib(default=None, metadata={"serializer": str}) + value = attr.ib(default=None, metadata={"serializer": int}) try: @@ -15,13 +22,19 @@ class AttrsItem: except ImportError: DataClassItem = None else: - DataClassItem = make_dataclass( - "DataClassItem", - [ - ("name", str, field(default_factory=lambda: None, metadata={"serializer": str})), - ("value", int, field(default_factory=lambda: None, metadata={"serializer": int})), - ], - ) + if ( + os.environ.get('ITEMADAPTER_NO_EXTRA_DEPS') + and (3, 6) <= sys.version_info < (3, 7) + ): + DataClassItem = None + else: + DataClassItem = make_dataclass( + "DataClassItem", + [ + ("name", str, field(default_factory=lambda: None, metadata={"serializer": str})), + ("value", int, field(default_factory=lambda: None, metadata={"serializer": int})), + ], + ) try: @@ -30,7 +43,85 @@ class AttrsItem: ScrapyItem = None ScrapySubclassedItem = None else: + if os.environ.get('ITEMADAPTER_NO_EXTRA_DEPS'): + ScrapyItem = None + ScrapySubclassedItem = None + else: + class ScrapySubclassedItem(ScrapyItem): + name = Field(serializer=str) + value = Field(serializer=int) + + +class ImportRaiser: + + def __init__(self, *packages): + self.packages = set(packages) + + def find_spec(self, fullname, path, target=None): + if fullname in self.packages: + raise ImportError + + +class TestCase(_TestCase): + """Custom TestCase subclass which handles disabling installed extra + packages during tests when ITEMADAPTER_NO_EXTRA_DEPS is set, as well as + disabling test cases that require one or more unavailable extra + dependencies. + + This is needed to disable packages that cannot be uninstalled because + pytest depends on them. + """ + + _extra_modules = ('attr', 'scrapy') + + def setUp(self): + super().setUp() + + required_extra_modules = getattr(self, 'required_extra_modules', None) + if required_extra_modules: + requirement_map = { + 'attr': AttrsItem, + 'dataclasses': DataClassItem, + 'scrapy': ScrapyItem, + } + unknown_extra_modules = [module + for module in required_extra_modules + if module not in requirement_map] + if unknown_extra_modules: + raise NotImplementedError( + 'Unknown extra modules: {}'.format(unknown_extra_modules) + ) + unavaliable_extra_modules = [module + for module in required_extra_modules + if not requirement_map[module]] + if unavaliable_extra_modules: + self.skipTest( + 'cannot import; {}'.format( + ', '.join(unavaliable_extra_modules) + ) + ) + + + self._removed_modules = {} + if os.environ.get('ITEMADAPTER_NO_EXTRA_DEPS'): + if (3, 6) <= sys.version_info < (3, 7): + self._extra_modules = self._extra_modules + ('dataclasses',) + sys.meta_path.insert(0, ImportRaiser(*self._extra_modules)) + for package in self._extra_modules: + if package in sys.modules: + self._removed_modules[package] = sys.modules[package] + del sys.modules[package] + + def tearDown(self): + super().tearDown() + + if os.environ.get('ITEMADAPTER_NO_EXTRA_DEPS'): + del sys.meta_path[0] + for package in self._extra_modules: + if package in self._removed_modules: + sys.modules[package] = self._removed_modules[package] + - class ScrapySubclassedItem(ScrapyItem): - name = Field(serializer=str) - value = Field(serializer=int) +requires_attr = skipIf(not AttrsItem, "cannot import attr") +requires_dataclasses = skipIf(not DataClassItem, "cannot import dataclasses") +requires_scrapy = skipIf(not ScrapyItem, "cannot import scrapy") diff --git a/tests/test_adapter.py b/tests/test_adapter.py index 1bc7ac0..f93440a 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -2,18 +2,28 @@ from types import MappingProxyType from typing import KeysView +import pytest + from itemadapter.adapter import ItemAdapter -from tests import AttrsItem, DataClassItem, ScrapySubclassedItem +from tests import ( + AttrsItem, + DataClassItem, + requires_attr, + requires_dataclasses, + requires_scrapy, + ScrapySubclassedItem, + TestCase, +) -class ItemAdapterReprTestCase(unittest.TestCase): +class ItemAdapterReprTestCase(TestCase): def test_repr_dict(self): item = dict(name="asdf") adapter = ItemAdapter(item) self.assertEqual(repr(adapter), "ItemAdapter for type dict: {'name': 'asdf'}") - @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") + @requires_scrapy def test_repr_scrapy_item(self): item = ScrapySubclassedItem(name="asdf", value=1234) adapter = ItemAdapter(item) @@ -22,7 +32,7 @@ def test_repr_scrapy_item(self): "ItemAdapter for type ScrapySubclassedItem: {'name': 'asdf', 'value': 1234}", ) - @unittest.skipIf(not DataClassItem, "dataclasses module is not available") + @requires_dataclasses def test_repr_dataclass(self): item = DataClassItem(name="asdf", value=1234) adapter = ItemAdapter(item) @@ -31,6 +41,7 @@ def test_repr_dataclass(self): "ItemAdapter for type DataClassItem: DataClassItem(name='asdf', value=1234)", ) + @requires_attr def test_repr_attrs(self): item = AttrsItem(name="asdf", value=1234) adapter = ItemAdapter(item) @@ -39,7 +50,7 @@ def test_repr_attrs(self): ) -class ItemAdapterInitError(unittest.TestCase): +class ItemAdapterInitError(TestCase): def test_non_item(self): with self.assertRaises(TypeError): ItemAdapter(ScrapySubclassedItem) @@ -54,6 +65,7 @@ class BaseTestMixin: item_class = None def setUp(self): + super().setUp() if self.item_class is None: raise unittest.SkipTest() @@ -135,7 +147,7 @@ def test_delitem_len_iter(self): del adapter["undefined_field"] -class DictTestCase(unittest.TestCase, BaseTestMixin): +class DictTestCase(TestCase, BaseTestMixin): item_class = dict @@ -158,7 +170,7 @@ def test_field_names_updated(self): self.assertEqual(sorted(field_names), ["name", "value"]) -class ScrapySubclassedItemTestCase(NonDictTestMixin, unittest.TestCase): +class ScrapySubclassedItemTestCase(NonDictTestMixin, TestCase): item_class = ScrapySubclassedItem @@ -169,11 +181,11 @@ def test_get_value_keyerror_item_dict(self): adapter["name"] -class DataClassItemTestCase(NonDictTestMixin, unittest.TestCase): +class DataClassItemTestCase(NonDictTestMixin, TestCase): item_class = DataClassItem -class AttrsItemTestCase(NonDictTestMixin, unittest.TestCase): +class AttrsItemTestCase(NonDictTestMixin, TestCase): item_class = AttrsItem diff --git a/tests/test_utils.py b/tests/test_utils.py index c620610..3168ad4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,12 +1,22 @@ +import os import unittest from unittest import mock from itemadapter.utils import is_item, is_attrs_instance, is_dataclass_instance, is_scrapy_item -from tests import AttrsItem, DataClassItem, ScrapyItem, ScrapySubclassedItem +from tests import ( + AttrsItem, + DataClassItem, + requires_attr, + requires_dataclasses, + requires_scrapy, + ScrapyItem, + ScrapySubclassedItem, + TestCase, +) -class ItemLikeTestCase(unittest.TestCase): +class ItemLikeTestCase(TestCase): def test_false(self): self.assertFalse(is_item(int)) self.assertFalse(is_item(sum)) @@ -26,21 +36,21 @@ def test_false(self): def test_true_dict(self): self.assertTrue(is_item({"a": "dict"})) - @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") + @requires_scrapy def test_true_scrapy(self): self.assertTrue(is_item(ScrapyItem())) self.assertTrue(is_item(ScrapySubclassedItem(name="asdf", value=1234))) - @unittest.skipIf(not DataClassItem, "dataclasses module is not available") + @requires_dataclasses def test_true_dataclass(self): self.assertTrue(is_item(DataClassItem(name="asdf", value=1234))) - @unittest.skipIf(not AttrsItem, "attrs module is not available") + @requires_attr def test_true_attrs(self): self.assertTrue(is_item(AttrsItem(name="asdf", value=1234))) -class AttrsTestCase(unittest.TestCase): +class AttrsTestCase(TestCase): def test_false(self): self.assertFalse(is_attrs_instance(int)) self.assertFalse(is_attrs_instance(sum)) @@ -54,13 +64,13 @@ def test_false(self): self.assertFalse(is_attrs_instance({"a", "set"})) self.assertFalse(is_attrs_instance(AttrsItem)) - @unittest.skipIf(not AttrsItem, "attrs module is not available") + @requires_attr def test_true(self): self.assertTrue(is_attrs_instance(AttrsItem())) self.assertTrue(is_attrs_instance(AttrsItem(name="asdf", value=1234))) -class DataclassTestCase(unittest.TestCase): +class DataclassTestCase(TestCase): def test_false(self): self.assertFalse(is_dataclass_instance(int)) self.assertFalse(is_dataclass_instance(sum)) @@ -74,13 +84,13 @@ def test_false(self): self.assertFalse(is_dataclass_instance({"a", "set"})) self.assertFalse(is_dataclass_instance(DataClassItem)) - @unittest.skipIf(not DataClassItem, "dataclasses module is not available") + @requires_dataclasses def test_true(self): self.assertTrue(is_dataclass_instance(DataClassItem())) self.assertTrue(is_dataclass_instance(DataClassItem(name="asdf", value=1234))) -class ScrapyItemTestCase(unittest.TestCase): +class ScrapyItemTestCase(TestCase): def test_false(self): self.assertFalse(is_scrapy_item(int)) self.assertFalse(is_scrapy_item(sum)) @@ -94,7 +104,7 @@ def test_false(self): self.assertFalse(is_scrapy_item({"a", "set"})) self.assertFalse(is_scrapy_item(ScrapySubclassedItem)) - @unittest.skipIf(not ScrapySubclassedItem, "scrapy module is not available") + @requires_scrapy def test_true(self): self.assertTrue(is_scrapy_item(ScrapyItem())) self.assertTrue(is_scrapy_item(ScrapySubclassedItem())) @@ -107,13 +117,15 @@ def test_true(self): scrapy = None -class ScrapyDeprecatedBaseItemTestCase(unittest.TestCase): +class ScrapyDeprecatedBaseItemTestCase(TestCase): """ - Tests for deprecated classes. These will go away once the upstream classes are removed. + Tests for deprecated classes. These will go away once the upstream classes + are removed. """ + required_extra_modules = ('scrapy',) @unittest.skipIf( - scrapy is None or not hasattr(scrapy.item, "_BaseItem"), + not hasattr(scrapy.item, "_BaseItem"), "scrapy.item._BaseItem not available", ) def test_deprecated_underscore_baseitem(self): @@ -124,7 +136,7 @@ class SubClassed_BaseItem(scrapy.item._BaseItem): self.assertTrue(is_scrapy_item(SubClassed_BaseItem())) @unittest.skipIf( - scrapy is None or not hasattr(scrapy.item, "BaseItem"), + not hasattr(scrapy.item, "BaseItem"), "scrapy.item.BaseItem not available", ) def test_deprecated_baseitem(self): @@ -134,7 +146,6 @@ class SubClassedBaseItem(scrapy.item.BaseItem): self.assertTrue(is_scrapy_item(scrapy.item.BaseItem())) self.assertTrue(is_scrapy_item(SubClassedBaseItem())) - @unittest.skipIf(scrapy is None, "scrapy module is not available") def test_removed_baseitem(self): class MockItemModule: Item = ScrapyItem diff --git a/tox.ini b/tox.ini index c9275bb..26fb2b4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,63 +1,39 @@ [tox] -envlist = flake8,typing,black,py3.5,py3.6,py3.7,py3.8,no-deps +envlist = flake8,typing,black,py,no-extra-deps -[testenv] +[base] deps = pytest>=5.4 pytest-cov>=2.8 -commands = - pytest --verbose --cov=itemadapter --cov-report=term-missing --cov-report=html --cov-report=xml --cov-append {posargs: itemadapter tests} - -[testenv:py3.5] -basepython = python3.5 -deps = - {[testenv]deps} - attrs - scrapy>=2.0 - -[testenv:py3.6] -basepython = python3.6 -deps = - {[testenv]deps} - attrs - dataclasses - scrapy>=2.0 -[testenv:py3.7] -basepython = python3.7 -deps = - {[testenv]deps} - attrs - scrapy>=2.0 - -[testenv:py3.8] -basepython = python3.8 +[testenv] deps = - {[testenv]deps} + {[base]deps} attrs + dataclasses; python_version >= '3.6' and python_version < '3.7' scrapy>=2.0 +commands = + pytest --verbose --cov=itemadapter --cov-report=term-missing --cov-report=html --cov-report=xml --cov-append {posargs: itemadapter tests} -[testenv:no-deps] +[testenv:no-extra-deps] deps = {[testenv]deps} -basepython = python3.6 +setenv = + ITEMADAPTER_NO_EXTRA_DEPS=True [testenv:flake8] -basepython = python3.8 deps = flake8>=3.7.9 commands = flake8 --exclude=.git,.tox,venv* itemadapter tests [testenv:typing] -basepython = python3.8 deps = mypy==0.770 commands = mypy --ignore-missing-imports --follow-imports=skip itemadapter [testenv:black] -basepython = python3.8 deps = black>=19.10b0 commands = From ddf91bb0b09ec5e082f4e0021ecaf85128045fcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 12:37:38 +0200 Subject: [PATCH 04/11] Do not import from itemadapter outside a test method --- tests/__init__.py | 31 +--------------------- tests/attr_utils.py | 19 ++++++++++++++ tests/dataclasses_utils.py | 22 ++++++++++++++++ tests/test_adapter.py | 53 +++++++++++++++++++++++++++++++++----- tests/test_utils.py | 30 +++++++++++++++++++-- 5 files changed, 116 insertions(+), 39 deletions(-) create mode 100644 tests/attr_utils.py create mode 100644 tests/dataclasses_utils.py diff --git a/tests/__init__.py b/tests/__init__.py index 6450eb6..e57d95b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -2,18 +2,14 @@ import sys from unittest import skipIf, TestCase as _TestCase -from itemadapter.adapter import ItemAdapter - try: import attr except ImportError: AttrsItem = None - AttrsItemNested = None else: if os.environ.get("ITEMADAPTER_NO_EXTRA_DEPS"): AttrsItem = None - AttrsItemNested = None else: @attr.s @@ -21,26 +17,14 @@ class AttrsItem: name = attr.ib(default=None, metadata={"serializer": str}) value = attr.ib(default=None, metadata={"serializer": int}) - @attr.s - class AttrsItemNested: - nested = attr.ib(type=AttrsItem) - adapter = attr.ib(type=ItemAdapter) - dict_ = attr.ib(type=dict) - list_ = attr.ib(type=list) - set_ = attr.ib(type=set) - tuple_ = attr.ib(type=tuple) - int_ = attr.ib(type=int) - try: - from dataclasses import make_dataclass, field + from dataclasses import field, make_dataclass except ImportError: DataClassItem = None - DataClassItemNested = None else: if os.environ.get("ITEMADAPTER_NO_EXTRA_DEPS") and (3, 6) <= sys.version_info < (3, 7): DataClassItem = None - DataClassItemNested = None else: DataClassItem = make_dataclass( "DataClassItem", @@ -50,19 +34,6 @@ class AttrsItemNested: ], ) - DataClassItemNested = make_dataclass( - "DataClassItem", - [ - ("nested", DataClassItem), - ("adapter", ItemAdapter), - ("dict_", dict), - ("list_", list), - ("set_", set), - ("tuple_", tuple), - ("int_", int), - ], - ) - try: from scrapy.item import Item as ScrapyItem, Field diff --git a/tests/attr_utils.py b/tests/attr_utils.py new file mode 100644 index 0000000..362627b --- /dev/null +++ b/tests/attr_utils.py @@ -0,0 +1,19 @@ +"""Code for attr.s tests that must only be imported from within tests because +it imports from itemadapter.""" + +import attr + +from itemadapter import ItemAdapter + +from tests import AttrsItem + + +@attr.s +class AttrsItemNested: + nested = attr.ib(type=AttrsItem) + adapter = attr.ib(type=ItemAdapter) + dict_ = attr.ib(type=dict) + list_ = attr.ib(type=list) + set_ = attr.ib(type=set) + tuple_ = attr.ib(type=tuple) + int_ = attr.ib(type=int) diff --git a/tests/dataclasses_utils.py b/tests/dataclasses_utils.py new file mode 100644 index 0000000..60d65bf --- /dev/null +++ b/tests/dataclasses_utils.py @@ -0,0 +1,22 @@ +"""Code for dataclass tests that must only be imported from within tests +because it imports from itemadapter.""" + +from dataclasses import make_dataclass + +from itemadapter import ItemAdapter + +from tests import DataClassItem + + +DataClassItemNested = make_dataclass( + "DataClassItem", + [ + ("nested", DataClassItem), + ("adapter", ItemAdapter), + ("dict_", dict), + ("list_", list), + ("set_", set), + ("tuple_", tuple), + ("int_", int), + ], +) diff --git a/tests/test_adapter.py b/tests/test_adapter.py index 37e4c57..789229b 100644 --- a/tests/test_adapter.py +++ b/tests/test_adapter.py @@ -1,14 +1,11 @@ import unittest +from importlib import import_module from types import MappingProxyType from typing import KeysView -from itemadapter.adapter import ItemAdapter - from tests import ( AttrsItem, - AttrsItemNested, DataClassItem, - DataClassItemNested, requires_attr, requires_dataclasses, requires_scrapy, @@ -20,6 +17,8 @@ class ItemAdapterReprTestCase(TestCase): def test_repr_dict(self): + from itemadapter.adapter import ItemAdapter + item = dict(name="asdf", value=1234) adapter = ItemAdapter(item) # dicts are not guarantied to be sorted in py35 @@ -30,6 +29,8 @@ def test_repr_dict(self): @requires_scrapy def test_repr_scrapy_item(self): + from itemadapter.adapter import ItemAdapter + item = ScrapySubclassedItem(name="asdf", value=1234) adapter = ItemAdapter(item) # Scrapy fields are stored in a dict, which is not guarantied to be sorted in py35 @@ -40,6 +41,8 @@ def test_repr_scrapy_item(self): @requires_dataclasses def test_repr_dataclass(self): + from itemadapter.adapter import ItemAdapter + item = DataClassItem(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual( @@ -48,6 +51,8 @@ def test_repr_dataclass(self): @requires_attr def test_repr_attrs(self): + from itemadapter.adapter import ItemAdapter + item = AttrsItem(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual( @@ -57,6 +62,8 @@ def test_repr_attrs(self): class ItemAdapterInitError(TestCase): def test_non_item(self): + from itemadapter.adapter import ItemAdapter + with self.assertRaises(TypeError): ItemAdapter(ScrapySubclassedItem) with self.assertRaises(TypeError): @@ -68,14 +75,22 @@ def test_non_item(self): class BaseTestMixin: item_class = None - item_class_nested = None + item_class_nested_path = None def setUp(self): super().setUp() if self.item_class is None: raise unittest.SkipTest() + @property + def item_class_nested(self): + module_path, class_name = self.item_class_nested_path.rsplit(".", maxsplit=1) + module = import_module(module_path) + return getattr(module, class_name) + def test_get_set_value(self): + from itemadapter.adapter import ItemAdapter + item = self.item_class() adapter = ItemAdapter(item) self.assertEqual(adapter.get("name"), None) @@ -95,17 +110,23 @@ def test_get_set_value(self): self.assertEqual(adapter["value"], 1234) def test_get_value_keyerror(self): + from itemadapter.adapter import ItemAdapter + item = self.item_class() adapter = ItemAdapter(item) with self.assertRaises(KeyError): adapter["undefined_field"] def test_as_dict(self): + from itemadapter.adapter import ItemAdapter + item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual(dict(name="asdf", value=1234), dict(adapter)) def test_as_dict_nested(self): + from itemadapter.adapter import ItemAdapter + item = self.item_class_nested( nested=self.item_class(name="asdf", value=1234), adapter=ItemAdapter(dict(foo="bar", nested_list=[1, 2, 3, 4, 5])), @@ -130,6 +151,8 @@ def test_as_dict_nested(self): ) def test_field_names(self): + from itemadapter.adapter import ItemAdapter + item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertIsInstance(adapter.field_names(), KeysView) @@ -138,12 +161,16 @@ def test_field_names(self): class NonDictTestMixin(BaseTestMixin): def test_set_value_keyerror(self): + from itemadapter.adapter import ItemAdapter + item = self.item_class() adapter = ItemAdapter(item) with self.assertRaises(KeyError): adapter["undefined_field"] = "some value" def test_metadata_common(self): + from itemadapter.adapter import ItemAdapter + adapter = ItemAdapter(self.item_class()) self.assertIsInstance(adapter.get_field_meta("name"), MappingProxyType) self.assertIsInstance(adapter.get_field_meta("value"), MappingProxyType) @@ -151,11 +178,15 @@ def test_metadata_common(self): adapter.get_field_meta("undefined_field") def test_get_field_meta_defined_fields(self): + from itemadapter.adapter import ItemAdapter + adapter = ItemAdapter(self.item_class()) self.assertEqual(adapter.get_field_meta("name"), MappingProxyType({"serializer": str})) self.assertEqual(adapter.get_field_meta("value"), MappingProxyType({"serializer": int})) def test_delitem_len_iter(self): + from itemadapter.adapter import ItemAdapter + item = self.item_class(name="asdf", value=1234) adapter = ItemAdapter(item) self.assertEqual(len(adapter), 2) @@ -184,16 +215,22 @@ class DictTestCase(TestCase, BaseTestMixin): def test_get_value_keyerror_item_dict(self): """Instantiate without default values""" + from itemadapter.adapter import ItemAdapter + adapter = ItemAdapter(self.item_class()) with self.assertRaises(KeyError): adapter["name"] def test_empty_metadata(self): + from itemadapter.adapter import ItemAdapter + adapter = ItemAdapter(self.item_class(name="foo", value=5)) for field_name in ("name", "value", "undefined_field"): self.assertEqual(adapter.get_field_meta(field_name), MappingProxyType({})) def test_field_names_updated(self): + from itemadapter.adapter import ItemAdapter + item = self.item_class(name="asdf") field_names = ItemAdapter(item).field_names() self.assertEqual(sorted(field_names), ["name"]) @@ -208,6 +245,8 @@ class ScrapySubclassedItemTestCase(NonDictTestMixin, TestCase): def test_get_value_keyerror_item_dict(self): """Instantiate without default values""" + from itemadapter.adapter import ItemAdapter + adapter = ItemAdapter(self.item_class()) with self.assertRaises(KeyError): adapter["name"] @@ -216,10 +255,10 @@ def test_get_value_keyerror_item_dict(self): class DataClassItemTestCase(NonDictTestMixin, TestCase): item_class = DataClassItem - item_class_nested = DataClassItemNested + item_class_nested_path = "tests.dataclasses_utils.DataClassItemNested" class AttrsItemTestCase(NonDictTestMixin, TestCase): item_class = AttrsItem - item_class_nested = AttrsItemNested + item_class_nested_path = "tests.attr_utils.AttrsItemNested" diff --git a/tests/test_utils.py b/tests/test_utils.py index fdad127..164fb8d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,8 +1,6 @@ import unittest from unittest import mock -from itemadapter.utils import is_item, is_attrs_instance, is_dataclass_instance, is_scrapy_item - from tests import ( AttrsItem, DataClassItem, @@ -17,6 +15,8 @@ class ItemLikeTestCase(TestCase): def test_false(self): + from itemadapter.utils import is_item + self.assertFalse(is_item(int)) self.assertFalse(is_item(sum)) self.assertFalse(is_item(1234)) @@ -33,24 +33,34 @@ def test_false(self): self.assertFalse(is_item(AttrsItem)) def test_true_dict(self): + from itemadapter.utils import is_item + self.assertTrue(is_item({"a": "dict"})) @requires_scrapy def test_true_scrapy(self): + from itemadapter.utils import is_item + self.assertTrue(is_item(ScrapyItem())) self.assertTrue(is_item(ScrapySubclassedItem(name="asdf", value=1234))) @requires_dataclasses def test_true_dataclass(self): + from itemadapter.utils import is_item + self.assertTrue(is_item(DataClassItem(name="asdf", value=1234))) @requires_attr def test_true_attrs(self): + from itemadapter.utils import is_item + self.assertTrue(is_item(AttrsItem(name="asdf", value=1234))) class AttrsTestCase(TestCase): def test_false(self): + from itemadapter.utils import is_attrs_instance + self.assertFalse(is_attrs_instance(int)) self.assertFalse(is_attrs_instance(sum)) self.assertFalse(is_attrs_instance(1234)) @@ -65,12 +75,16 @@ def test_false(self): @requires_attr def test_true(self): + from itemadapter.utils import is_attrs_instance + self.assertTrue(is_attrs_instance(AttrsItem())) self.assertTrue(is_attrs_instance(AttrsItem(name="asdf", value=1234))) class DataclassTestCase(TestCase): def test_false(self): + from itemadapter.utils import is_dataclass_instance + self.assertFalse(is_dataclass_instance(int)) self.assertFalse(is_dataclass_instance(sum)) self.assertFalse(is_dataclass_instance(1234)) @@ -85,12 +99,16 @@ def test_false(self): @requires_dataclasses def test_true(self): + from itemadapter.utils import is_dataclass_instance + self.assertTrue(is_dataclass_instance(DataClassItem())) self.assertTrue(is_dataclass_instance(DataClassItem(name="asdf", value=1234))) class ScrapyItemTestCase(TestCase): def test_false(self): + from itemadapter.utils import is_scrapy_item + self.assertFalse(is_scrapy_item(int)) self.assertFalse(is_scrapy_item(sum)) self.assertFalse(is_scrapy_item(1234)) @@ -105,6 +123,8 @@ def test_false(self): @requires_scrapy def test_true(self): + from itemadapter.utils import is_scrapy_item + self.assertTrue(is_scrapy_item(ScrapyItem())) self.assertTrue(is_scrapy_item(ScrapySubclassedItem())) self.assertTrue(is_scrapy_item(ScrapySubclassedItem(name="asdf", value=1234))) @@ -128,6 +148,8 @@ class ScrapyDeprecatedBaseItemTestCase(TestCase): not hasattr(scrapy.item, "_BaseItem"), "scrapy.item._BaseItem not available", ) def test_deprecated_underscore_baseitem(self): + from itemadapter.utils import is_scrapy_item + class SubClassed_BaseItem(scrapy.item._BaseItem): pass @@ -138,6 +160,8 @@ class SubClassed_BaseItem(scrapy.item._BaseItem): not hasattr(scrapy.item, "BaseItem"), "scrapy.item.BaseItem not available", ) def test_deprecated_baseitem(self): + from itemadapter.utils import is_scrapy_item + class SubClassedBaseItem(scrapy.item.BaseItem): pass @@ -145,6 +169,8 @@ class SubClassedBaseItem(scrapy.item.BaseItem): self.assertTrue(is_scrapy_item(SubClassedBaseItem())) def test_removed_baseitem(self): + from itemadapter.utils import is_scrapy_item + class MockItemModule: Item = ScrapyItem From f7624e6a3ef4e87a1e34a9628adde8bd2cff043b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 12:41:26 +0200 Subject: [PATCH 05/11] Enable GitHub actions on pull requests --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aebb77d..9b4b015 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,8 @@ name: Build -on: [push] +on: +- pull_request +- push jobs: checks: From 6a4c8c5bf33d5e730db51f94df26c2490a8f49fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 12:43:49 +0200 Subject: [PATCH 06/11] Fix GitHub Actions configuration --- .github/workflows/main.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b4b015..064a1d3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,8 +1,6 @@ name: Build -on: -- pull_request -- push +on: [push] jobs: checks: @@ -58,6 +56,9 @@ jobs: tests-no-extra-deps: name: "Test: python3.6, Ubuntu, no deps" runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.8] steps: - uses: actions/checkout@v2 @@ -65,7 +66,7 @@ jobs: - name: Set up Python 3.6 uses: actions/setup-python@v1 with: - python-version: [3.5, 3.6, 3.8] + python-version: ${{ matrix.python-version }} - name: Install tox run: pip install tox From 87aea246d0f2c24847ade56e5ed6153b19c6f2f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 12:46:03 +0200 Subject: [PATCH 07/11] Re-enable pull requests in GitHub Actions --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 064a1d3..d098e74 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,8 @@ name: Build -on: [push] +on: +- push +- pull_request jobs: checks: From abd035bef54ce65bbf4b347f5aa73ee75e3530f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Aug 2020 12:48:26 +0200 Subject: [PATCH 08/11] Fix the new GitHub Action names --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d098e74..93d299b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -56,7 +56,7 @@ jobs: tests-no-extra-deps: - name: "Test: python3.6, Ubuntu, no deps" + name: "Test: py${{ matrix.python-version }}, Ubuntu, no extra deps" runs-on: ubuntu-latest strategy: matrix: From 5b4b116cfb950d7d57b53dba5174b8c1ca246861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Nov 2020 22:10:23 +0100 Subject: [PATCH 09/11] Remove Python 3.5 from no-extra-deps tests --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd3e1ed..9cd872d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,12 +58,12 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.8] + python-version: [3.6, 3.7, 3.8] steps: - uses: actions/checkout@v2 - - name: Set up Python 3.6 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v1 with: python-version: ${{ matrix.python-version }} From 514f48de4d16223578bdbb050b6fb1e8efeb3667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Nov 2020 22:15:09 +0100 Subject: [PATCH 10/11] Restore unintended change --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9cd872d..8e2b0d9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: checks: - name: Flake8, typing, black + name: Lint, typing, coverage runs-on: ubuntu-latest steps: From adcf7bca6ab71d0206d9a916865691273064121d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 18 Nov 2020 22:18:35 +0100 Subject: [PATCH 11/11] Undo unintended change --- itemadapter/adapter.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/itemadapter/adapter.py b/itemadapter/adapter.py index 33820c2..ac2f9ee 100644 --- a/itemadapter/adapter.py +++ b/itemadapter/adapter.py @@ -170,7 +170,14 @@ class ItemAdapter(MutableMapping): to extract and set data without having to take the object's type into account. """ - ADAPTER_CLASSES = deque([ScrapyItemAdapter, DictAdapter, DataclassAdapter, AttrsAdapter]) + ADAPTER_CLASSES = deque( + [ + ScrapyItemAdapter, + DictAdapter, + DataclassAdapter, + AttrsAdapter, + ] + ) def __init__(self, item: Any) -> None: self.adapter_class = None