DEV-134 add linters and formatters

For consistency, black has been added as a formatter.
Bandit and flake8 have also been added for linting.

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+exclude =
+	.git
+	.idea
+	.tox
+	venv
+max-line-length = 88

.pre-commit-config.yaml

@@ -0,0 +1,14 @@
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/psf/black
+    rev: 19.10b0
+    hooks:
+    -   id: black
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.9
+    hooks:
+    -   id: flake8

README.md

@@ -62,3 +62,14 @@ Testing
 ```shell
 $ tox
 ```
+
+Contributing
+---
+
+We use `pre-commit` to enforce formatting and linting.  It needs to be installed 
+in your local copy of this repo.
+
+```shell script
+pip install -r dev-requirements.txt
+pre-commit install
+```

dev-requirements.in

@@ -1,7 +1,12 @@
 -c requirements.txt
 
+pip-tools>=4.5.1,<5.0.0
 httmock>=1.3.0,<2
 pytest>=4.6.6,<5
 pytest-cov>=2.8.1,<3
 mock>=3.0.5,<4
 pytest-threadleak>=0.2.0,<1
+pre-commit>=2.2.0,<3
+bandit>=1.6.2,<2
+flake8>=3.7.9,<4
+black>=19.10b0,<20

dev-requirements.txt

@@ -4,25 +4,53 @@
 #
 #    pip-compile dev-requirements.in
 #
+appdirs==1.4.3            # via black, virtualenv
 atomicwrites==1.3.0       # via pytest
-attrs==19.3.0             # via pytest
+attrs==19.3.0             # via black, pytest
+bandit==1.6.2             # via -r dev-requirements.in
+black==19.10b0            # via -r dev-requirements.in
 certifi==2019.11.28       # via -c requirements.txt, requests
+cfgv==3.1.0               # via pre-commit
 chardet==3.0.4            # via -c requirements.txt, requests
+click==7.1.1              # via black, pip-tools
 coverage==5.0.4           # via pytest-cov
+distlib==0.3.0            # via virtualenv
+entrypoints==0.3          # via flake8
+filelock==3.0.12          # via virtualenv
+flake8==3.7.9             # via -r dev-requirements.in
+gitdb==4.0.2              # via gitpython
+gitpython==3.1.0          # via bandit
 httmock==1.3.0            # via -r dev-requirements.in
+identify==1.4.13          # via pre-commit
 idna==2.8                 # via -c requirements.txt, requests
-importlib-metadata==1.5.2  # via pluggy, pytest
+importlib-metadata==1.5.2  # via importlib-resources, pluggy, pre-commit, pytest, virtualenv
+importlib-resources==1.4.0  # via pre-commit, virtualenv
+mccabe==0.6.1             # via flake8
 mock==3.0.5               # via -r dev-requirements.in
 more-itertools==8.2.0     # via pytest
+nodeenv==1.3.5            # via pre-commit
 packaging==20.3           # via pytest
+pathspec==0.7.0           # via black
+pbr==5.4.4                # via stevedore
+pip-tools==4.5.1          # via -r dev-requirements.in
 pluggy==0.13.1            # via pytest
+pre-commit==2.2.0         # via -r dev-requirements.in
 py==1.8.1                 # via pytest
+pycodestyle==2.5.0        # via flake8
+pyflakes==2.1.1           # via flake8
 pyparsing==2.4.6          # via packaging
 pytest-cov==2.8.1         # via -r dev-requirements.in
 pytest-threadleak==0.2.0  # via -r dev-requirements.in
 pytest==4.6.6             # via -r dev-requirements.in, pytest-cov, pytest-threadleak
+pyyaml==5.3.1             # via bandit, pre-commit
+regex==2020.2.20          # via black
 requests==2.22.0          # via -c requirements.txt, httmock
-six==1.14.0               # via mock, packaging, pytest
+six==1.14.0               # via bandit, mock, packaging, pip-tools, pytest, stevedore, virtualenv
+smmap==3.0.1              # via gitdb
+stevedore==1.32.0         # via bandit
+toml==0.10.0              # via black, pre-commit
+typed-ast==1.4.1          # via black
 urllib3==1.25.8           # via -c requirements.txt, requests
+virtualenv==20.0.15       # via pre-commit
 wcwidth==0.1.9            # via pytest
-zipp==3.1.0               # via importlib-metadata
+zipp==3.1.0               # via importlib-metadata, importlib-resources

gdc_maf_tool/defer.py

@@ -40,7 +40,7 @@ def _validate_checksum(self, content):
         if not self._md5sum:
             return
 
-        hash_md5 = hashlib.md5()
+        hash_md5 = hashlib.md5()  # nosec
         hash_md5.update(content)
         md5 = hash_md5.hexdigest()
         if self._md5sum != md5:

gdc_maf_tool/gdc_api_client.py

@@ -1,7 +1,5 @@
-import hashlib
-import io
 import json
-from typing import Any, Dict, Iterator, List
+from typing import Any, Dict, List
 
 import requests
 from aliquot_level_maf.aggregation import AliquotLevelMaf
@@ -23,7 +21,8 @@ def query_hits(
     """
 
     # All queries start out as filtering on a MAF file that's a Masked Somatic Mutation.
-    # Adding the analysis.workflow_type filter will ensure we don't get extra mafs we don't want
+    # Adding the analysis.workflow_type filter will ensure we don't get extra mafs we
+    # don't want
     base_content = [
         {"op": "in", "content": {"field": "files.data_format", "value": ["MAF"]}},
         {
@@ -156,7 +155,6 @@ def only_one_project_id(hit_map: Dict) -> None:
         )
 
 
-
 def collect_criteria(hit_map: Dict) -> List[PrimaryAliquotSelectionCriterion]:
     criteria = []
     for hit in hit_map.values():
@@ -181,12 +179,13 @@ def collect_criteria(hit_map: Dict) -> List[PrimaryAliquotSelectionCriterion]:
 def collect_mafs(
     project_id: str, case_ids: List[str], file_ids: List[str], token: str
 ) -> List[AliquotLevelMaf]:
-    """Put together a list of mafs given one of the following: project_id, case_ids, file_ids.
+    """Put together a list of mafs given one of: project_id, case_ids, file_ids.
 
     - If a list of ids is provided then ensure that those ids share the same project_id.
     - If case_ids then gather all the mafs related to those cases.
     - If file_ids then gather all the mafs of those file_ids.
-    - If a project_id is provided then gather all the aliquot level mafs for that project.
+    - If a project_id is provided then gather all the aliquot level mafs for that
+    project.
     """
 
     mafs = []
@@ -207,8 +206,7 @@ def collect_mafs(
         )
         mafs.append(
             AliquotLevelMaf(
-                file=maf_file_contents,
-                tumor_aliquot_submitter_id=sample_id,
+                file=maf_file_contents, tumor_aliquot_submitter_id=sample_id,
             )
         )
     # TODO: Return list of delayed download maf objects

tests/.bandit

@@ -0,0 +1,2 @@
+[bandit]
+skips: B101

tox.ini

@@ -4,10 +4,19 @@ minversion = 3.13.2
 requires = pip == 19.2.2
 
 [testenv]
+whitelist_externals=
+    black
+    bandit
+    flake8
+    pytest
 deps =
     -rrequirements.txt
     -rdev-requirements.txt
 commands_pre=
     pip freeze
 commands =
+    black --check .
+    bandit -r gdc_maf_tool/
+    bandit -r tests/
+    flake8
     pytest {posargs: -lv tests/ --cov=gdc-maf-tool --cov-report=html}