Simplify & complete configuration + Docker release (#15)

- Simplify configuration
- Add Singer usage data collection
- Automate Docker Hub release

GH Issues:
- #12

.circleci/config.yml

@@ -1,15 +1,11 @@
 version: 2
 jobs:
-  build:
+  test:
     docker:
       - image: dataworld/pyenv-tox
 
     working_directory: /root/target-datadotworld
 
-    environment:
-      PRERELEASE_BRANCH: prerelease
-      RELEASE_BRANCH: release
-
     steps:
       - checkout
 
@@ -26,23 +22,86 @@ jobs:
           name: tox
           command: tox --pre
 
+      - persist_to_workspace:
+          root: .
+          paths:
+            - ./*
+
       - save_cache:
           key: tox_cache-{{ checksum "tox.ini" }}
           paths:
             - .eggs
             - .tox
 
+  pypi-release:
+
+    docker:
+      - image: dataworld/pyenv-tox
+
+    working_directory: /root/target-datadotworld
+
+    steps:
+      - attach_workspace:
+          at: /root/target-datadotworld
+
+      - run:
+          name: build dist
+          command: python setup.py sdist bdist_wheel --universal
+
       - deploy:
-          name: Pre-release to pypi
+          name: pypi release
+          command: twine upload -u $PYPI_USERNAME -p $PYPI_PASSWORD dist/*
+
+  docker-release:
+    docker:
+      - image: dataworld/pyenv-tox
+
+    working_directory: /root/target-datadotworld
+
+    steps:
+      - attach_workspace:
+          at: /root/target-datadotworld
+
+      - run:
+          name: define PACKAGE_VERSION
           command: |
-            if [[ "${CIRCLE_BRANCH}" =~ ^(${PRERELEASE_BRANCH})$ ]]; then
-              echo 'Do a prerelease with twine here'
-            fi
+            echo "export PACKAGE_VERSION=$(python -c "import pkg_resources; print(pkg_resources.get_distribution('target-datadotworld').version)")" >> $BASH_ENV
+            source $BASH_ENV
+
+      - setup_remote_docker:
+          docker_layer_caching: true
+
+      - run:
+          name: docker setup
+          command: curl -sSL https://get.docker.com/ | sh
+
+      - run:
+          name: docker build
+          command: docker build -t dataworld/target-datadotworld -t dataworld/target-datadotworld:$PACKAGE_VERSION .
 
       - deploy:
-          name: Release to pypi
+          name: docker-hub release
           command: |
-            if [[ "${CIRCLE_BRANCH}" =~ ^(${RELEASE_BRANCH})$ ]]; then
-              python setup.py sdist bdist_wheel --universal
-              twine upload -u $PYPI_USERNAME -p $PYPI_PASSWORD dist/*
-            fi
+            docker login -u $DOCKER_USER -p $DOCKER_PASS
+            docker push dataworld/target-datadotworld:latest
+            docker push dataworld/target-datadotworld:$PACKAGE_VERSION
+
+workflows:
+  version: 2
+  test-double-release:
+    jobs:
+      - test
+      - pypi-release:
+          filters:
+            branches:
+              only:
+                - release
+          requires:
+            - test
+      - docker-release:
+          filters:
+            branches:
+              only:
+                - release
+          requires:
+            - test

.pytest_cache/v/cache/lastfailed

@@ -0,0 +1,3 @@
+{
+  "tests/target_datadotworld/test_target.py::TestTarget::()": true
+}

.pytest_cache/v/cache/nodeids

@@ -0,0 +1,69 @@
+[
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream[5]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream[10]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream[15]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked[5-3]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked[5-5]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked[10-3]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked[10-5]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked[15-3]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked[15-5]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked_error[5-3]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked_error[5-5]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked_error[10-3]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked_error[10-5]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked_error[15-3]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_chunked_error[15-5]",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_append_stream_error",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_connection_check",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_connection_check_401",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_connection_check_offline",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_get_current_version",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_get_current_version_missing_column",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_get_current_version_missing_table",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_get_current_version_error",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_set_stream_schema",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_set_stream_schema_error",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_sync",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_sync_error",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_truncate_stream_records",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_truncate_stream_records_error",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_create_dataset",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_create_dataset_error",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_get_dataset",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test_get_dataset_error",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test__retry_if_throttled",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test__retry_if_throttled_delayed",
+  "tests/target_datadotworld/test_api_client.py::TestApiClient::()::test__retry_if_throttled_error",
+  "tests/target_datadotworld/test_exceptions.py::test_convert_requests_exception[400-ApiError]",
+  "tests/target_datadotworld/test_exceptions.py::test_convert_requests_exception[401-UnauthorizedError]",
+  "tests/target_datadotworld/test_exceptions.py::test_convert_requests_exception[403-ForbiddenError]",
+  "tests/target_datadotworld/test_exceptions.py::test_convert_requests_exception[404-NotFoundError]",
+  "tests/target_datadotworld/test_exceptions.py::test_convert_requests_exception[422-ApiError]",
+  "tests/target_datadotworld/test_exceptions.py::test_convert_requests_exception[429-TooManyRequestsError]",
+  "tests/target_datadotworld/test_exceptions.py::test_convert_requests_exception_offline",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_config_minimal",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_config_complete",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_config_incomplete",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_config_invalid[invalid_config0]",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_config_invalid[invalid_config1]",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_config_invalid[invalid_config2]",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_config_invalid[invalid_config3]",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_config_invalid[invalid_config4]",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_config_invalid[invalid_config5]",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_process_lines",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_process_lines_new_dataset",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_process_lines_unparseable",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_process_lines_missing_schema",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_process_lines_multiple_streams",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_process_no_state",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_process_multi_state",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_process_same_version",
+  "tests/target_datadotworld/test_target.py::TestTarget::()::test_process_new_version",
+  "tests/target_datadotworld/test_utils.py::test_to_jsonline",
+  "tests/target_datadotworld/test_utils.py::test_to_chunks[5]",
+  "tests/target_datadotworld/test_utils.py::test_to_chunks[10]",
+  "tests/target_datadotworld/test_utils.py::test_to_chunks[15]",
+  "tests/target_datadotworld/test_utils.py::test_to_streamid[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa]",
+  "tests/target_datadotworld/test_utils.py::test_to_streamid[a1!_b@2_c3-a-1-b-2-c-3]"
+]

Dockerfile

@@ -0,0 +1,7 @@
+FROM python:latest
+
+ADD . /code
+WORKDIR /code
+
+RUN pip install .
+CMD ["target-datadotworld"]

README.rst

@@ -7,8 +7,8 @@ A `Singer <https://singer.io>`_ target that writes data to `data.world <https://
 How to use it
 =============
 
-``target-datadotworld`` works together with any other `Singer Tap <https://www.singer.io/#taps>`_ to move
-data from sources like `SalesForce <https://github.com/singer-io/tap-salesforce>`_, `HubSpot <https://github.com/singer-io/tap-hubspot>`_, `Marketo <https://github.com/singer-io/tap-marketo>`_, `MySQL <https://github.com/singer-io/tap-mysql>`_  and `more <https://github.com/search?p=3&q=org%3Asinger-io+tap-&type=Repositories>`_ to data.world.
+``target-datadotworld`` works together with any other `Singer Tap <https://www.singer.io/#taps>`_ to store on data.world
+data extracted from sources like `SalesForce <https://github.com/singer-io/tap-salesforce>`_, `HubSpot <https://github.com/singer-io/tap-hubspot>`_, `Marketo <https://github.com/singer-io/tap-marketo>`_, `MySQL <https://github.com/singer-io/tap-mysql>`_  and `more <https://github.com/search?p=3&q=org%3Asinger-io+tap-&type=Repositories>`_.
 
 Install and Run
 ---------------
@@ -30,24 +30,21 @@ and then run them together, piping the output of ``tap-fixerio`` to
 The data will be written to the dataset specified in ``config.json``. In this specific case, under a stream named ``exchange-rates``.
 
 If you're using a different Tap, substitute ``tap-fixerio`` in the final
-command above to the command used to run your Tap.
+command above with the command used to run your Tap.
 
 Configuration
 -------------
 
-`target-datadotworld` requires configuration file that is used to store your data.world API token, dataset information and other additional configuration.
+`target-datadotworld` requires configuration file that is used to store your data.world API token and dataset information.
 
 The following attributes are required:
 
 * ``api_token``: Your data.world `API token <https://data.world/settings/advanced>`_
 * ``dataset_id``: The title of the dataset where the data is to be stored. Must only contain lowercase letters, numbers, and dashes.
 
-Additionally, the following optional attributes can be provided. They determine the parameters for creating a new dataset if ``dataset_id`` refers to a dataset that doesn't yet exist:
+Additionally, the following optional attributes can be provided.
 
-* ``dataset_title``: Text with no more than 60 characters
-* ``dataset_visibility``: OPEN or PRIVATE
-* ``dataset_license``: Public Domain, PDDL, CC-0, CC-BY, ODC-BY, CC-BY-SA, ODC-ODbL, CC BY-NC, CC BY-NC-SA or Other
-* ``dataset_owner``: If not the same as the owner of the API token (e.g. if the dataset is to be created under an organization account, as opposed to the user's own)
+* ``dataset_owner``: If not the same as the owner of the API token (e.g. if the dataset is to be accessed/created under an organization account, as opposed to the user's own)
 
 Example:
 
@@ -56,8 +53,5 @@ Example:
     {
         "api_token": "your_token",
         "dataset_id": "fixerio-data",
-        "dataset_title": "Fixerio Data",
-        "dataset_license": "Other",
         "dataset_owner": "my-company",
-        "dataset_visibility": "PRIVATE"
     }

target_datadotworld/__init__.py

@@ -21,6 +21,6 @@
 
 import singer
 
-__version__ = '1.0.0b4'
+__version__ = '1.0.0'
 
 logger = copy(singer.get_logger())  # copy needed in order to set level

target_datadotworld/cli.py

@@ -21,11 +21,13 @@
 import json
 import logging
 import warnings
+from concurrent.futures import ThreadPoolExecutor
 
 import click
 
 from target_datadotworld import logger
 from target_datadotworld.exceptions import Error
+from target_datadotworld.singer_analytics import send_usage_stats
 from target_datadotworld.target import TargetDataDotWorld
 
 
@@ -56,6 +58,13 @@ def cli(ctx, config, debug, file):
     try:
         config_obj = json.load(config)
 
+        if not config_obj.get('disable_collection', False):
+            logger.info('Sending version information to singer.io. ' +
+                        'To disable sending anonymous usage data, set ' +
+                        'the config parameter "disable_collection" to true')
+            loop.run_in_executor(ThreadPoolExecutor(max_workers=1),
+                                 send_usage_stats)
+
         target = TargetDataDotWorld(config_obj)
         data_file = file or click.get_text_stream('stdin')
 

target_datadotworld/singer_analytics.py

@@ -0,0 +1,38 @@
+# target-datadotworld
+# Copyright 2017 data.world, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the
+# License.
+#
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# This product includes software developed at
+# data.world, Inc.(http://data.world/).
+import requests
+import target_datadotworld
+from target_datadotworld import logger
+from requests import HTTPError
+
+
+def send_usage_stats():
+    try:
+        version = target_datadotworld.__version__
+        resp = requests.get('http://collector.singer.io/i',
+                            params={
+                                'e': 'se',
+                                'aid': 'singer',
+                                'se_ca': 'target-datadotworld',
+                                'se_ac': 'open',
+                                'se_la': version,
+                            }, timeout=0.5)
+        resp.raise_for_status()
+    except HTTPError:
+        logger.debug('Collection request failed')