diff --git a/.circleci/config.yml b/.circleci/config.yml index 5d2d11dae..756620446 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,7 +14,7 @@ workflows: jobs: unit-3.6: docker: - - image: 'python:3.6' + - image: python:3.6-slim steps: - checkout - run: @@ -36,7 +36,7 @@ jobs: when: always unit-3.7: docker: - - image: 'python:3.7' + - image: python:3.7-slim steps: - checkout - run: @@ -58,17 +58,17 @@ jobs: when: always showcase: docker: - - image: 'python:3.7' + - image: python:3.7-slim steps: - checkout - run: - name: Install nox. - command: pip install --pre nox-automation - - run: - name: Install pandoc and unzip. + name: Install system dependencies. command: | apt-get update - apt-get install -y pandoc unzip + apt-get install -y curl pandoc unzip + - run: + name: Install nox. + command: pip install --pre nox-automation - run: name: Install protoc 3.6.1. command: | @@ -94,7 +94,7 @@ jobs: command: nox -s showcase docs: docker: - - image: 'python:3.6' + - image: python:3.6-slim steps: - checkout - run: diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..67cb2311a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,61 @@ +# Version control scaffolding +.git +.gitignore + +# Docker scaffolding +Dockerfile +.dockerignore + +# Python scaffolding +*.py[cod] +*.egg +*.egg-info +dist +build +eggs +parts +bin +var +sdist +develop-eggs +.installed.cfg +lib +lib64 +__pycache__ + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.nox +.tox +.cache +.pytest_cache +htmlcov + +# Translations +*.mo + +# Mac +.DS_Store + +# Mr Developer +.mr.developer.cfg +.project +.pydevproject + +# JetBrains +.idea + +# Built documentation +docs/_build +docs/_build_doc2dash + +# Virtual environment +env/ +coverage.xml + +# Make sure a generated file isn't accidentally committed. +pylintrc +pylintrc.test diff --git a/.gitignore b/.gitignore index f2185a4d2..a9fe758f6 100644 --- a/.gitignore +++ b/.gitignore @@ -53,9 +53,6 @@ docs/_build_doc2dash env/ coverage.xml -# System test environment variables. -system_tests/local_test_setup - # Make sure a generated file isn't accidentally committed. pylintrc pylintrc.test diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..eada6a905 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.7-slim + +# Install system packages. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + pandoc \ + && rm -rf /var/lib/apt/lists/* + +# Add protoc and our common protos. +COPY --from=gcr.io/gapic-images/api-common-protos:latest /usr/local/bin/protoc /usr/local/bin/protoc +COPY --from=gcr.io/gapic-images/api-common-protos:latest /protos/ /protos/ + +# Add our code to the Docker image. +ADD . /usr/src/gapic-generator-python/ + +# Install the tool within the image. +RUN pip install /usr/src/gapic-generator-python + +# Define the generator as an entry point. +ENTRYPOINT protoc --proto_path=/protos/ --proto_path=/in/ \ + --python_gapic_out=/out/ \ + `find /in/ -name *.proto` diff --git a/docs/conf.py b/docs/conf.py index dffe343df..92849ebac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,10 +24,14 @@ author = 'Luke Sneeringer' # The short X.Y version -version = '0.0.4' +version = os.environ.get('CIRCLE_TAG', 'latest') # The full version, including alpha/beta/rc tags -release = '0.0.4' +release = os.environ.get('CIRCLE_TAG', 'latest') +# Replace |version| in the docs with the actual version string. +rst_epilog = """ +.. |version| replace:: {version} +""".format(version=version) # -- General configuration --------------------------------------------------- diff --git a/docs/getting-started.rst b/docs/getting-started.rst deleted file mode 100644 index 4d07ca3ac..000000000 --- a/docs/getting-started.rst +++ /dev/null @@ -1,125 +0,0 @@ -Getting Started ---------------- - -To use this plugin, you will need an API which is specified using a -protocol buffer. Additionally, this plugin makes some assumptions at the -margins according to `Google API design conventions`_, so following those -conventions is recommended. - -Example -~~~~~~~ - -If you want to experiment with an already-existing API, one example is -available. (Reminder that this is still considered experimental, so apologies -for this part being a bit strange.) - -You need to clone the `googleapis`_ repository from GitHub, and change to -a special branch: - -.. code-block:: shell - - $ git clone git@github.com:googleapis/googleapis.git - $ cd googleapis - $ git checkout --track -b input-contract origin/input-contract - $ cd .. - -The API available as an example (thus far) is the `Google Cloud Vision`_ API, -available in the ``google/cloud/vision/v1/`` subdirectory. This will be used -for the remainder of the examples on this page. - -You will also need the common protos, currently in experimental status, -which define certain client-specific annotations. These are in the -`api-common-protos`_ repository. Clone this from GitHub also: - -.. code-block:: shell - - $ git clone git@github.com:googleapis/api-common-protos.git - $ cd api-common-protos - $ git checkout --track -b input-contract origin/input-contract - $ cd .. - -.. _googleapis: https://github.com/googleapis/googleapis/tree/input-contract -.. _api-common-protos: https://github.com/googleapis/api-common-protos/tree/input-contract -.. _Google Cloud Vision: https://cloud.google.com/vision/ - - -Compiling an API -~~~~~~~~~~~~~~~~ - -To get a client library, you need to both compile the proto descriptors -into compiled message types (which is functionality built into ``protoc``) -and also into a client library (which is what this plugin does). - -These can be done in the same step. ``protoc`` requires an output destination -for each plugin invoked; you just want these to match: - -.. code-block:: shell - - # This is assumed to be in the `googleapis` project root, and we also - # assume that api-common-protos is next to it. - $ protoc google/cloud/vision/v1/*.proto \ - --proto_path=../api-common-protos/ --proto_path=. \ - --python_out=/dest/ --python_gapic_out=/dest/ - -.. note:: - - **A reminder about paths.** - - Remember that ``protoc`` is particular about paths. It requires all paths - where it expects to find protos, and *order matters*. In this case, - the common protos must come first, and then the path to the API being built. - - -Running a Client Library -~~~~~~~~~~~~~~~~~~~~~~~~ - -Once you have compiled a client library, it is time for the fun part: -actually running it! - -Create a virtual environment for the library: - -.. code-block:: shell - - $ virtualenv ~/.local/client-lib --python=`which python3.7` - $ source ~/.local/client-lib/bin/activate - -Next, install the library: - -.. code-block:: shell - - $ cd /dest/ - $ pip install --editable . - -Now it is time to play with it! -Here is a test script: - -.. code-block:: python - - # This is the client library generated by this plugin. - from google.cloud import vision - - # Instantiate the client. - # - # If you need to manually specify credentials, do so here. - # More info: https://cloud.google.com/docs/authentication/getting-started - # - # If you wish, you can send `transport='grpc'` or `transport='http'` - # to change which underlying transport layer is being used. - ia = vision.ImageAnnotator() - - # Send the request to the server and get the response. - response = ia.batch_annotate_images({ - 'requests': [{ - 'features': [{ - 'type': vision.types.image_annotator.Feature.Type.LABEL_DETECTION - }], - 'image': {'source': { - 'image_uri': 'https://s3.amazonaws.com/cdn0.michiganbulb.com' - '/images/350/66623.jpg', - }}, - }], - }) - print(response) - - -.. _Google API design conventions: https://cloud.google.com/apis/design/ diff --git a/docs/getting-started/_example.rst b/docs/getting-started/_example.rst new file mode 100644 index 000000000..1bed548dc --- /dev/null +++ b/docs/getting-started/_example.rst @@ -0,0 +1,20 @@ +If you want to experiment with an already-existing API, one example is +available. (Reminder that this is still considered experimental, so apologies +for this part being a bit strange.) + +You need to clone the `googleapis`_ repository from GitHub, and change to +a special branch: + +.. code-block:: shell + + $ git clone git@github.com:googleapis/googleapis.git + $ cd googleapis + $ git checkout --track -b input-contract origin/input-contract + $ cd .. + +The API available as an example (thus far) is the `Google Cloud Vision`_ API, +available in the ``google/cloud/vision/v1/`` subdirectory. This will be used +for the remainder of the examples on this page. + +.. _googleapis: https://github.com/googleapis/googleapis/tree/input-contract +.. _Google Cloud Vision: https://cloud.google.com/vision/ diff --git a/docs/getting-started/_usage_intro.rst b/docs/getting-started/_usage_intro.rst new file mode 100644 index 000000000..cad09dc6d --- /dev/null +++ b/docs/getting-started/_usage_intro.rst @@ -0,0 +1,6 @@ +To use this plugin, you will need an API which is specified using +protocol buffers. Additionally, this plugin makes some assumptions at the +margins according to `Google API design conventions`_, so following those +conventions is recommended. + +.. _Google API design conventions: https://cloud.google.com/apis/design/ diff --git a/docs/getting-started/_verifying.rst b/docs/getting-started/_verifying.rst new file mode 100644 index 000000000..e25eafa15 --- /dev/null +++ b/docs/getting-started/_verifying.rst @@ -0,0 +1,50 @@ +Verifying the Library +--------------------- + +Once you have compiled a client library, whether using a Docker image or +a local installation, it is time for the fun part: actually running it! + +Create a virtual environment for the library: + +.. code-block:: shell + + $ virtualenv ~/.local/client-lib --python=`which python3.7` + $ source ~/.local/client-lib/bin/activate + +Next, install the library: + +.. code-block:: shell + + $ cd /dest/ + $ pip install --editable . + +Now it is time to play with it! +Here is a test script: + +.. code-block:: python + + # This is the client library generated by this plugin. + from google.cloud import vision + + # Instantiate the client. + # + # If you need to manually specify credentials, do so here. + # More info: https://cloud.google.com/docs/authentication/getting-started + # + # If you wish, you can send `transport='grpc'` or `transport='http'` + # to change which underlying transport layer is being used. + ia = vision.ImageAnnotator() + + # Send the request to the server and get the response. + response = ia.batch_annotate_images({ + 'requests': [{ + 'features': [{ + 'type': vision.types.image_annotator.Feature.Type.LABEL_DETECTION, + }], + 'image': {'source': { + 'image_uri': 'https://s3.amazonaws.com/cdn0.michiganbulb.com' + '/images/350/66623.jpg', + }}, + }], + }) + print(response) diff --git a/docs/getting-started/docker-shortcut.rst b/docs/getting-started/docker-shortcut.rst new file mode 100644 index 000000000..88f3ef809 --- /dev/null +++ b/docs/getting-started/docker-shortcut.rst @@ -0,0 +1,28 @@ +:orphan: + +.. _docker-shortcut: + +Docker Shortcut Script +---------------------- + +Because code generation requires two mounts from the host machine into +the Docker image, and because the paths are somewhat pedantic, you may +find this shortcut script to be handy: + +.. literalinclude:: ../../gapic.sh + :language: shell + +Place it somewhere on your system, marked executable. + +Once available, it can be invoked using: + +.. code-block:: shell + + # This is assumed to be from the "proto root" directory. + $ gapic.sh --image gcr.io/gapic-images/gapic-generator-python \ + --in path/to/src/protos/ + --out dest/ + + +It will work not only with the Python code generator, but all of our code +generators that implement this Docker interface. diff --git a/docs/getting-started/docker.rst b/docs/getting-started/docker.rst new file mode 100644 index 000000000..9e1dd3559 --- /dev/null +++ b/docs/getting-started/docker.rst @@ -0,0 +1,116 @@ +.. _getting-started/docker: + +Docker Image +============ + +If you are just getting started with code generation for protobuf-based APIs, +or if you do not have a robust Python environment already available, we +recommend using our `Docker`_ image to build client libraries. + +However, this tool offers first-class support for local execution using +protoc: :ref:`getting-started/local`. It is still reasonably easy, but +initial setup will take a bit longer. + +.. note:: + + If you are interested in contributing, using a local installation + is recommended. + +.. _Docker: https://docker.com/ + + +Installing +---------- + +Docker +~~~~~~ + +In order to use a Docker image, you must have `Docker`_ installed. +Docker is a container management service, and is available on Linux, Mac, +and Windows (although most of these instructions will be biased toward +Linux and Mac). + +Install Docker according to their `installation instructions`_. + +.. note:: + + This image requires Docker 17.05 or later. + +.. _installation instructions: https://docs.docker.com/install/ + +Pull the Docker Image +~~~~~~~~~~~~~~~~~~~~~ + +Once Docker is installed, simply pull the Docker image for this tool: + +.. parsed-literal:: + + $ docker pull gcr.io/gapic-images/gapic-generator-python:\ |version|\ + + +Usage +----- + +.. include:: _usage_intro.rst + +Example +~~~~~~~ + +.. include:: _example.rst + + +Compiling an API +~~~~~~~~~~~~~~~~ + +.. note:: + + If you are running code generation repeatedly, executing the + long ``docker run`` command may be cumbersome. While you should ensure + you understand this section, a :ref:`shortcut script` + is available to make iterative work easier. + +Compile the API into a client library by invoking the Docker image. + +It is worth noting that the image must interact with the host machine +(your local machine) for two things: reading in the protos you wish to compile, +and writing the output. This means that when you run the image, two mount +points are required in order for anything useful to happen. + +In particular, the input protos are expected to be mounted into ``/in/``, +and the desired output location is expected to be mounted into ``/out/``. +The output directory must also be writable. + +.. note:: + + The ``/in/`` and ``/out/`` directories inside the image are + hard-coded; they can not be altered where they appear in the command + below. + +Perform that step with ``docker run``: + +.. code-block:: shell + + # This is assumed to be run from the `googleapis` project root. + $ docker run \ + --mount type=bind,source=google/cloud/vision/v1/,destination=/in/google/cloud/vision/v1/,readonly \ + --mount type=bind,source=dest/,destination=/out/ \ + --rm \ + --user $UID \ + gcr.io/gapic-images/gapic-generator-python + +.. warning:: + + ``protoc`` is *very* picky about paths, and the exact construction here + matters a lot. The source is ``google/cloud/vision/v1/``, and then + the destination is that full directory path after the ``/in/`` root; + therefore: ``/in/google/cloud/vision/v1/``. + + This matters because of how proto imports are resolved. The ``import`` + statement imports a *file*, relative to a base directory or set of + base directories, called the ``proto_path``. This is assumed + (and hard-coded) to ``/in/`` in the Docker image, and so any directory + structure present in the imports of the proto files must be preserved + beneath this for compilation to succeed. + + +.. include:: _verifying.rst diff --git a/docs/getting-started/index.rst b/docs/getting-started/index.rst new file mode 100644 index 000000000..f3d5e3ff6 --- /dev/null +++ b/docs/getting-started/index.rst @@ -0,0 +1,26 @@ +Getting Started +--------------- + +This code generator is implemented as a plugin to ``protoc``, the compiler +for `protocol buffers`_, and will run in any environment that Python 3.6+ and +protocol buffers do. + +Because dependency management and such can be a significant undertaking, we +offer a Docker image and interface which requires you only to have Docker +installed and provide the protos for your API. + +It is also possible to install the tool locally and run it through ``protoc``, +and this approach is fully supported. + +.. note:: + + The Docker approach is recommended for users new to this ecosystem, or + those which do not have a robust Python environment available. + +.. _protocol buffers: https://developers.google.com/protocol-buffers/ + +.. toctree:: + :maxdepth: 4 + + docker + local diff --git a/docs/installing.rst b/docs/getting-started/local.rst similarity index 61% rename from docs/installing.rst rename to docs/getting-started/local.rst index 2496ae77d..61107150b 100644 --- a/docs/installing.rst +++ b/docs/getting-started/local.rst @@ -1,3 +1,22 @@ +.. _getting-started/local: + +Local Installation +================== + +If you are just getting started with code generation for protobuf-based APIs, +or if you do not have a robust Python environment already available, it is +probably easier to get started using Docker: :ref:`getting-started/docker` + +However, this tool offers first-class support for local execution using +``protoc``. It is still reasonably easy, but initial setup will take a bit +longer. + +.. note:: + + If you are interested in contributing, setup according to these steps + is recommended. + + Installing ---------- @@ -87,3 +106,51 @@ To ensure the tool is installed properly: .. _pyenv: https://github.com/pyenv/pyenv .. _pipsi: https://github.com/mitsuhiko/pipsi + +Usage +----- + +.. include:: _usage_intro.rst + +Example +~~~~~~~ + +.. include:: _example.rst + +You will also need the common protos, currently in experimental status, +which define certain client-specific annotations. These are in the +`api-common-protos`_ repository. Clone this from GitHub also: + +.. code-block:: shell + + $ git clone git@github.com:googleapis/api-common-protos.git + $ cd api-common-protos + $ git checkout --track -b input-contract origin/input-contract + $ cd .. + +.. _api-common-protos: https://github.com/googleapis/api-common-protos/tree/input-contract + + +Compiling an API +~~~~~~~~~~~~~~~~ + +Compile the API into a client library by invoking ``protoc`` directly. +This plugin is invoked under the hood via. the ``--python_gapic_out`` switch. + +.. code-block:: shell + + # This is assumed to be in the `googleapis` project root, and we also + # assume that api-common-protos is next to it. + $ protoc google/cloud/vision/v1/*.proto \ + --proto_path=../api-common-protos/ --proto_path=. \ + --python_gapic_out=/dest/ + +.. note:: + + **A reminder about paths.** + + Remember that ``protoc`` is particular about paths. It requires all paths + where it expects to find protos, and *order matters*. In this case, + the common protos must come first, and then the path to the API being built. + +.. include:: _verifying.rst diff --git a/docs/index.rst b/docs/index.rst index 33b4c12d0..9f05fc87f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,8 +18,7 @@ implemented as a plugin to ``protoc``, the protocol buffer compiler. .. toctree:: :maxdepth: 2 - installing - getting-started + getting-started/index api-configuration process templates diff --git a/gapic.sh b/gapic.sh new file mode 100755 index 000000000..6102f05f6 --- /dev/null +++ b/gapic.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CMD="$0" + +# Set variables used by this script. +# All of these are set in options below, and all but $PATH are required. +IMAGE= +IN= +OUT= +PROTO_PATH=`pwd` + +# Print help and exit. +function show_help { + echo "Usage: $CMD --image IMAGE --in IN_DIR --out OUT_DIR [--path PATH_DIR]" + echo "" + echo "Required arguments:" + echo " --image The Docker image to use. The script will attempt to pull" + echo " it if it is not present." + echo " -i, --in A directory containing the protos describing the API" + echo " to be generated." + echo " -o, --out Destination directory for the completed client library." + echo "" + echo "Optional arguments:" + echo " -p, --path The base import path for the protos. Assumed to be the" + echo " current working directory if unspecified." + echo " -h, --help This help information." + exit 0 +} + +# Parse out options. +while true; do + case "$1" in + -h | --help ) show_help ;; + --image ) IMAGE="$2"; shift 2 ;; + -i | --in ) IN="$2"; shift 2 ;; + -o | --out ) OUT="$2"; shift 2 ;; + -p | --path ) PROTO_PATH=$2; shift 2 ;; + -- ) shift; break; ;; + * ) break ;; + esac +done + +# Ensure that all required options are set. +if [ -z "$IMAGE" ] || [ -z "$IN" ] || [ -z "$OUT" ]; then + >&2 echo "Required argument missing." + >&2 echo "The --image, --in, and --out arguments are all required." + >&2 echo "Run $CMD --help for more information." + exit 64 +fi + +# Ensure that the input directory exists (and is a directory). +if ! [ -d $IN ]; then + >&2 echo "Directory does not exist: $IN" + exit 2 +fi + +# Ensure Docker is running and seems healthy. +# This is mostly a check to bubble useful errors quickly. +if ! docker ps > /dev/null; then + exit $? +fi + +# If the output directory does not exist, create it. +if ! mkdir -p $OUT ; then + exit $? +fi + +# If the output directory is not empty, warn (but continue). +if [ "$(ls -A $OUT )"]; then + >&2 echo "Warning: Output directory is not empty." +fi + +# If the image is not yet on the machine, pull it. +if ! docker images $IMAGE > /dev/null; then + echo "Image $IMAGE not found; pulling." + if ! docker pull $IMAGE; then + exit $? + fi +fi + +# Generate the client library. +docker run \ + --mount type=bind,source=${PROTO_PATH}/${IN},destination=/in/${IN},readonly \ + --mount type=bind,source=$OUT,destination=/out \ + --rm \ + --user $UID \ + $IMAGE +exit $?