[refactor] New annotations (#92)

This updates gapic-generator-python to use the current annotations.

.circleci/config.yml

@@ -59,7 +59,7 @@ jobs:
       - run:
           name: Install nox and codecov.
           command: |
-            pip install --pre nox-automation
+            pip install nox
             pip install codecov
       - run:
           name: Run unit tests.
@@ -81,7 +81,7 @@ jobs:
       - run:
           name: Install nox and codecov.
           command: |
-            pip install --pre nox-automation
+            pip install nox
             pip install codecov
       - run:
           name: Run unit tests.
@@ -93,7 +93,7 @@ jobs:
   showcase:
     docker:
       - image: python:3.7-slim
-      - image: gcr.io/gapic-showcase/gapic-showcase:0.0.9
+      - image: gcr.io/gapic-showcase/gapic-showcase:0.0.12
     steps:
       - checkout
       - run:
@@ -103,7 +103,7 @@ jobs:
             apt-get install -y curl pandoc unzip
       - run:
           name: Install nox.
-          command: pip install --pre nox-automation
+          command: pip install nox
       - run:
           name: Install protoc 3.6.1.
           command: |
@@ -122,7 +122,7 @@ jobs:
       - checkout
       - run:
           name: Install nox.
-          command: pip install --pre nox-automation
+          command: pip install nox
       - run:
           name: Build the documentation.
           command: nox -s docs

Dockerfile

@@ -7,8 +7,8 @@ RUN apt-get update \
   && rm -rf /var/lib/apt/lists/*
 
 # Add protoc and our common protos.
-COPY --from=gcr.io/gapic-images/api-common-protos:latest /usr/local/bin/protoc /usr/local/bin/protoc
-COPY --from=gcr.io/gapic-images/api-common-protos:latest /protos/ /protos/
+COPY --from=gcr.io/gapic-images/api-common-protos:beta /usr/local/bin/protoc /usr/local/bin/protoc
+COPY --from=gcr.io/gapic-images/api-common-protos:beta /protos/ /protos/
 
 # Add our code to the Docker image.
 ADD . /usr/src/gapic-generator-python/

gapic/schema/api.py

@@ -23,7 +23,7 @@
 from itertools import chain
 from typing import Callable, List, Mapping, Sequence, Set, Tuple
 
-from google.api import annotations_pb2
+from google.longrunning import operations_pb2
 from google.protobuf import descriptor_pb2
 
 from gapic.schema import metadata
@@ -525,7 +525,7 @@ def _get_methods(self, methods: List[descriptor_pb2.MethodDescriptorProto],
         # Iterate over the methods and collect them into a dictionary.
         answer = collections.OrderedDict()
         for meth_pb, i in zip(methods, range(0, sys.maxsize)):
-            lro = meth_pb.options.Extensions[annotations_pb2.operation]
+            lro = meth_pb.options.Extensions[operations_pb2.operation_info]
 
             # If the output type is google.longrunning.Operation, we use
             # a specialized object in its place.

gapic/schema/naming.py

@@ -17,7 +17,7 @@
 import re
 from typing import Iterable, Sequence, Tuple
 
-from google.api import annotations_pb2
+from google.api import client_pb2
 from google.protobuf import descriptor_pb2
 
 from gapic import utils
@@ -37,9 +37,12 @@ class Naming:
     namespace: Tuple[str] = dataclasses.field(default_factory=tuple)
     version: str = ''
     product_name: str = ''
-    product_url: str = ''
     proto_package: str = ''
 
+    def __post_init__(self):
+        if not self.product_name:
+            self.__dict__['product_name'] = self.name
+
     @classmethod
     def build(cls,
             *file_descriptors: Iterable[descriptor_pb2.FileDescriptorProto]
@@ -118,38 +121,33 @@ def build(cls,
         #
         # This creates a naming class non-empty metadata annotation and
         # uses Python's set logic to de-duplicate. There should only be one.
-        metadata_info = set()
+        explicit_pkgs = set()
         for fd in file_descriptors:
-            meta = fd.options.Extensions[annotations_pb2.metadata]
+            pkg = fd.options.Extensions[client_pb2.client_package]
             naming = cls(
-                name=meta.package_name or meta.product_name,
-                namespace=tuple(meta.package_namespace),
-                product_name=meta.product_name,
-                product_url=meta.product_uri,
-                version='',
+                name=pkg.title or pkg.product_title,
+                namespace=tuple(pkg.namespace),
+                version=pkg.version,
             )
             if naming:
-                metadata_info.add(naming)
+                explicit_pkgs.add(naming)
 
         # Sanity check: Ensure that any google.api.metadata provisions were
         # consistent.
-        if len(metadata_info) > 1:
+        if len(explicit_pkgs) > 1:
             raise ValueError(
-                'If the google.api.metadata annotation is provided in more '
-                'than one file, it must be consistent.',
+                'If the google.api.client_package annotation is provided in '
+                'more than one file, it must be consistent.',
             )
 
         # Merge the package naming information and the metadata naming
         # information, with the latter being preferred.
         # Return a Naming object which effectively merges them.
-        answer = package_info
-        if len(metadata_info):
-            for k, v in dataclasses.asdict(metadata_info.pop()).items():
-                # Sanity check: We only want to overwrite anything if the
-                # new value is truthy.
-                if v:
-                    answer = dataclasses.replace(answer, **{k: v})
-        return answer
+        if len(explicit_pkgs):
+            return dataclasses.replace(package_info,
+                **dataclasses.asdict(explicit_pkgs.pop()),
+            )
+        return package_info
 
     def __bool__(self):
         """Return True if any of the fields are truthy, False otherwise."""

gapic/schema/wrappers.py

@@ -34,7 +34,8 @@
 from typing import Iterable, List, Mapping, Sequence, Set, Tuple, Union
 
 from google.api import annotations_pb2
-from google.api import signature_pb2
+from google.api import client_pb2
+from google.api import field_behavior_pb2
 from google.protobuf import descriptor_pb2
 
 from gapic import utils
@@ -92,7 +93,8 @@ def required(self) -> bool:
         Returns:
             bool: Whether this field is required.
         """
-        return bool(self.options.Extensions[annotations_pb2.required])
+        return (field_behavior_pb2.FieldBehavior.Value('REQUIRED') in
+                self.options.Extensions[field_behavior_pb2.field_behavior])
 
     @utils.cached_property
     def type(self) -> Union['MessageType', 'EnumType', 'PythonType']:
@@ -434,28 +436,22 @@ def ref_types(self) -> Sequence[Union[MessageType, EnumType]]:
         return tuple(answer)
 
     @utils.cached_property
-    def signatures(self) -> Tuple[signature_pb2.MethodSignature]:
+    def signatures(self) -> 'MethodSignatures':
         """Return the signature defined for this method."""
-        sig_pb2 = self.options.Extensions[annotations_pb2.method_signature]
-
-        # Sanity check: If there are no signatures (which should be by far
-        # the common case), just abort now.
-        if len(sig_pb2.fields) == 0:
-            return MethodSignatures(all=())
+        signatures = self.options.Extensions[client_pb2.method_signature]
 
         # Signatures are annotated with an `additional_signatures` key that
         # allows for specifying additional signatures. This is an uncommon
         # case but we still want to deal with it.
         answer = []
-        for sig in (sig_pb2,) + tuple(sig_pb2.additional_signatures):
+        for sig in signatures:
             # Build a MethodSignature object with the appropriate name
             # and fields. The fields are field objects, retrieved from
             # the method's `input` message.
             answer.append(MethodSignature(
-                name=sig.function_name if sig.function_name else self.name,
                 fields=collections.OrderedDict([
                     (f.split('.')[-1], self.input.get_field(*f.split('.')))
-                    for f in sig.fields
+                    for f in sig.split(',')
                 ]),
             ))
 
@@ -478,7 +474,6 @@ def with_context(self, *, collisions: Set[str]) -> 'Method':
 
 @dataclasses.dataclass(frozen=True)
 class MethodSignature:
-    name: str
     fields: Mapping[str, Field]
 
     @utils.cached_property
@@ -551,8 +546,8 @@ def host(self) -> str:
         Returns:
             str: The hostname, with no protocol and no trailing ``/``.
         """
-        if self.options.Extensions[annotations_pb2.default_host]:
-            return self.options.Extensions[annotations_pb2.default_host]
+        if self.options.Extensions[client_pb2.default_host]:
+            return self.options.Extensions[client_pb2.default_host]
         return None
 
     @property
@@ -562,8 +557,10 @@ def oauth_scopes(self) -> Sequence[str]:
         Returns:
             Sequence[str]: A sequence of OAuth scopes.
         """
-        oauth = self.options.Extensions[annotations_pb2.oauth]
-        return tuple(oauth.scopes)
+        # Return the OAuth scopes, split on comma.
+        return tuple([i.strip() for i in
+            self.options.Extensions[client_pb2.oauth_scopes].split(',')
+            if i])
 
     @property
     def module_name(self) -> str:

gapic/templates/setup.py.j2

@@ -20,7 +20,7 @@ setuptools.setup(
     include_package_data=True,
     install_requires=(
         'google-api-core >= 1.3.0, < 2.0.0dev',
-        'googleapis-common-protos >= 1.6.0b6',
+        'googleapis-common-protos >= 1.6.0b7',
         'grpcio >= 1.10.0',
         'proto-plus >= 0.3.0',
     ),

nox.py → noxfile.py

@@ -56,7 +56,7 @@ def showcase(session):
 
     # Install a client library for Showcase.
     with tempfile.TemporaryDirectory() as tmp_dir:
-        showcase_version = '0.0.9'
+        showcase_version = '0.0.12'
 
         # Download the Showcase descriptor.
         session.run(

setup.py

@@ -42,7 +42,7 @@
     include_package_data=True,
     install_requires=(
         'click >= 6.7',
-        'googleapis-common-protos >= 1.6.0b6',
+        'googleapis-common-protos >= 1.6.0b7',
         'jinja2 >= 2.10',
         'protobuf >= 3.5.1',
         'pypandoc >= 1.4',

tests/unit/generator/test_generator.py

@@ -260,5 +260,4 @@ def make_naming(**kwargs) -> naming.Naming:
     kwargs.setdefault('namespace', ('Google', 'Cloud'))
     kwargs.setdefault('version', 'v1')
     kwargs.setdefault('product_name', 'Hatstand')
-    kwargs.setdefault('product_url', 'https://cloud.google.com/hatstand/')
     return naming.Naming(**kwargs)

tests/unit/schema/test_api.py

@@ -17,8 +17,7 @@
 
 import pytest
 
-from google.api import annotations_pb2
-from google.api import longrunning_pb2
+from google.longrunning import operations_pb2
 from google.protobuf import descriptor_pb2
 
 from gapic.schema import api
@@ -461,8 +460,8 @@ def test_lro():
         input_type='google.example.v3.AsyncDoThingRequest',
         output_type='google.longrunning.Operation',
     )
-    method_pb2.options.Extensions[annotations_pb2.operation].MergeFrom(
-        longrunning_pb2.OperationData(
+    method_pb2.options.Extensions[operations_pb2.operation_info].MergeFrom(
+        operations_pb2.OperationInfo(
             response_type='google.example.v3.AsyncDoThingResponse',
             metadata_type='google.example.v3.AsyncDoThingMetadata',
         ),
@@ -606,5 +605,4 @@ def make_naming(**kwargs) -> naming.Naming:
     kwargs.setdefault('namespace', ('Google', 'Cloud'))
     kwargs.setdefault('version', 'v1')
     kwargs.setdefault('product_name', 'Hatstand')
-    kwargs.setdefault('product_url', 'https://cloud.google.com/hatstand/')
     return naming.Naming(**kwargs)

tests/unit/schema/test_naming.py

@@ -14,8 +14,7 @@
 
 import pytest
 
-from google.api import annotations_pb2
-from google.api import metadata_pb2
+from google.api import client_pb2
 from google.protobuf import descriptor_pb2
 
 from gapic.schema import naming
@@ -116,8 +115,12 @@ def test_build_with_annotations():
         name='spanner.proto',
         package='google.spanner.v1',
     )
-    proto.options.Extensions[annotations_pb2.metadata].MergeFrom(
-        metadata_pb2.Metadata(package_namespace=['Google', 'Cloud']),
+    proto.options.Extensions[client_pb2.client_package].MergeFrom(
+        client_pb2.Package(
+            namespace=['Google', 'Cloud'],
+            title='Spanner',
+            version='v1',
+        ),
     )
     n = naming.Naming.build(proto)
     assert n.name == 'Spanner'
@@ -146,8 +149,8 @@ def test_inconsistent_metadata_error():
         name='spanner.proto',
         package='google.spanner.v1',
     )
-    proto1.options.Extensions[annotations_pb2.metadata].MergeFrom(
-        metadata_pb2.Metadata(package_namespace=['Google', 'Cloud']),
+    proto1.options.Extensions[client_pb2.client_package].MergeFrom(
+        client_pb2.Package(namespace=['Google', 'Cloud']),
     )
 
     # Set up the second proto.
@@ -156,9 +159,8 @@ def test_inconsistent_metadata_error():
         name='spanner2.proto',
         package='google.spanner.v1',
     )
-    proto2.options.Extensions[annotations_pb2.metadata].MergeFrom(
-        metadata_pb2.Metadata(package_namespace=['Google', 'Cloud'],
-                              package_name='Spanner'),
+    proto2.options.Extensions[client_pb2.client_package].MergeFrom(
+        client_pb2.Package(title='Spanner', namespace=['Google', 'Cloud']),
     )
 
     # This should error. Even though the data in the metadata is consistent,
@@ -193,5 +195,4 @@ def make_naming(**kwargs) -> naming.Naming:
     kwargs.setdefault('namespace', ('Google', 'Cloud'))
     kwargs.setdefault('version', 'v1')
     kwargs.setdefault('product_name', 'Hatstand')
-    kwargs.setdefault('product_url', 'https://cloud.google.com/hatstand/')
     return naming.Naming(**kwargs)

tests/unit/schema/wrappers/test_field.py

@@ -14,7 +14,7 @@
 
 import pytest
 
-from google.api import annotations_pb2
+from google.api import field_behavior_pb2
 from google.protobuf import descriptor_pb2
 
 from gapic.schema import wrappers
@@ -83,7 +83,9 @@ def test_not_repeated():
 
 def test_required():
     field = make_field()
-    field.options.Extensions[annotations_pb2.required] = True
+    field.options.Extensions[field_behavior_pb2.field_behavior].append(
+        field_behavior_pb2.FieldBehavior.Value('REQUIRED')
+    )
     assert field.required