From e8d69a834952200335e6c6c42d3f08ce2fce170d Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Wed, 16 Oct 2024 17:53:52 -0700 Subject: [PATCH 1/8] watermark Change-Id: I271993e07bfc034d89cd74013339046a21d0b472 --- google/generativeai/__init__.py | 1 - google/generativeai/types/content_types.py | 5 +- .../vision_models/_vision_models.py | 64 ++++++++++++++++++- 3 files changed, 66 insertions(+), 4 deletions(-) diff --git a/google/generativeai/__init__.py b/google/generativeai/__init__.py index 73025a1b4..66af17641 100644 --- a/google/generativeai/__init__.py +++ b/google/generativeai/__init__.py @@ -79,7 +79,6 @@ __version__ = version.__version__ -del embedding del files del generative_models del models diff --git a/google/generativeai/types/content_types.py b/google/generativeai/types/content_types.py index f3db610e1..6d4668989 100644 --- a/google/generativeai/types/content_types.py +++ b/google/generativeai/types/content_types.py @@ -35,6 +35,7 @@ import IPython.display IMAGE_TYPES = (PIL.Image.Image, IPython.display.Image) + ImageType = PIL.Image.Image | IPython.display.Image else: IMAGE_TYPES = () try: @@ -52,6 +53,8 @@ except ImportError: IPython = None + ImageType = Union[*IMAGE_TYPES] + __all__ = [ "BlobDict", @@ -123,7 +126,7 @@ def webp_blob(image: PIL.Image.Image) -> protos.Blob: return file_blob(image) or webp_blob(image) -def image_to_blob(image) -> protos.Blob: +def image_to_blob(image: ImageType) -> protos.Blob: if PIL is not None: if isinstance(image, PIL.Image.Image): return _pil_to_blob(image) diff --git a/google/generativeai/vision_models/_vision_models.py b/google/generativeai/vision_models/_vision_models.py index 3e7dbfa22..4779874fc 100644 --- a/google/generativeai/vision_models/_vision_models.py +++ b/google/generativeai/vision_models/_vision_models.py @@ -18,15 +18,16 @@ import base64 import collections import dataclasses -import hashlib import io import json +import os import pathlib import typing from typing import Any, Dict, List, Literal, Optional, Union from google.generativeai import client from google.generativeai import protos +from google.generativeai.types import content_types from google.protobuf import struct_pb2 @@ -110,6 +111,8 @@ def to_mapping_value(value) -> struct_pb2.Struct: PersonGeneration = Literal["dont_allow", "allow_adult"] PERSON_GENERATIONS = PersonGeneration.__args__ # type: ignore +ImageLikeType = Union["Image", pathlib.Path(), content_types.ImageType] + class Image: """Image.""" @@ -131,7 +134,7 @@ def __init__( self._image_bytes = image_bytes @staticmethod - def load_from_file(location: str) -> "Image": + def load_from_file(location: os.PathLike) -> "Image": """Loads image from local file or Google Cloud Storage. Args: @@ -206,6 +209,63 @@ def _as_base64_string(self) -> str: def _repr_png_(self): return self._pil_image._repr_png_() # type:ignore + def check_watermark(self: ImageLikeType, model_id: str = "models/image-verification-001"): + img = None + if isinstance(self, Image): + img = self + elif isinstance(self, pathlib.Path): + img = Image.load_from_file(self) + elif IPython_display is not None and isinstance(self, IPython_display.Image): + img = Image(image_bytes=self.data) + elif PIL_Image is not None and isinstance(self, PIL_Image.Image): + blob = content_types._pil_to_blob(self) + img = Image(image_bytes=blob.data) + elif isinstance(self, protos.Blob): + img = Image(image_bytes=self.data) + else: + raise TypeError( + f"Not implemented: Could not convert a {type(img)} into `Image`\n {img=}" + ) + + prediction_client = client.get_default_prediction_client() + if not model_id.startswith("models/"): + model_id = f"models/{model_id}" + + # Note: Only a single prompt is supported by the service. + instance = {"image": {"bytesBase64Encoded": base64.b64encode(img._loaded_bytes).decode()}} + parameters = {"watermarkVerification": True} + + # This is to get around https://github.com/googleapis/proto-plus-python/issues/488 + pr = protos.PredictRequest.pb() + request = pr( + model=model_id, instances=[to_value(instance)], parameters=to_value(parameters) + ) + + response = prediction_client.predict(request) + + return CheckWatermarkResult(response.predictions) + + +class CheckWatermarkResult: + def __init__(self, predictions): + self._predictions = predictions + + @property + def decision(self): + return self._predictions[0]["decision"] + + def __str__(self): + return f"CheckWatermarkResult([{{'decision': {self.decision!r}}}])" + + def __bool__(self): + decision = self.decision + if decision == "ACCEPT": + return True + elif decision == "REJECT": + return False + else: + raise ValueError("Unrecognized result") + class ImageGenerationModel: """Generates images from text prompt. From 78ec5e83fcdc33309a941f80c162c90dd8da30a7 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Thu, 17 Oct 2024 10:33:57 -0700 Subject: [PATCH 2/8] fix typing Change-Id: Ib026f3b66fb44c6ad35a15030b03204306b35443 --- google/generativeai/types/content_types.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/google/generativeai/types/content_types.py b/google/generativeai/types/content_types.py index 6d4668989..e415870be 100644 --- a/google/generativeai/types/content_types.py +++ b/google/generativeai/types/content_types.py @@ -37,23 +37,29 @@ IMAGE_TYPES = (PIL.Image.Image, IPython.display.Image) ImageType = PIL.Image.Image | IPython.display.Image else: - IMAGE_TYPES = () try: import PIL.Image import PIL.ImageFile - - IMAGE_TYPES = IMAGE_TYPES + (PIL.Image.Image,) except ImportError: PIL = None try: import IPython.display - - IMAGE_TYPES = IMAGE_TYPES + (IPython.display.Image,) except ImportError: IPython = None - ImageType = Union[*IMAGE_TYPES] + if PIL is not None and IPython is not None: + IMAGE_TYPES = ( PIL.Image.Image, IPython.display.Image) + ImageType = Union[PIL.Image.Image, IPython.display.Image] + elif PIL is not None and IPython is None: + IMAGE_TYPES = (PIL.Image.Image,) + ImageType = Union[PIL.Image.Image] + elif PIL is None and IPython is not None: + IMAGE_TYPES = (IPython.display.Image,) + ImageType = Union[IPython.display.Image,] + else: + IMAGE_TYPES=() + ImageType=Union __all__ = [ From 10af103fde7b4f38f07b8833b0c6eec99bcac553 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Thu, 17 Oct 2024 10:34:47 -0700 Subject: [PATCH 3/8] format Change-Id: I24304a80e6689b4fcec3155f7274e24db712b402 --- google/generativeai/types/content_types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/google/generativeai/types/content_types.py b/google/generativeai/types/content_types.py index e415870be..1dd49c9c9 100644 --- a/google/generativeai/types/content_types.py +++ b/google/generativeai/types/content_types.py @@ -49,7 +49,7 @@ IPython = None if PIL is not None and IPython is not None: - IMAGE_TYPES = ( PIL.Image.Image, IPython.display.Image) + IMAGE_TYPES = (PIL.Image.Image, IPython.display.Image) ImageType = Union[PIL.Image.Image, IPython.display.Image] elif PIL is not None and IPython is None: IMAGE_TYPES = (PIL.Image.Image,) @@ -58,8 +58,8 @@ IMAGE_TYPES = (IPython.display.Image,) ImageType = Union[IPython.display.Image,] else: - IMAGE_TYPES=() - ImageType=Union + IMAGE_TYPES = () + ImageType = Union __all__ = [ From 63e050124dffd42809697d12ec8f026dfcf0e9c8 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Thu, 17 Oct 2024 10:38:28 -0700 Subject: [PATCH 4/8] fix test Change-Id: I68cb74f8ee5f224942417ea4e5fc4232ec688977 --- google/generativeai/vision_models/_vision_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/generativeai/vision_models/_vision_models.py b/google/generativeai/vision_models/_vision_models.py index 4779874fc..843d1354e 100644 --- a/google/generativeai/vision_models/_vision_models.py +++ b/google/generativeai/vision_models/_vision_models.py @@ -111,7 +111,7 @@ def to_mapping_value(value) -> struct_pb2.Struct: PersonGeneration = Literal["dont_allow", "allow_adult"] PERSON_GENERATIONS = PersonGeneration.__args__ # type: ignore -ImageLikeType = Union["Image", pathlib.Path(), content_types.ImageType] +ImageLikeType = Union["Image", pathlib.Path, content_types.ImageType] class Image: From 17b5d568bea63eff2c23b69e4c7a9bf4ca58bd1e Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Thu, 17 Oct 2024 10:56:08 -0700 Subject: [PATCH 5/8] make check_watermark a stand alone function Change-Id: I2d72620359dcc70fe8e720a14f78d83f75a42d90 --- google/generativeai/vision_models/__init__.py | 1 + .../vision_models/_vision_models.py | 82 +++++++++++-------- 2 files changed, 47 insertions(+), 36 deletions(-) diff --git a/google/generativeai/vision_models/__init__.py b/google/generativeai/vision_models/__init__.py index 65a545831..2a4a27e32 100644 --- a/google/generativeai/vision_models/__init__.py +++ b/google/generativeai/vision_models/__init__.py @@ -15,6 +15,7 @@ """Classes for working with vision models.""" from google.generativeai.vision_models._vision_models import ( + check_watermark, Image, GeneratedImage, ImageGenerationModel, diff --git a/google/generativeai/vision_models/_vision_models.py b/google/generativeai/vision_models/_vision_models.py index 843d1354e..63ca7a25d 100644 --- a/google/generativeai/vision_models/_vision_models.py +++ b/google/generativeai/vision_models/_vision_models.py @@ -114,6 +114,50 @@ def to_mapping_value(value) -> struct_pb2.Struct: ImageLikeType = Union["Image", pathlib.Path, content_types.ImageType] +def check_watermark( + img: ImageLikeType, model_id: str = "models/image-verification-001" +) -> "CheckWatermarkResult": + """Checks if an image has a Google-AI watermark. + + Args: + img: can be a `pathlib.Path` or a `PIL.Image.Image`, `IPythin.display.Image`, or `google.generativeai.Image`. + model_id: Which version of the image-verification model to send the image to. + + Returns: + + """ + if isinstance(img, Image): + pass + elif isinstance(img, pathlib.Path): + img = Image.load_from_file(img) + elif IPython_display is not None and isinstance(img, IPython_display.Image): + img = Image(image_bytes=img.data) + elif PIL_Image is not None and isinstance(img, PIL_Image.Image): + blob = content_types._pil_to_blob(img) + img = Image(image_bytes=blob.data) + elif isinstance(img, protos.Blob): + img = Image(image_bytes=img.data) + else: + raise TypeError( + f"Not implemented: Could not convert a {type(img)} into `Image`\n {img=}" + ) + + prediction_client = client.get_default_prediction_client() + if not model_id.startswith("models/"): + model_id = f"models/{model_id}" + + instance = {"image": {"bytesBase64Encoded": base64.b64encode(img._loaded_bytes).decode()}} + parameters = {"watermarkVerification": True} + + # This is to get around https://github.com/googleapis/proto-plus-python/issues/488 + pr = protos.PredictRequest.pb() + request = pr(model=model_id, instances=[to_value(instance)], parameters=to_value(parameters)) + + response = prediction_client.predict(request) + + return CheckWatermarkResult(response.predictions) + + class Image: """Image.""" @@ -209,41 +253,7 @@ def _as_base64_string(self) -> str: def _repr_png_(self): return self._pil_image._repr_png_() # type:ignore - def check_watermark(self: ImageLikeType, model_id: str = "models/image-verification-001"): - img = None - if isinstance(self, Image): - img = self - elif isinstance(self, pathlib.Path): - img = Image.load_from_file(self) - elif IPython_display is not None and isinstance(self, IPython_display.Image): - img = Image(image_bytes=self.data) - elif PIL_Image is not None and isinstance(self, PIL_Image.Image): - blob = content_types._pil_to_blob(self) - img = Image(image_bytes=blob.data) - elif isinstance(self, protos.Blob): - img = Image(image_bytes=self.data) - else: - raise TypeError( - f"Not implemented: Could not convert a {type(img)} into `Image`\n {img=}" - ) - - prediction_client = client.get_default_prediction_client() - if not model_id.startswith("models/"): - model_id = f"models/{model_id}" - - # Note: Only a single prompt is supported by the service. - instance = {"image": {"bytesBase64Encoded": base64.b64encode(img._loaded_bytes).decode()}} - parameters = {"watermarkVerification": True} - - # This is to get around https://github.com/googleapis/proto-plus-python/issues/488 - pr = protos.PredictRequest.pb() - request = pr( - model=model_id, instances=[to_value(instance)], parameters=to_value(parameters) - ) - - response = prediction_client.predict(request) - - return CheckWatermarkResult(response.predictions) + check_watermark = check_watermark class CheckWatermarkResult: @@ -539,7 +549,7 @@ def generation_parameters(self): return self._generation_parameters @staticmethod - def load_from_file(location: str) -> "GeneratedImage": + def load_from_file(location: os.PathLike) -> "GeneratedImage": """Loads image from file. Args: From e339835f340e9a77e85d8209a206ee245d0b2ddd Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Thu, 17 Oct 2024 11:20:48 -0700 Subject: [PATCH 6/8] simplify typing. Change-Id: I1b901cc40b4b029cb09699fc6eac77690622b6e8 --- google/generativeai/types/content_types.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/google/generativeai/types/content_types.py b/google/generativeai/types/content_types.py index 1dd49c9c9..23241a536 100644 --- a/google/generativeai/types/content_types.py +++ b/google/generativeai/types/content_types.py @@ -37,29 +37,23 @@ IMAGE_TYPES = (PIL.Image.Image, IPython.display.Image) ImageType = PIL.Image.Image | IPython.display.Image else: + IMAGE_TYPES = () try: import PIL.Image import PIL.ImageFile + + IMAGE_TYPES = IMAGE_TYPES + (PIL.Image.Image,) except ImportError: PIL = None try: import IPython.display + + IMAGE_TYPES = IMAGE_TYPES + (IPython.display.Image,) except ImportError: IPython = None - if PIL is not None and IPython is not None: - IMAGE_TYPES = (PIL.Image.Image, IPython.display.Image) - ImageType = Union[PIL.Image.Image, IPython.display.Image] - elif PIL is not None and IPython is None: - IMAGE_TYPES = (PIL.Image.Image,) - ImageType = Union[PIL.Image.Image] - elif PIL is None and IPython is not None: - IMAGE_TYPES = (IPython.display.Image,) - ImageType = Union[IPython.display.Image,] - else: - IMAGE_TYPES = () - ImageType = Union + ImageType = Union["PIL.Image.Image", "IPython.display.Image"] __all__ = [ From 7b557dfc417a175313050d093142fe63aa56145c Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Thu, 24 Oct 2024 10:47:26 -0700 Subject: [PATCH 7/8] Update google/generativeai/vision_models/_vision_models.py --- google/generativeai/vision_models/_vision_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/generativeai/vision_models/_vision_models.py b/google/generativeai/vision_models/_vision_models.py index 63ca7a25d..d0d5d7a04 100644 --- a/google/generativeai/vision_models/_vision_models.py +++ b/google/generativeai/vision_models/_vision_models.py @@ -274,7 +274,7 @@ def __bool__(self): elif decision == "REJECT": return False else: - raise ValueError("Unrecognized result") + raise ValueError(f"Unrecognized result: {decision}") class ImageGenerationModel: From 869c84c7fd7076d93f7cd4b58755612bf15cd3b2 Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Tue, 29 Oct 2024 06:42:05 -0700 Subject: [PATCH 8/8] Typo Co-authored-by: Mark McDonald --- google/generativeai/vision_models/_vision_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/generativeai/vision_models/_vision_models.py b/google/generativeai/vision_models/_vision_models.py index d0d5d7a04..52ec689a9 100644 --- a/google/generativeai/vision_models/_vision_models.py +++ b/google/generativeai/vision_models/_vision_models.py @@ -120,7 +120,7 @@ def check_watermark( """Checks if an image has a Google-AI watermark. Args: - img: can be a `pathlib.Path` or a `PIL.Image.Image`, `IPythin.display.Image`, or `google.generativeai.Image`. + img: can be a `pathlib.Path` or a `PIL.Image.Image`, `IPython.display.Image`, or `google.generativeai.Image`. model_id: Which version of the image-verification model to send the image to. Returns: