Merge branch 'main' of github.com:pinecone-io/pinecone-python-client

* 'main' of github.com:pinecone-io/pinecone-python-client:
  [skip ci] Bump version to v4.1.0
  Bump tqdm from 4.66.1 to 4.66.3 (pinecone-io#344)
  Bump idna from 3.4 to 3.7 (pinecone-io#345)
  Bump jinja2 from 3.1.3 to 3.1.4 (pinecone-io#343)
  Add better error messages for mistaken `from_texts` and `from_documents` (pinecone-io#342)
  Support proxy_url and ssl_ca_certs options for gRPC (pinecone-io#341)
  Remove serverless public preview warnings (pinecone-io#340)
  [skip ci] Bump version to v4.0.0
  Improve upsert throughput by 3x (pinecone-io#334)
  Remove `merge` workflow and update `build-and-publish-docs` workflow to be manually runnable (pinecone-io#335)
  [skip ci] Bump version to v3.2.2
  [Fix] openapi_config deprecation warning incorrectly shown (pinecone-io#327)
  Add grpc unit test run, expand testing of VectorFactory (pinecone-io#326)
  [skip ci] Bump version to v3.2.1
  Allow clients to tag requests with a source_tag (pinecone-io#324)
  [skip ci] Bump version to v3.2.0
  Revise proxy configuration, add integration testing (pinecone-io#325)
  [Fix] Configuring SSL proxy via openapi_config object (pinecone-io#321)
  Update README.md (pinecone-io#323)

.github/workflows/merge.yaml → .github/workflows/build-and-publish-docs.yaml

@@ -1,10 +1,11 @@
-name: 'CI (main)'
+name: "Build and publish documentation to sdk-docs"
 
 on:
-  push:
-    branches:
-      - main
   workflow_dispatch: {}
+  workflow_call:
+    secrets:
+      SSH_DEPLOY_KEY:
+        required: true
 
 jobs:
   build-and-deploy-documentation:
@@ -29,4 +30,4 @@ jobs:
           user-email: clients@pinecone.io
           target-branch: main
           target-directory: python
-          commit-message: 'Python: automated documentation build - pinecone-python-client merge SHA: ${{ github.sha }}'
+          commit-message: "Python: automated documentation build - pinecone-python-client merge SHA: ${{ github.sha }}"

.github/workflows/testing-dependency.yaml

@@ -56,12 +56,11 @@ jobs:
           # - 4.1.0
           - 4.3.3
         protobuf_version:
-          - 3.20.3
+          - 4.25.3
+        protoc-gen-openapiv2:
+          - 0.0.1
         googleapis_common_protos_version:
-          - 1.53.0
           - 1.62.0
-        grpc_gateway_protoc_gen_openapiv2_version:
-          - 0.1.0
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test-dependency-grpc
@@ -92,12 +91,11 @@ jobs:
           - 3.1.3
           - 4.3.3
         protobuf_version:
-          - 3.20.3
+          - 4.25.3
+        protoc-gen-openapiv2:
+          - 0.0.1
         googleapis_common_protos_version:
-          - 1.53.0
           - 1.62.0
-        grpc_gateway_protoc_gen_openapiv2_version:
-          - 0.1.0
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/test-dependency-grpc
@@ -158,3 +156,19 @@ jobs:
           index_name: '${{ needs.dependency-matrix-setup.outputs.index_name }}'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
           urllib3_version: '${{ matrix.urllib3_version }}'
+
+  deps-cleanup:
+    name: Deps cleanup
+    runs-on: ubuntu-latest
+    needs:
+      - dependency-matrix-setup
+      - dependency-matrix-grpc
+      - dependency-matrix-grpc-312
+      - dependency-matrix-rest
+      - dependency-matrix-rest-312
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/delete-index
+        with:
+          index_name: '${{ needs.dependency-matrix-setup.outputs.index_name }}'
+          PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'

.github/workflows/testing-integration.yaml

@@ -3,6 +3,54 @@ name: "Integration Tests"
   workflow_call: {}
 
 jobs:
+  # setup-index:
+  #   name: Setup proxyconfig test index
+  #   runs-on: ubuntu-latest
+  #   outputs:
+  #     index_name: ${{ steps.create-index.outputs.index_name }}
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - name: Create index
+  #       id: create-index
+  #       uses: ./.github/actions/create-index
+  #       with:
+  #         PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+  #         NAME_PREFIX: 'proxyconfig-'
+  #         REGION: 'us-west-2'
+  #         CLOUD: 'aws'
+  #         DIMENSION: 1536
+  #         METRIC: 'cosine'
+
+  # proxy-config:
+  #   name: Proxy config tests
+  #   needs: [setup-index]
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-python@v5
+  #       with:
+  #         python-version: 3.9
+  #     - name: Setup Poetry
+  #       uses: ./.github/actions/setup-poetry
+  #     - name: 'Run integration tests (proxy config)'
+  #       run: |
+  #         poetry run pytest tests/integration/proxy_config -s -v
+  #       env:
+  #         PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
+  #         PINECONE_INDEX_NAME: ${{ needs.setup-index.outputs.index_name }}
+  #     - name: Upload logs
+  #       if: always()
+  #       uses: actions/upload-artifact@v4
+  #       with:
+  #         name: proxy_config_test_logs
+  #         path: tests/integration/proxy_config/logs
+  #     - name: Cleanup index
+  #       if: always()
+  #       uses: ./.github/actions/delete-index
+  #       with:
+  #         PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
+  #         INDEX_NAME: ${{ needs.setup-index.outputs.index_name }}
+
   data-plane-serverless:
     name: Data plane serverless integration tests
     runs-on: ubuntu-latest
@@ -27,7 +75,6 @@ jobs:
           spec: '${{ matrix.spec }}'
           PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
           freshness_timeout_seconds: 600
-
 #   data-plane-pod:
 #     name: Data plane pod integration tests
 #     runs-on: ubuntu-latest

.github/workflows/testing-unit.yaml

@@ -7,6 +7,7 @@ jobs:
     name: Unit tests
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version:
           - 3.8
@@ -28,8 +29,11 @@ jobs:
         with:
           include_grpc: '${{ matrix.use_grpc }}'
           include_types: true
-      - name: Run unit tests
+      - name: Run unit tests (REST)
         run: poetry run pytest --cov=pinecone --timeout=120 tests/unit
+      - name: Run unit tests (GRPC)
+        if: ${{ matrix.use_grpc == true }}
+        run: poetry run pytest --cov=pinecone/grpc --timeout=120 tests/unit_grpc
       - name: mypy check
         env:
           INCLUDE_GRPC: '${{ matrix.use_grpc }}'

.gitignore

@@ -154,3 +154,5 @@ dmypy.json
 # Datasets
 *.hdf5
 *~
+
+tests/integration/proxy_config/logs

README.md

@@ -43,10 +43,10 @@ pip3 install "pinecone-client[grpc]"==3.0.0
 
 ```shell
 # Install the latest version
-poetry add pinecone
+poetry add pinecone-client
 
 # Install the latest version, with grpc extras
-poetry add pinecone --extras grpc
+poetry add pinecone-client --extras grpc
 
 # Install a specific version
 poetry add pinecone-client==3.0.0
@@ -71,14 +71,14 @@ from pinecone import Pinecone
 pc = Pinecone() # This reads the PINECONE_API_KEY env var
 ```
 
-#### Using a configuration object
+#### Using configuration keyword params
 
 If you prefer to pass configuration in code, for example if you have a complex application that needs to interact with multiple different Pinecone projects, the constructor accepts a keyword argument for `api_key`.
 
 If you pass configuration in this way, you can have full control over what name to use for the environment variable, sidestepping any issues that would result
 from two different client instances both needing to read the same `PINECONE_API_KEY` variable that the client implicitly checks for.
 
-Configuration passed with keyword arguments takes precedent over environment variables.
+Configuration passed with keyword arguments takes precedence over environment variables.
 
 ```python
 import os
@@ -87,6 +87,81 @@ from pinecone import Pinecone
 pc = Pinecone(api_key=os.environ.get('CUSTOM_VAR'))
 ```
 
+### Proxy configuration
+
+If your network setup requires you to interact with Pinecone via a proxy, you will need
+to pass additional configuration using optional keyword parameters. These optional parameters are forwarded to `urllib3`, which is the underlying library currently used by the Pinecone client to make HTTP requests. You may find it helpful to refer to the [urllib3 documentation on working with proxies](https://urllib3.readthedocs.io/en/stable/advanced-usage.html#http-and-https-proxies) while troubleshooting these settings.
+
+Here is a basic example:
+
+```python
+from pinecone import Pinecone
+
+pc = Pinecone(
+    api_key='YOUR_API_KEY',
+    proxy_url='https://your-proxy.com'
+)
+
+pc.list_indexes()
+```
+
+If your proxy requires authentication, you can pass those values in a header dictionary using the `proxy_headers` parameter.
+
+```python
+from pinecone import Pinecone
+import urllib3 import make_headers
+
+pc = Pinecone(
+    api_key='YOUR_API_KEY',
+    proxy_url='https://your-proxy.com',
+    proxy_headers=make_headers(proxy_basic_auth='username:password')
+)
+
+pc.list_indexes()
+```
+
+### Using proxies with self-signed certificates
+
+By default the Pinecone Python client will perform SSL certificate verification
+using the CA bundle maintained by Mozilla in the [certifi](https://pypi.org/project/certifi/) package.
+
+If your proxy server is using a self-signed certificate, you will need to pass the path to the certificate in PEM format using the `ssl_ca_certs` parameter.
+
+```python
+from pinecone import Pinecone
+import urllib3 import make_headers
+
+pc = Pinecone(
+    api_key="YOUR_API_KEY",
+    proxy_url='https://your-proxy.com',
+    proxy_headers=make_headers(proxy_basic_auth='username:password'),
+    ssl_ca_certs='path/to/cert-bundle.pem'
+)
+
+pc.list_indexes()
+```
+
+### Disabling SSL verification
+
+If you would like to disable SSL verification, you can pass the `ssl_verify`
+parameter with a value of `False`. We do not recommend going to production with SSL verification disabled.
+
+```python
+from pinecone import Pinecone
+import urllib3 import make_headers
+
+pc = Pinecone(
+    api_key='YOUR_API_KEY',
+    proxy_url='https://your-proxy.com',
+    proxy_headers=make_headers(proxy_basic_auth='username:password'),
+    ssl_ca_certs='path/to/cert-bundle.pem',
+    ssl_verify=False
+)
+
+pc.list_indexes()
+
+```
+
 ### Working with GRPC (for improved performance)
 
 If you've followed instructions above to install with optional `grpc` extras, you can unlock some performance improvements by working with an alternative version of the client imported from the `pinecone.grpc` subpackage.
@@ -110,9 +185,8 @@ index.query(vector=[...], top_key=10)
 
 ### Create a serverless index
 
-> [!WARNING]  
-> Serverless indexes are in **public preview** and are available only on AWS in the
-> `us-west-2` region. Check the [current limitations](https://docs.pinecone.io/docs/limits#serverless-index-limitations) and test thoroughly before using it in production.
+The following example creates a serverless index in the `us-west-2`
+region of AWS. For more information on serverless and regional availability, see [Understanding indexes](https://docs.pinecone.io/guides/indexes/understanding-indexes#serverless-indexes).
 
 ```python
 from pinecone import Pinecone, ServerlessSpec
@@ -341,9 +415,9 @@ update_response = index.update(
 
 ## List vectors
 
-The `list` and `list_paginated` methods can be used to list vector ids matching a particular id prefix. 
+The `list` and `list_paginated` methods can be used to list vector ids matching a particular id prefix.
 With clever assignment of vector ids, this can be used to help model hierarchical relationships between
-different vectors such as when there are embeddings for multiple chunks or fragments related to the 
+different vectors such as when there are embeddings for multiple chunks or fragments related to the
 same document.
 
 The `list` method returns a generator that handles pagination on your behalf.

pinecone/__version__

@@ -1 +1 @@
-3.1.0
+4.1.0

pinecone/config/config.py

@@ -5,12 +5,17 @@
 from pinecone.config.openapi import OpenApiConfigFactory
 from pinecone.core.client.configuration import Configuration as OpenApiConfiguration
 from pinecone.utils import normalize_host
+from pinecone.utils.constants import SOURCE_TAG
 
 class Config(NamedTuple):
     api_key: str = ""
     host: str = ""
-    openapi_config: Optional[OpenApiConfiguration] = None
+    proxy_url: Optional[str] = None
+    proxy_headers: Optional[Dict[str, str]] = None
+    ssl_ca_certs: Optional[str] = None
+    ssl_verify: Optional[bool] = None
     additional_headers: Optional[Dict[str, str]] = {}
+    source_tag: Optional[str] = None
 
 class ConfigBuilder:
     """
@@ -33,23 +38,45 @@ class ConfigBuilder:
     def build(
         api_key: Optional[str] = None,
         host: Optional[str] = None,
-        openapi_config: Optional[OpenApiConfiguration] = None,
+        proxy_url: Optional[str] = None,
+        proxy_headers: Optional[Dict[str, str]] = None,
+        ssl_ca_certs: Optional[str] = None,
+        ssl_verify: Optional[bool] = None,
         additional_headers: Optional[Dict[str, str]] = {},
         **kwargs,
     ) -> Config:
         api_key = api_key or kwargs.pop("api_key", None) or os.getenv("PINECONE_API_KEY")
         host = host or kwargs.pop("host", None)
         host = normalize_host(host)
+        source_tag = kwargs.pop(SOURCE_TAG, None)
 
         if not api_key:
             raise PineconeConfigurationError("You haven't specified an Api-Key.")
         if not host:
             raise PineconeConfigurationError("You haven't specified a host.")
 
-        openapi_config = (
-            openapi_config
-            or kwargs.pop("openapi_config", None)
-            or OpenApiConfigFactory.build(api_key=api_key, host=host)
-        )
+        return Config(api_key, host, proxy_url, proxy_headers, ssl_ca_certs, ssl_verify, additional_headers, source_tag)
+
+    @staticmethod
+    def build_openapi_config(
+        config: Config, openapi_config: Optional[OpenApiConfiguration] = None, **kwargs
+    ) -> OpenApiConfiguration:
+        if openapi_config:
+            openapi_config = OpenApiConfigFactory.copy(openapi_config=openapi_config, api_key=config.api_key, host=config.host)
+        elif openapi_config is None:
+            openapi_config = OpenApiConfigFactory.build(api_key=config.api_key, host=config.host)
 
-        return Config(api_key, host, openapi_config, additional_headers)
+        # Check if value passed before overriding any values present
+        # in the openapi_config. This means if the user has passed 
+        # an openapi_config object and a kwarg for the same setting,
+        # the kwarg will take precedence.
+        if (config.proxy_url):
+            openapi_config.proxy = config.proxy_url
+        if (config.proxy_headers):
+            openapi_config.proxy_headers = config.proxy_headers
+        if (config.ssl_ca_certs):
+            openapi_config.ssl_ca_cert = config.ssl_ca_certs
+        if (config.ssl_verify != None):
+            openapi_config.verify_ssl = config.ssl_verify
+
+        return openapi_config