add regctl

CHANGELOG.md

@@ -4,6 +4,11 @@
 ## Unreleased
 
 
+## 1.6.0
+### Added
+- add regctl normalization schema
+
+
 ## 1.5.2
 ### Fixed
 - issue introduced in 1.5.1 that could cause some contact points to be lost

CHANGELOG.yaml

@@ -5,6 +5,9 @@ Unreleased:
   deprecated: []
   removed: []
   security: []
+1.6.0:
+  added:
+  - add regctl normalization schema
 1.5.2:
   fixed:
   - issue introduced in 1.5.1 that could cause some contact points to be lost

Ctl/VERSION

@@ -1 +1 @@
-1.5.2
+1.6.0

Ctl/config.yml

@@ -20,7 +20,7 @@ ctl:
         sign: true
 
     - name: version
-      type: version
+      type: semver2
       config:
         branch_dev: main
         branch_release: main

docs/api.md

@@ -0,0 +1,116 @@
+# API
+
+## Retrieve objects
+
+```python
+import rdap
+
+# instantiate client
+client = rdap.RdapClient()
+
+# asn
+as63311 = client.get_asn(63311)
+
+# domain
+domain = client.get_domain('example.com')
+
+# ip
+ip = client.get_ip('206.41.110.0')
+
+# entity
+entity = client.get_entity('NETWO7047-ARIN')
+```
+
+## Output normalized data (ASN example)
+
+```python
+import json
+import rdap
+
+# instantiate client
+client = rdap.RdapClient()
+
+# request asn
+as63311 = client.get_asn(63311)
+
+# normalized dict
+print(json.dumps(as63311.normalized, indent=2))
+```
+
+Output:
+```json
+{
+  "created": "2014-11-17T14:28:43-05:00",
+  "updated": "2018-10-24T22:58:16-04:00",
+  "asn": 63311,
+  "name": "20C",
+  "organization": {
+    "name": "20C, LLC"
+  },
+  "locations": [
+    {
+      "updated": "2014-08-05T15:21:11-04:00",
+      "country": null,
+      "city": null,
+      "postal_code": null,
+      "address": "303 W Ohio #1701\nChicago\nIL\n60654\nUnited States",
+      "geo": null,
+      "floor": null,
+      "suite": null
+    },
+    {
+      "updated": "2023-08-02T14:15:09-04:00",
+      "country": null,
+      "city": null,
+      "postal_code": null,
+      "address": "603 Discovery Dr\nWest Chicago\nIL\n60185\nUnited States",
+      "geo": null,
+      "floor": null,
+      "suite": null
+    }
+  ],
+  "contacts": [
+    {
+      "created": "2014-07-03T23:22:49-04:00",
+      "updated": "2023-08-02T14:15:09-04:00",
+      "name": "Network Engineers",
+      "roles": [
+        "abuse",
+        "admin",
+        "technical"
+      ],
+      "phone": "+1 978-636-0020",
+      "email": "neteng@20c.com"
+    }
+  ],
+  "sources": [
+    {
+      "created": "2014-11-17T14:28:43-05:00",
+      "updated": "2018-10-24T22:58:16-04:00",
+      "handle": "AS63311",
+      "urls": [
+        "https://rdap.org/autnum/63311",
+        "https://rdap.arin.net/registry/autnum/63311"
+      ],
+      "description": null
+    }
+  ]
+}
+```
+
+## Work with normalized data through pydantic models
+
+```python
+import rdap
+
+from rdap.schema.normalized import Network
+
+# instantiate client
+client = rdap.RdapClient()
+
+# request asn
+as63311 = Network(**client.get_asn(63311).normalized)
+
+for contact in as63311.contacts:
+    print(contact.name, contact.email)
+```

pyproject.toml

@@ -30,7 +30,9 @@ rdap = "rdap.cli:main"
 python = "^3.8"
 requests = ">=2.25.1"
 munge = {version = ">=1.3", extras = ["yaml"]}
-
+pydantic = ">=2.8.2"
+googlemaps = ">=4.10"
+phonenumbers = ">=8.13.0"
 
 [tool.poetry.dev-dependencies]
 # testing

rdap/cli.py

@@ -6,6 +6,7 @@
 
 import rdap
 from rdap.config import Config
+from rdap.context import RdapRequestContext
 
 
 def add_options(parser, options):
@@ -49,6 +50,9 @@
     parser.add_argument(
         "--parse", action="store_true", help="parse data into object before display"
     )
+    parser.add_argument(
+        "--normalize", action="store_true", help="normalize data before display"
+    )
     parser.add_argument("--rir", action="store_true", help="display rir", default=False)
     parser.add_argument(
         "--write-bootstrap-data",
@@ -75,13 +79,16 @@
 
     codec = munge.get_codec(output_format)()
     for each in argd["query"]:
-        obj = client.get(each)
-        if argd.get("rir", False):
-            print(f"rir: {obj.get_rir()}")
-        if argd.get("parse", False):
-            print(codec.dumps(obj.parsed()))
-        else:
-            print(codec.dumps(obj.data))
+        with RdapRequestContext(client=client):
+            obj = client.get(each)
+            if argd.get("rir", False):
+                print(f"rir: {obj.get_rir()}")
+            if argd.get("parse", False):
+                print(codec.dumps(obj.parsed()))
+            elif argd.get("normalize", False):
+                print(codec.dumps(obj.normalized))
+            else:
+                print(codec.dumps(obj.data))
 
     if argd.get("show_requests", False):
         print("# Requests")

rdap/client.py

@@ -11,6 +11,7 @@
 import rdap
 import rdap.bootstrap
 from rdap.config import Config
+from rdap.context import RdapRequestContext
 from rdap.exceptions import RdapHTTPError, RdapNotFoundError
 from rdap.objects import RdapAsn, RdapDomain, RdapEntity, RdapNetwork
 
@@ -129,20 +130,23 @@
         )
 
     def _get(self, url):
-        res = self.http.get(url, timeout=self.timeout)
-        for redir in res.history:
-            self._history.append((strip_auth(redir.url), redir.status_code))
-        self._history.append((strip_auth(res.url), res.status_code))
+        with RdapRequestContext(url) as ctx:
+            res = self.http.get(url, timeout=self.timeout)
+            for redir in res.history:
+                self._history.append((strip_auth(redir.url), redir.status_code))
+            self._history.append((strip_auth(res.url), res.status_code))
 
-        if res.status_code == 200:
-            return res
+            ctx.push_url(strip_auth(res.url))
 
-        msg = "RDAP lookup to {} returned {}".format(
-            strip_auth(res.url), res.status_code
-        )
-        if res.status_code == 404:
-            raise RdapNotFoundError(msg)
-        raise RdapHTTPError(msg)
+            if res.status_code == 200:
+                return res
+
+            msg = "RDAP lookup to {} returned {}".format(
+                strip_auth(res.url), res.status_code
+            )
+            if res.status_code == 404:
+                raise RdapNotFoundError(msg)
+            raise RdapHTTPError(msg)
 
     def asn_url(self, asn):
         """Gets the correct url for specified ASN."""
@@ -243,6 +247,8 @@
         if qstr.startswith("as"):
             return self.get_asn(qstr[2:])
 
+        return self.get_entity(qstr, self.url)
+
         raise NotImplementedError(f"unknown query {query}")
 
     @lru_cache(maxsize=1024)
@@ -315,7 +321,7 @@
         query = f"/ip/{address}"
         return RdapNetwork(self._rdap_get(query).json(), self)
 
-    def get_entity(self, handle, base_url):
+    def get_entity(self, handle, base_url=None):
         """
         get entity information in object form
         """

rdap/context.py

@@ -0,0 +1,130 @@
+"""
+Contact management for RDAP requests.
+"""
+
+from contextvars import ContextVar
+from datetime import datetime
+
+import pydantic
+
+from rdap.exceptions import RdapHTTPError
+
+__all__ = [
+    "rdap_request",
+    "RdapRequestContext",
+    "RdapRequestState",
+]
+
+
+class RdapSource(pydantic.BaseModel):
+    """
+    Describes a source of RDAP data.
+    """
+
+    # urls requested for this source
+    urls: list[str] = pydantic.Field(default_factory=list)
+
+    # rdap object handle
+    handle: str | None = None
+
+    # source creation date (if available)
+    created: datetime | None = None
+
+    # source last update date (if available)
+    updated: datetime | None = None
+
+
+class RdapRequestState(pydantic.BaseModel):
+    """
+    Describe the current rdap request, tracking sources queried
+    and entities retrieved.
+    """
+
+    # list of sources for the current request
+    sources: list[RdapSource] = pydantic.Field(default_factory=list)
+
+    # reference to the rdap client instance
+    client: object | None = None
+
+    # cache of entities (to avoid duplicate requests to the same entity
+    # within the current request context)
+    entities: dict = pydantic.Field(default_factory=dict)
+
+    def update_source(
+        self, handle: str, created: datetime | None, updated: datetime | None
+    ):
+        """
+        Update the current source with the handle and dates.
+        """
+
+        self.sources[-1].handle = handle
+        self.sources[-1].created = created
+        self.sources[-1].updated = updated
+
+
+# context that holds the currently requested rdap url
+
+rdap_request = ContextVar("rdap_request", default=RdapRequestState())
+
+# context manager to set the rdap url
+# can be nested
+
+
+class RdapRequestContext:
+    """
+    Opens a request context
+
+    If no state is present, a new state is created.
+
+    If a state is present, a new source is added to the state.
+    """
+
+    def __init__(self, url: str = None, client: object = None):
+        self.url = url
+        self.token = None
+        self.client = client
+
+    def __enter__(self):
+
+        # get existing state
+
+        state = rdap_request.get()
+
+        if state and self.url:
+            state.sources.append(RdapSource(urls=[self.url]))
+        else:
+            state = RdapRequestState(
+                sources=[RdapSource(urls=[self.url] if self.url else [])]
+            )
+            self.token = rdap_request.set(state)
+
+        if self.client:
+            state.client = self.client
+
+        return self
+
+    def __exit__(self, *exc):
+        if self.token:
+            rdap_request.reset(self.token)
+
+    def push_url(self, url: str):
+        state = rdap_request.get()
+        state.sources[-1].urls.append(url)
+
+    def get(self, typ: str, handle: str):
+        state = rdap_request.get()
+        client = state.client
+
+        if typ not in ["entity", "ip", "domain", "autnum"]:
+            raise ValueError(f"Invalid type: {typ}")
+
+        if state.entities.get(handle):
+            return state.entities[handle]
+        try:
+            get = getattr(client, f"get_{typ}")
+            r_entity = get(handle).normalized
+            state.entities[handle] = r_entity
+            return r_entity
+        except RdapHTTPError:
+            state.entities[handle] = {}
+            return {}