From 72815d6c5985f81123f66bde40415cd25437b076 Mon Sep 17 00:00:00 2001 From: Helmut Hoffer von Ankershoffen Date: Sat, 4 Jan 2025 22:33:08 +0100 Subject: [PATCH] style(web,search,main,atlassian): improve codestyle --- .fixme | 14 ++ pyproject.toml | 40 ++--- src/starbridge/__init__.py | 11 +- src/starbridge/atlassian/__init__.py | 2 + src/starbridge/atlassian/settings.py | 7 +- src/starbridge/cli.py | 42 +++-- src/starbridge/hello/__init__.py | 2 + src/starbridge/hello/cli.py | 16 +- src/starbridge/hello/service.py | 81 +++++++-- src/starbridge/search/__init__.py | 2 + src/starbridge/search/service.py | 25 ++- src/starbridge/service.py | 20 ++- src/starbridge/utils/__init__.py | 2 + src/starbridge/utils/di.py | 22 ++- src/starbridge/web/__init__.py | 6 +- src/starbridge/web/cli.py | 12 +- src/starbridge/web/{types.py => models.py} | 52 +++++- src/starbridge/web/service.py | 59 ++++--- src/starbridge/web/utils.py | 188 ++++++++++++++++----- tests/web/starbridge_web_mcp_test.py | 33 ++-- tests/web/starbridge_web_service_test.py | 4 +- tests/web/starbridge_web_utils_test.py | 6 +- 22 files changed, 477 insertions(+), 169 deletions(-) create mode 100644 .fixme rename src/starbridge/web/{types.py => models.py} (67%) diff --git a/.fixme b/.fixme new file mode 100644 index 0000000..dfe8598 --- /dev/null +++ b/.fixme @@ -0,0 +1,14 @@ +============================= test session starts ============================== +platform darwin -- Python 3.11.6, pytest-8.3.4, pluggy-1.5.0 -- /Users/helmut/Code/starbridge/.venv/bin/python +cachedir: .pytest_cache +rootdir: /Users/helmut/Code/starbridge +configfile: pyproject.toml +testpaths: tests +plugins: datadir-1.5.0, cov-6.0.0, env-1.1.5, logfire-2.11.1, regressions-2.6.0, anyio-4.6.2.post1, xdist-3.6.1, asyncio-0.25.1, docker-3.1.1 +asyncio: mode=Mode.AUTO, asyncio_default_fixture_loop_scope=function +collecting ... collected 0 items + +!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! KeyboardInterrupt !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +/Users/helmut/.local/share/uv/python/cpython-3.11.6-macos-aarch64-none/lib/python3.11/ast.py:273: KeyboardInterrupt +(to show a full traceback on KeyboardInterrupt use --full-trace) +============================ no tests ran in 1.22s ============================= diff --git a/pyproject.toml b/pyproject.toml index 7e37cd7..de088d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "starbridge" version = "0.0.67" -description = "⭐ Integrates Claude Desktop with the web, Google workspace and Atlassian workspaces" +description = "⭐ Integrates Claude Desktop with the web, Google and Atlassian workspaces." readme = "README.md" authors = [ { name = "Helmut Hoffer von Ankershoffen", email = "helmuthva@googlemail.com" }, @@ -143,33 +143,29 @@ target-version = "py311" preview = true fix = true line-length = 120 +exclude = [".fixme"] [tool.ruff.lint] -# select = ["ALL"] -select = [ - # "E", # pycodestyle errors - "F", # pyflakes - "I", # isort - "UP", # pyupgrade - "B", # flake8-bugbear - "C", # flake8-comprehensions -] +select = ["ALL"] # sekect= [ ignore = [ - "ANN002", # Missing type annotation for `*args` - "ANN003", # Missing type annotation for `**kwargs`` - "CPY001", # missing copyright notice - "DOC502", # docstrings with exceptions not raised in the code of the function - "D203", # incomptatible with D211 - "D212", # incompatible with #D213 - "FBT001", # boolean positional arguments - "FBT002", # boolean defautl value positionl arguments - "PGH003", # Use specific rule codes when ignoring type issues - "TRY300", # else instead of return before except. Simply disagree this is more readable. - "COM812", # conflicts with ruff formatter - "ISC001", # conflicts with ruff formatter + "ANN002", # Missing type annotation for `*args` + "ANN003", # Missing type annotation for `**kwargs`` + "ASYNC109", # Async function definition with a `timeout` parameter + "CPY001", # missing copyright notice + "DOC502", # docstrings with exceptions not raised in the code of the function + "D203", # incomptatible with D211 + "D212", # incompatible with #D213 + "FBT001", # boolean positional arguments + "FBT002", # boolean defautl value positionl arguments + "FBT003", # Boolean positional value in function call + "PGH003", # Use specific rule codes when ignoring type issues + "TRY300", # else instead of return before except. Simply disagree this is more readable. + "COM812", # conflicts with ruff formatter + "ISC001", # conflicts with ruff formatter + "S404", # preciew rule: subprocess` module is possibly insecure ] [tool.ruff.lint.per-file-ignores] diff --git a/src/starbridge/__init__.py b/src/starbridge/__init__.py index 947131b..37520fb 100644 --- a/src/starbridge/__init__.py +++ b/src/starbridge/__init__.py @@ -1,3 +1,5 @@ +"""⭐ Integrates Claude Desktop with the web, Google and Atlassian workspaces.""" + import importlib.metadata import os import pathlib @@ -38,13 +40,18 @@ def _amend_library_path() -> None: def _log_boot_message() -> None: # Local import as this initializes logging and instrumentation # which might depend on environment arguments parsed from argv - from starbridge.utils import get_logger, get_process_info + from starbridge.utils import get_logger, get_process_info # noqa: PLC0415 logger = get_logger(__name__) process_info = get_process_info() logger.debug( - f"⭐ Booting Starbridge v{__version__} (project root {process_info.project_root}, pid {process_info.pid}), parent '{process_info.parent.name}' (pid {process_info.parent.pid})", + "⭐ Booting Starbridge v%s (project root %s, pid %s), parent '%s' (pid %s)", + __version__, + process_info.project_root, + process_info.pid, + process_info.parent.name, + process_info.parent.pid, ) diff --git a/src/starbridge/atlassian/__init__.py b/src/starbridge/atlassian/__init__.py index 76bd427..c9c12d1 100644 --- a/src/starbridge/atlassian/__init__.py +++ b/src/starbridge/atlassian/__init__.py @@ -1,3 +1,5 @@ +"""Package containing Atlassian-related functionality and settings.""" + from .settings import Settings __all__ = [ diff --git a/src/starbridge/atlassian/settings.py b/src/starbridge/atlassian/settings.py index 0704539..f3d5648 100644 --- a/src/starbridge/atlassian/settings.py +++ b/src/starbridge/atlassian/settings.py @@ -1,3 +1,5 @@ +"""Module containing Atlassian-related settings and configuration.""" + from typing import Annotated from pydantic import AnyHttpUrl, EmailStr, Field, SecretStr @@ -7,6 +9,8 @@ class Settings(BaseSettings): + """Configuration settings for Atlassian services including Confluence and Jira authentication.""" + model_config = SettingsConfigDict( env_prefix=f"{__project_name__.upper()}_ATLASSIAN_", extra="ignore", @@ -33,7 +37,8 @@ class Settings(BaseSettings): api_token: Annotated[ SecretStr, Field( - description="API token of your Atlassian account. Go to https://id.atlassian.com/manage-profile/security/api-tokens to create a token.", + description="API token of your Atlassian account. " + "Go to https://id.atlassian.com/manage-profile/security/api-tokens to create a token.", examples=["YOUR_TOKEN"], ), ] diff --git a/src/starbridge/cli.py b/src/starbridge/cli.py index e2dbde2..77a470c 100644 --- a/src/starbridge/cli.py +++ b/src/starbridge/cli.py @@ -1,4 +1,7 @@ +"""Command-line interface module for Starbridge, providing various commands for service management and configuration.""" + import sys +from pathlib import Path from typing import Annotated import typer @@ -47,11 +50,16 @@ def main( help="Debug mode", ), ] = True, - env: Annotated[ + env: Annotated[ # noqa: ARG001 list[str] | None, typer.Option( "--env", - help='Environment variables in key=value format. Can be used multiple times in one call. Only STARBRIDGE_ prefixed vars are evaluated. Example --env STARBRIDGE_ATLASSIAN_URL="https://your-domain.atlassian.net" --env STARBRIDGE_ATLASSIAN_EMAIL="YOUR_EMAIL"', + help=( + "Environment variables in key=value format. Can be used multiple times in one call. " + "Only STARBRIDGE_ prefixed vars are evaluated. Example --env " + 'STARBRIDGE_ATLASSIAN_URL="https://your-domain.atlassian.net" --env ' + 'STARBRIDGE_ATLASSIAN_EMAIL="YOUR_EMAIL"' + ), ), ] = None, ) -> None: @@ -66,7 +74,7 @@ def health(json: Annotated[bool, typer.Option(help="Output health as JSON")] = F """Check health of services and their dependencies.""" health = MCPServer().health() if not health.healthy: - logger.warning(f"health: {health}") + logger.warning("health: %s", health) if json: console.print(health.model_dump_json()) else: @@ -83,14 +91,20 @@ def info() -> None: @cli.command() def create_dot_env() -> None: - """Create .env file for Starbridge. You will be prompted for settings.""" + """ + Create .env file for Starbridge. You will be prompted for settings. + + Raises: + RuntimeError: If not running in development mode. + + """ if not __is_development_mode__: msg = "This command is only available in development mode" raise RuntimeError(msg) - - with open(".env", "w", encoding="utf-8") as f: + with Path(".env").open("w", encoding="utf-8") as f: for key, value in iter(prompt_for_env().items()): f.write(f"{key}={value}\n") + f.write(f"{key}={value}\n") @cli.command() @@ -108,7 +122,11 @@ def install( ), ] = "helmuthva/starbridge:latest", ) -> None: - """Install starbridge within Claude Desktop application by adding to configuration and restarting Claude Desktop app.""" + """ + Install starbridge within Claude Desktop application. + + Adds starbridge configuration and restarts Claude Desktop app. + """ if ClaudeService.install_mcp_server( generate_mcp_server_config(prompt_for_env(), image), restart=restart_claude, @@ -131,7 +149,11 @@ def uninstall( ), ] = ClaudeService.platform_supports_restart(), ) -> None: - """Install starbridge from Claude Desktop application by removing from configuration and restarting Claude Desktop app.""" + """ + Uninstall starbridge from Claude Desktop application. + + Removes starbridge configuration and restarts Claude Desktop app. + """ if ClaudeService.uninstall_mcp_server(restart=restart_claude): console.print("Starbridge uninstalled from Claude Destkop application.") else: @@ -143,7 +165,7 @@ def uninstall( if __name__ == "__main__": try: cli() - except Exception as e: - logger.critical(f"Fatal error occurred: {e}") + except Exception as e: # noqa: BLE001 + logger.critical("Fatal error occurred: %s", e) console.print(f"Fatal error occurred: {e}", style="error") sys.exit(1) diff --git a/src/starbridge/hello/__init__.py b/src/starbridge/hello/__init__.py index 672698c..d793c6b 100644 --- a/src/starbridge/hello/__init__.py +++ b/src/starbridge/hello/__init__.py @@ -1,3 +1,5 @@ +"""Hello World module of Starbridge serving as an example.""" + from .cli import cli from .service import Service diff --git a/src/starbridge/hello/cli.py b/src/starbridge/hello/cli.py index 315802e..eedc48c 100644 --- a/src/starbridge/hello/cli.py +++ b/src/starbridge/hello/cli.py @@ -4,6 +4,7 @@ import subprocess import sys import tempfile +import time from pathlib import Path from typing import Annotated @@ -32,7 +33,7 @@ def info() -> None: @cli.command() def hello(locale: Annotated[str, typer.Option(help="Locale to use")] = "en_US") -> None: - """Print Hello World!""" + """Print Hello World.""" console.print(Service().hello(locale)) @@ -43,7 +44,8 @@ def bridge( dump: Annotated[ bool, typer.Option( - help="If set, will dump to file starbridge.png in current working directory. Defaults to opening viewer to show the image.", + help="If set, will dump to file starbridge.png in current working directory. " + "Defaults to opening viewer to show the image.", ), ] = False, ) -> None: @@ -77,7 +79,8 @@ def pdf( dump: Annotated[ bool, typer.Option( - help="If set, will dump to file starbridge.pdf in current working directory. Defaults to opening viewer to show the document.", + help="If set, will dump to file starbridge.pdf in current working directory. " + "Defaults to opening viewer to show the document.", ), ] = False, ) -> None: @@ -94,14 +97,13 @@ def pdf( tmp_path = Path(tmp.name) try: if sys.platform == "darwin": # macOS - subprocess.run(["open", tmp_path], check=True) + subprocess.run(["/usr/bin/open", tmp_path], check=True) # noqa: S603 elif sys.platform == "win32": # Windows - os.startfile(tmp_path) # type: ignore + os.startfile(tmp_path) # type: ignore # noqa: S606 else: # Linux and others - subprocess.run(["xdg-open", tmp_path], check=True) + subprocess.run(["xdg-open", tmp_path], check=True) # noqa: S607, S603 # Give the viewer some time to open the file - import time time.sleep(2) finally: diff --git a/src/starbridge/hello/service.py b/src/starbridge/hello/service.py index a60c569..a2a040e 100644 --- a/src/starbridge/hello/service.py +++ b/src/starbridge/hello/service.py @@ -18,18 +18,43 @@ class Service(MCPBaseService): """Service class for Hello World operations.""" @mcp_tool() - def health(self, context: MCPContext | None = None) -> Health: - """Check health of Hello World service.""" + def health(self, context: MCPContext | None = None) -> Health: # noqa: ARG002, PLR6301 + """ + Check health of Hello World service. + + Returns: + Health: The health status of the service. + + """ return Health(status=Health.Status.UP) @mcp_tool() - def info(self, context: MCPContext | None = None) -> dict: - """Info about Hello world environment.""" + def info(self, context: MCPContext | None = None) -> dict: # noqa: ARG002, PLR6301 + """ + Info about Hello world environment. + + Returns: + dict: Information about the Hello World service environment. + + """ return {"locale": "en_US"} @mcp_tool() - def hello(self, locale: str = "en_US", context: MCPContext | None = None) -> str: - """Print hello world!""" + def hello(self, locale: str = "en_US", context: MCPContext | None = None) -> str: # noqa: ARG002, PLR6301 + """ + Print hello world in different languages. + + Args: + locale: Language/region code to use for the greeting. + context: Optional MCP context. + + Returns: + str: The greeting message in the specified language. + + Raises: + RuntimeError: If the service is configured to fail. + + """ if "starbridge_hello_service_hello_fail" in os.environ.get("MOCKS", "").split( ",", ): @@ -42,10 +67,16 @@ def hello(self, locale: str = "en_US", context: MCPContext | None = None) -> str if find_spec("cairosvg"): @mcp_tool() - def bridge(self, context: MCPContext | None = None): - """Show image of starbridge.""" - import cairosvg - from PIL import Image + def bridge(self, context: MCPContext | None = None): # noqa: ARG002, ANN201, PLR6301 + """ + Show image of starbridge. + + Returns: + PIL.Image.Image: Image object containing the starbridge logo + + """ + import cairosvg # noqa: PLC0415 + from PIL import Image # noqa: PLC0415 return Image.open( io.BytesIO( @@ -54,8 +85,14 @@ def bridge(self, context: MCPContext | None = None): ) @mcp_tool() - def pdf(self, context: MCPContext | None = None) -> EmbeddedResource: - """Show pdf document with Hello World.""" + def pdf(self, context: MCPContext | None = None) -> EmbeddedResource: # noqa: ARG002, PLR6301 + """ + Show pdf document with Hello World. + + Returns: + EmbeddedResource: A PDF document containing Hello World + + """ return EmbeddedResource( type="resource", resource=BlobResourceContents( @@ -66,13 +103,25 @@ def pdf(self, context: MCPContext | None = None) -> EmbeddedResource: ) @staticmethod - def pdf_bytes(context: MCPContext | None = None) -> bytes: - """Show pdf document with Hello World.""" + def pdf_bytes(context: MCPContext | None = None) -> bytes: # noqa: ARG004 + """ + Show pdf document with Hello World. + + Returns: + bytes: PDF document containing Hello World as bytes + + """ return base64.b64decode(Service._starbridge_pdf_base64()) @staticmethod def _starbridge_svg() -> str: - """Image of starbridge, generated with Claude (Sonnet 3.5 new).""" + """ + Image of starbridge, generated with Claude (Sonnet 3.5 new). + + Returns: + str: SVG markup of the starbridge image + + """ return """ @@ -112,4 +161,4 @@ def _starbridge_svg() -> str: @staticmethod def _starbridge_pdf_base64() -> str: - return "JVBERi0xLjMKJcTl8uXrp/Og0MTGCjMgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAyMDIgPj4Kc3RyZWFtCngBXY9Lq8JADIX3/RXHnS7udNLOdBIQF1ZBLwhXGHAhrnzgYqrU/n8w9YF4ySI5cE7ypcUaLayWF28KBjsyLLgdscEFed0R9h3oUd1efS9Do4HqLVIvsh9ryIqwd0hq/JJnnJD/YTxGvqqXM103mWA6qx/nfWkcCwl8ZSoppIBjNqWlgFCSCTZI9g+oJ7ZQoFZX9SN9wiF4w44VvME0gl6GZ4tNlsfY/xNP2GK4OKZ0xWYEHzC83tJhMMIO8RfzqGzzlRLeAQbqP/QKZW5kc3RyZWFtCmVuZG9iagoxIDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL1Jlc291cmNlcyA0IDAgUiAvQ29udGVudHMgMyAwIFIgL01lZGlhQm94IFswIDAgNTk1LjI4IDg0MS44OV0KPj4KZW5kb2JqCjQgMCBvYmoKPDwgL1Byb2NTZXQgWyAvUERGIC9UZXh0IF0gL0NvbG9yU3BhY2UgPDwgL0NzMSA1IDAgUiA+PiAvRm9udCA8PCAvVFQxIDYgMCBSCj4+ID4+CmVuZG9iago4IDAgb2JqCjw8IC9OIDMgL0FsdGVybmF0ZSAvRGV2aWNlUkdCIC9MZW5ndGggMjYxMiAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeAGdlndUU9kWh8+9N73QEiIgJfQaegkg0jtIFQRRiUmAUAKGhCZ2RAVGFBEpVmRUwAFHhyJjRRQLg4Ji1wnyEFDGwVFEReXdjGsJ7601896a/cdZ39nnt9fZZ+9917oAUPyCBMJ0WAGANKFYFO7rwVwSE8vE9wIYEAEOWAHA4WZmBEf4RALU/L09mZmoSMaz9u4ugGS72yy/UCZz1v9/kSI3QyQGAApF1TY8fiYX5QKUU7PFGTL/BMr0lSkyhjEyFqEJoqwi48SvbPan5iu7yZiXJuShGlnOGbw0noy7UN6aJeGjjAShXJgl4GejfAdlvVRJmgDl9yjT0/icTAAwFJlfzOcmoWyJMkUUGe6J8gIACJTEObxyDov5OWieAHimZ+SKBIlJYqYR15hp5ejIZvrxs1P5YjErlMNN4Yh4TM/0tAyOMBeAr2+WRQElWW2ZaJHtrRzt7VnW5mj5v9nfHn5T/T3IevtV8Sbsz55BjJ5Z32zsrC+9FgD2JFqbHbO+lVUAtG0GQOXhrE/vIADyBQC03pzzHoZsXpLE4gwnC4vs7GxzAZ9rLivoN/ufgm/Kv4Y595nL7vtWO6YXP4EjSRUzZUXlpqemS0TMzAwOl89k/fcQ/+PAOWnNycMsnJ/AF/GF6FVR6JQJhIlou4U8gViQLmQKhH/V4X8YNicHGX6daxRodV8AfYU5ULhJB8hvPQBDIwMkbj96An3rWxAxCsi+vGitka9zjzJ6/uf6Hwtcim7hTEEiU+b2DI9kciWiLBmj34RswQISkAd0oAo0gS4wAixgDRyAM3AD3iAAhIBIEAOWAy5IAmlABLJBPtgACkEx2AF2g2pwANSBetAEToI2cAZcBFfADXALDIBHQAqGwUswAd6BaQiC8BAVokGqkBakD5lC1hAbWgh5Q0FQOBQDxUOJkBCSQPnQJqgYKoOqoUNQPfQjdBq6CF2D+qAH0CA0Bv0BfYQRmALTYQ3YALaA2bA7HAhHwsvgRHgVnAcXwNvhSrgWPg63whfhG/AALIVfwpMIQMgIA9FGWAgb8URCkFgkAREha5EipAKpRZqQDqQbuY1IkXHkAwaHoWGYGBbGGeOHWYzhYlZh1mJKMNWYY5hWTBfmNmYQM4H5gqVi1bGmWCesP3YJNhGbjS3EVmCPYFuwl7ED2GHsOxwOx8AZ4hxwfrgYXDJuNa4Etw/XjLuA68MN4SbxeLwq3hTvgg/Bc/BifCG+Cn8cfx7fjx/GvyeQCVoEa4IPIZYgJGwkVBAaCOcI/YQRwjRRgahPdCKGEHnEXGIpsY7YQbxJHCZOkxRJhiQXUiQpmbSBVElqIl0mPSa9IZPJOmRHchhZQF5PriSfIF8lD5I/UJQoJhRPShxFQtlOOUq5QHlAeUOlUg2obtRYqpi6nVpPvUR9Sn0vR5Mzl/OX48mtk6uRa5Xrl3slT5TXl3eXXy6fJ18hf0r+pvy4AlHBQMFTgaOwVqFG4bTCPYVJRZqilWKIYppiiWKD4jXFUSW8koGStxJPqUDpsNIlpSEaQtOledK4tE20Otpl2jAdRzek+9OT6cX0H+i99AllJWVb5SjlHOUa5bPKUgbCMGD4M1IZpYyTjLuMj/M05rnP48/bNq9pXv+8KZX5Km4qfJUilWaVAZWPqkxVb9UU1Z2qbapP1DBqJmphatlq+9Uuq43Pp893ns+dXzT/5PyH6rC6iXq4+mr1w+o96pMamhq+GhkaVRqXNMY1GZpumsma5ZrnNMe0aFoLtQRa5VrntV4wlZnuzFRmJbOLOaGtru2nLdE+pN2rPa1jqLNYZ6NOs84TXZIuWzdBt1y3U3dCT0svWC9fr1HvoT5Rn62fpL9Hv1t/ysDQINpgi0GbwaihiqG/YZ5ho+FjI6qRq9Eqo1qjO8Y4Y7ZxivE+41smsImdSZJJjclNU9jU3lRgus+0zwxr5mgmNKs1u8eisNxZWaxG1qA5wzzIfKN5m/krCz2LWIudFt0WXyztLFMt6ywfWSlZBVhttOqw+sPaxJprXWN9x4Zq42Ozzqbd5rWtqS3fdr/tfTuaXbDdFrtOu8/2DvYi+yb7MQc9h3iHvQ732HR2KLuEfdUR6+jhuM7xjOMHJ3snsdNJp9+dWc4pzg3OowsMF/AX1C0YctFx4bgccpEuZC6MX3hwodRV25XjWuv6zE3Xjed2xG3E3dg92f24+ysPSw+RR4vHlKeT5xrPC16Il69XkVevt5L3Yu9q76c+Oj6JPo0+E752vqt9L/hh/QL9dvrd89fw5/rX+08EOASsCegKpARGBFYHPgsyCRIFdQTDwQHBu4IfL9JfJFzUFgJC/EN2hTwJNQxdFfpzGC4sNKwm7Hm4VXh+eHcELWJFREPEu0iPyNLIR4uNFksWd0bJR8VF1UdNRXtFl0VLl1gsWbPkRoxajCCmPRYfGxV7JHZyqffS3UuH4+ziCuPuLjNclrPs2nK15anLz66QX8FZcSoeGx8d3xD/iRPCqeVMrvRfuXflBNeTu4f7kufGK+eN8V34ZfyRBJeEsoTRRJfEXYljSa5JFUnjAk9BteB1sl/ygeSplJCUoykzqdGpzWmEtPi000IlYYqwK10zPSe9L8M0ozBDuspp1e5VE6JA0ZFMKHNZZruYjv5M9UiMJJslg1kLs2qy3mdHZZ/KUcwR5vTkmuRuyx3J88n7fjVmNXd1Z752/ob8wTXuaw6thdauXNu5Tnddwbrh9b7rj20gbUjZ8MtGy41lG99uit7UUaBRsL5gaLPv5sZCuUJR4b0tzlsObMVsFWzt3WazrWrblyJe0fViy+KK4k8l3JLr31l9V/ndzPaE7b2l9qX7d+B2CHfc3em681iZYlle2dCu4F2t5czyovK3u1fsvlZhW3FgD2mPZI+0MqiyvUqvakfVp+qk6oEaj5rmvep7t+2d2sfb17/fbX/TAY0DxQc+HhQcvH/I91BrrUFtxWHc4azDz+ui6rq/Z39ff0TtSPGRz0eFR6XHwo911TvU1zeoN5Q2wo2SxrHjccdv/eD1Q3sTq+lQM6O5+AQ4ITnx4sf4H++eDDzZeYp9qukn/Z/2ttBailqh1tzWibakNml7THvf6YDTnR3OHS0/m/989Iz2mZqzymdLz5HOFZybOZ93fvJCxoXxi4kXhzpXdD66tOTSna6wrt7LgZevXvG5cqnbvfv8VZerZ645XTt9nX297Yb9jdYeu56WX+x+aem172296XCz/ZbjrY6+BX3n+l37L972un3ljv+dGwOLBvruLr57/17cPel93v3RB6kPXj/Mejj9aP1j7OOiJwpPKp6qP6391fjXZqm99Oyg12DPs4hnj4a4Qy//lfmvT8MFz6nPK0a0RupHrUfPjPmM3Xqx9MXwy4yX0+OFvyn+tveV0auffnf7vWdiycTwa9HrmT9K3qi+OfrW9m3nZOjk03dp76anit6rvj/2gf2h+2P0x5Hp7E/4T5WfjT93fAn88ngmbWbm3/eE8/sKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqClsgL0lDQ0Jhc2VkIDggMCBSIF0KZW5kb2JqCjEwIDAgb2JqCjw8IC9UeXBlIC9TdHJ1Y3RUcmVlUm9vdCAvSyA5IDAgUiA+PgplbmRvYmoKOSAwIG9iago8PCAvVHlwZSAvU3RydWN0RWxlbSAvUyAvRG9jdW1lbnQgL1AgMTAgMCBSIC9LIFsgMTEgMCBSIF0gID4+CmVuZG9iagoxMSAwIG9iago8PCAvVHlwZSAvU3RydWN0RWxlbSAvUyAvUCAvUCA5IDAgUiAvUGcgMSAwIFIgL0sgMSAgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9NZWRpYUJveCBbMCAwIDU5NS4yOCA4NDEuODldIC9Db3VudCAxIC9LaWRzIFsgMSAwIFIgXSA+PgplbmRvYmoKMTIgMCBvYmoKPDwgL1R5cGUgL0NhdGFsb2cgL1BhZ2VzIDIgMCBSIC9NYXJrSW5mbyA8PCAvTWFya2VkIHRydWUgPj4gL1N0cnVjdFRyZWVSb290CjEwIDAgUiA+PgplbmRvYmoKNyAwIG9iagpbIDEgMCBSICAvWFlaIDAgODQxLjg5IDAgXQplbmRvYmoKNiAwIG9iago8PCAvVHlwZSAvRm9udCAvU3VidHlwZSAvVHJ1ZVR5cGUgL0Jhc2VGb250IC9BQUFBQUIrSGVsdmV0aWNhTmV1ZSAvRm9udERlc2NyaXB0b3IKMTMgMCBSIC9FbmNvZGluZyAvTWFjUm9tYW5FbmNvZGluZyAvRmlyc3RDaGFyIDMyIC9MYXN0Q2hhciAxMTQgL1dpZHRocyBbIDI3OAoyNTkgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwCjAgMCAwIDcyMiAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgOTI2IDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDU5MyA1MzcKMCAwIDAgMCAwIDAgMjIyIDAgMCA1NzQgMCAwIDMzMyBdID4+CmVuZG9iagoxMyAwIG9iago8PCAvVHlwZSAvRm9udERlc2NyaXB0b3IgL0ZvbnROYW1lIC9BQUFBQUIrSGVsdmV0aWNhTmV1ZSAvRmxhZ3MgMzIgL0ZvbnRCQm94ClstOTUxIC00ODEgMTk4NyAxMDc3XSAvSXRhbGljQW5nbGUgMCAvQXNjZW50IDk1MiAvRGVzY2VudCAtMjEzIC9DYXBIZWlnaHQKNzE0IC9TdGVtViA5NSAvTGVhZGluZyAyOCAvWEhlaWdodCA1MTcgL1N0ZW1IIDgwIC9BdmdXaWR0aCA0NDcgL01heFdpZHRoIDIyMjUKL0ZvbnRGaWxlMiAxNCAwIFIgPj4KZW5kb2JqCjE0IDAgb2JqCjw8IC9MZW5ndGgxIDMzMjQgL0xlbmd0aCAxNzYyIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4Aa1XfWhbVRQ/974kbdP0I03Sj6Qf7zVZmjZpszZLatc2ptp0Tcfm3IfkdXS17eJaWe3QOuYfzoIfaBBRQWFDRBB1goxMZKYR3EDwWxh+IIwi/qOI+JdM/9G2/u57z9BucwzZS8+755x73rm/87v3naSLDz6cIRstkUSJmfmpY6RdlmoMH84cX5R1m53BWH/fsSPzhn2RyBQ4cvSR+3Tb8guR+b3ZzNRh3aa/McZm4dBttg2jb3Z+8YRumxFPgaMLM8a85RLs2vmpE8b6tAJbfmBqPqPH274V8ccWHlo07Fcw3nbswYwRz9Kwd+lzG+4MupU6iWs+/d5MME0BzSPmIb+frDdNVg38weySwEXn77xXDPTDj532v1NrLSUfmaIwy4w82jNSfj1IjaXnML9U8pHIsumy5skZZAU8wak2yC6A3jj1UpBayIHAxiBdwMzYZleBTPh4gnlicvLRufrhPFlh4Ckip/jbSeNQ7evbqZybycY/JzumQzvzVLYnfY6x59Q8W38yP0xNy0ArTR7qRKqQLCfnhnPsXhg8BEeHAk0KySM5acvI3rRXlbNyNnU4K4/Is1OHc6Yt2oiJTFYNyznal57DfX9aySVUT1HNqOp25DGJPHgE4VkVGe43MmDUXOFVBJlDO+Wc5N+TvjudWxr25BLDqkdR5GTu4p507uKwR1FVRFmKSIFYlK9jLgFmSwfmS/Us+5ADKdRsVuSExf1K7mI268miEs3jVfKMDAcqFTHSlmSeJfakxVTCq3iEw6t4FeBQh5G7LLRzXzoJJIpAYr2GUhreQGl5EShibYBXrlFacYsorbwZSqtuitLqItJNlNqBuVpQWnN9Sr03ILTIcOI6DC/pDC9dh2HHJoadN2bYVcQNkLVA69IYrrtFDNffDMMNN8Wwu4h0E8MeYHYLhhuLDCc8OSoeWjC8dNWRpf88w/+X8qYNlLMrFGG1aF0SjfGvabf0KvrH3eTjDeSjHyjJ4+Rju2gQ/YUZ3cxGFsrBlmkfPHr3hHlLLumGWUz/OWsGpo1XiWaUavcyY2KURuk4vUu/sSz7ndv4O/wyZjhFiLEv+adoxSWUEf0ZLSWM/gkprUaPuQQJLyNQugJv9TJ6sdDKVghNM5lGrwt7hNMaVw2HWTjMZBIOEx7AKnjADA19/8rWbqbYFYddsbNTa1+xSGTtCD+9+hI/tdrHP0HEGOKX0cMl9OruAgYiCZik6oJWpGSv6ROIkFAgMIcLCGBaCFVv7e51RaJS1OtyRMYWSicbOhcWFthjKytrn2lRuwHlCeQupx0F3PTM5Vpms5bCgorLIRxVmzGWXsIagooSCBOLYj0rQAlIVqznsEdQScTuxX33JDsxObn2NP98NcZ2rKEIUZ9AZ8NKL2LdKooK1rQ8oBCjyF2OnGJvGUorkNg1Xas0tK3disMrGR9HBB/+zYXJF/irR17jz069P/MGfx1LnuV7NYnx9OoZ8Ohb/4NXYk0HheiDPHViERfVasBdqKwT0ooqWwWIlSGPRi7eBPJDYpARyD0Q8VvmOOQpyMuQNyHnIR9DKiaGzPQdlJ8gfAJNCVmtyGpFVqHXQ6839A7oHUARpFZyabsoIkKCYRfcVdSmgavCdjTh+AiGm8SObovzSE8zdzkrube1i5vBtbdL8rZWwtWMqVhvxHWy2jcQDMXbauz+gWD7YJuDPX+AV7Z3dTn71f6m5n51IDbu4mv+0dtaldgOv2+kt1WOJS9jh/6sa3GUBlPTvb3TYyF/cEQUonEncTt22k8HC/iJ0aLBcQKaCwECmuDQCWGoijDaMNrCJOrwFetwg1UR7MaDDGmEzhDsEw+siDchLona6ryVTNTWq8S6UCQqa4Y/xt5a+57X+qOKHG2r27+/PBnriAdqGHucu3oPJqPqkI+3xA/G04tsW3M0UFfXFjsb6WkMD7aGZ9N9gdHp/v7Do4E0Vk4C9Gntnarb+EYZZ5HC+jlOHhAnF2FFDtzgwIF9+SJPAexiOyQA6E7U6oR+685NO7I2r+DcrOBAXQbvv0L4BI5lGZQGSDukD5KCqJA5yCOQZyCnIG9DliGfQSomtH1oKO6D1zhPXvHGXnWe2rRDFOfRbTr1JxtC/YrSH2r4dxyXE4cG4xMJWU5MxAcPJWTGw6ket7snFQ6nut3u7lS4bzrV3p6a7uubSXV0pGbAHxPfGWwXd1AltQnO9RYloc4KsFcR1rgvoLxSDWWZ3kuiAlzUbynxRgcPOEOjkeR4C+fm1b9+7tkVaxznA7fjXw3jO2f9FeoRW3XNZYVHQkcJ0nbaga5/F+2lA1oUoxoAEZdF/HQeEtcdwdHM0eOZxbmZqd0Z/NdD/wBAr7VpCmVuZHN0cmVhbQplbmRvYmoKMTUgMCBvYmoKPDwgL1RpdGxlIChIZWxsbyBXb3JsZCkgL1Byb2R1Y2VyIChtYWNPUyBWZXJzaW9uIDE0LjQuMSBcKEJ1aWxkIDIzRTIyNFwpIFF1YXJ0eiBQREZDb250ZXh0KQovQ3JlYXRvciAoUGFnZXMpIC9DcmVhdGlvbkRhdGUgKEQ6MjAyNDEyMjAwODE1NDhaMDAnMDAnKSAvTW9kRGF0ZSAoRDoyMDI0MTIyMDA4MTU0OFowMCcwMCcpCj4+CmVuZG9iagp4cmVmCjAgMTYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMjk2IDAwMDAwIG4gCjAwMDAwMDM0NTEgMDAwMDAgbiAKMDAwMDAwMDAyMiAwMDAwMCBuIAowMDAwMDAwNDA2IDAwMDAwIG4gCjAwMDAwMDMyMTUgMDAwMDAgbiAKMDAwMDAwMzY4NCAwMDAwMCBuIAowMDAwMDAzNjQyIDAwMDAwIG4gCjAwMDAwMDA1MDMgMDAwMDAgbiAKMDAwMDAwMzMwMyAwMDAwMCBuIAowMDAwMDAzMjUwIDAwMDAwIG4gCjAwMDAwMDMzODAgMDAwMDAgbiAKMDAwMDAwMzU0MCAwMDAwMCBuIAowMDAwMDA0MDQzIDAwMDAwIG4gCjAwMDAwMDQzMDkgMDAwMDAgbiAKMDAwMDAwNjE1OSAwMDAwMCBuIAp0cmFpbGVyCjw8IC9TaXplIDE2IC9Sb290IDEyIDAgUiAvSW5mbyAxNSAwIFIgL0lEIFsgPDgxZmRjNzY2OWY1ZmRkZmI1YWIwOTQ5ODdjNDU0ZDU1Pgo8ODFmZGM3NjY5ZjVmZGRmYjVhYjA5NDk4N2M0NTRkNTU+IF0gPj4Kc3RhcnR4cmVmCjYzNjIKJSVFT0YK" + return "JVBERi0xLjMKJcTl8uXrp/Og0MTGCjMgMCBvYmoKPDwgL0ZpbHRlciAvRmxhdGVEZWNvZGUgL0xlbmd0aCAyMDIgPj4Kc3RyZWFtCngBXY9Lq8JADIX3/RXHnS7udNLOdBIQF1ZBLwhXGHAhrnzgYqrU/n8w9YF4ySI5cE7ypcUaLayWF28KBjsyLLgdscEFed0R9h3oUd1efS9Do4HqLVIvsh9ryIqwd0hq/JJnnJD/YTxGvqqXM103mWA6qx/nfWkcCwl8ZSoppIBjNqWlgFCSCTZI9g+oJ7ZQoFZX9SN9wiF4w44VvME0gl6GZ4tNlsfY/xNP2GK4OKZ0xWYEHzC83tJhMMIO8RfzqGzzlRLeAQbqP/QKZW5kc3RyZWFtCmVuZG9iagoxIDAgb2JqCjw8IC9UeXBlIC9QYWdlIC9QYXJlbnQgMiAwIFIgL1Jlc291cmNlcyA0IDAgUiAvQ29udGVudHMgMyAwIFIgL01lZGlhQm94IFswIDAgNTk1LjI4IDg0MS44OV0KPj4KZW5kb2JqCjQgMCBvYmoKPDwgL1Byb2NTZXQgWyAvUERGIC9UZXh0IF0gL0NvbG9yU3BhY2UgPDwgL0NzMSA1IDAgUiA+PiAvRm9udCA8PCAvVFQxIDYgMCBSCj4+ID4+CmVuZG9iago4IDAgb2JqCjw8IC9OIDMgL0FsdGVybmF0ZSAvRGV2aWNlUkdCIC9MZW5ndGggMjYxMiAvRmlsdGVyIC9GbGF0ZURlY29kZSA+PgpzdHJlYW0KeAGdlndUU9kWh8+9N73QEiIgJfQaegkg0jtIFQRRiUmAUAKGhCZ2RAVGFBEpVmRUwAFHhyJjRRQLg4Ji1wnyEFDGwVFEReXdjGsJ7601896a/cdZ39nnt9fZZ+9917oAUPyCBMJ0WAGANKFYFO7rwVwSE8vE9wIYEAEOWAHA4WZmBEf4RALU/L09mZmoSMaz9u4ugGS72yy/UCZz1v9/kSI3QyQGAApF1TY8fiYX5QKUU7PFGTL/BMr0lSkyhjEyFqEJoqwi48SvbPan5iu7yZiXJuShGlnOGbw0noy7UN6aJeGjjAShXJgl4GejfAdlvVRJmgDl9yjT0/icTAAwFJlfzOcmoWyJMkUUGe6J8gIACJTEObxyDov5OWieAHimZ+SKBIlJYqYR15hp5ejIZvrxs1P5YjErlMNN4Yh4TM/0tAyOMBeAr2+WRQElWW2ZaJHtrRzt7VnW5mj5v9nfHn5T/T3IevtV8Sbsz55BjJ5Z32zsrC+9FgD2JFqbHbO+lVUAtG0GQOXhrE/vIADyBQC03pzzHoZsXpLE4gwnC4vs7GxzAZ9rLivoN/ufgm/Kv4Y595nL7vtWO6YXP4EjSRUzZUXlpqemS0TMzAwOl89k/fcQ/+PAOWnNycMsnJ/AF/GF6FVR6JQJhIlou4U8gViQLmQKhH/V4X8YNicHGX6daxRodV8AfYU5ULhJB8hvPQBDIwMkbj96An3rWxAxCsi+vGitka9zjzJ6/uf6Hwtcim7hTEEiU+b2DI9kciWiLBmj34RswQISkAd0oAo0gS4wAixgDRyAM3AD3iAAhIBIEAOWAy5IAmlABLJBPtgACkEx2AF2g2pwANSBetAEToI2cAZcBFfADXALDIBHQAqGwUswAd6BaQiC8BAVokGqkBakD5lC1hAbWgh5Q0FQOBQDxUOJkBCSQPnQJqgYKoOqoUNQPfQjdBq6CF2D+qAH0CA0Bv0BfYQRmALTYQ3YALaA2bA7HAhHwsvgRHgVnAcXwNvhSrgWPg63whfhG/AALIVfwpMIQMgIA9FGWAgb8URCkFgkAREha5EipAKpRZqQDqQbuY1IkXHkAwaHoWGYGBbGGeOHWYzhYlZh1mJKMNWYY5hWTBfmNmYQM4H5gqVi1bGmWCesP3YJNhGbjS3EVmCPYFuwl7ED2GHsOxwOx8AZ4hxwfrgYXDJuNa4Etw/XjLuA68MN4SbxeLwq3hTvgg/Bc/BifCG+Cn8cfx7fjx/GvyeQCVoEa4IPIZYgJGwkVBAaCOcI/YQRwjRRgahPdCKGEHnEXGIpsY7YQbxJHCZOkxRJhiQXUiQpmbSBVElqIl0mPSa9IZPJOmRHchhZQF5PriSfIF8lD5I/UJQoJhRPShxFQtlOOUq5QHlAeUOlUg2obtRYqpi6nVpPvUR9Sn0vR5Mzl/OX48mtk6uRa5Xrl3slT5TXl3eXXy6fJ18hf0r+pvy4AlHBQMFTgaOwVqFG4bTCPYVJRZqilWKIYppiiWKD4jXFUSW8koGStxJPqUDpsNIlpSEaQtOledK4tE20Otpl2jAdRzek+9OT6cX0H+i99AllJWVb5SjlHOUa5bPKUgbCMGD4M1IZpYyTjLuMj/M05rnP48/bNq9pXv+8KZX5Km4qfJUilWaVAZWPqkxVb9UU1Z2qbapP1DBqJmphatlq+9Uuq43Pp893ns+dXzT/5PyH6rC6iXq4+mr1w+o96pMamhq+GhkaVRqXNMY1GZpumsma5ZrnNMe0aFoLtQRa5VrntV4wlZnuzFRmJbOLOaGtru2nLdE+pN2rPa1jqLNYZ6NOs84TXZIuWzdBt1y3U3dCT0svWC9fr1HvoT5Rn62fpL9Hv1t/ysDQINpgi0GbwaihiqG/YZ5ho+FjI6qRq9Eqo1qjO8Y4Y7ZxivE+41smsImdSZJJjclNU9jU3lRgus+0zwxr5mgmNKs1u8eisNxZWaxG1qA5wzzIfKN5m/krCz2LWIudFt0WXyztLFMt6ywfWSlZBVhttOqw+sPaxJprXWN9x4Zq42Ozzqbd5rWtqS3fdr/tfTuaXbDdFrtOu8/2DvYi+yb7MQc9h3iHvQ732HR2KLuEfdUR6+jhuM7xjOMHJ3snsdNJp9+dWc4pzg3OowsMF/AX1C0YctFx4bgccpEuZC6MX3hwodRV25XjWuv6zE3Xjed2xG3E3dg92f24+ysPSw+RR4vHlKeT5xrPC16Il69XkVevt5L3Yu9q76c+Oj6JPo0+E752vqt9L/hh/QL9dvrd89fw5/rX+08EOASsCegKpARGBFYHPgsyCRIFdQTDwQHBu4IfL9JfJFzUFgJC/EN2hTwJNQxdFfpzGC4sNKwm7Hm4VXh+eHcELWJFREPEu0iPyNLIR4uNFksWd0bJR8VF1UdNRXtFl0VLl1gsWbPkRoxajCCmPRYfGxV7JHZyqffS3UuH4+ziCuPuLjNclrPs2nK15anLz66QX8FZcSoeGx8d3xD/iRPCqeVMrvRfuXflBNeTu4f7kufGK+eN8V34ZfyRBJeEsoTRRJfEXYljSa5JFUnjAk9BteB1sl/ygeSplJCUoykzqdGpzWmEtPi000IlYYqwK10zPSe9L8M0ozBDuspp1e5VE6JA0ZFMKHNZZruYjv5M9UiMJJslg1kLs2qy3mdHZZ/KUcwR5vTkmuRuyx3J88n7fjVmNXd1Z752/ob8wTXuaw6thdauXNu5Tnddwbrh9b7rj20gbUjZ8MtGy41lG99uit7UUaBRsL5gaLPv5sZCuUJR4b0tzlsObMVsFWzt3WazrWrblyJe0fViy+KK4k8l3JLr31l9V/ndzPaE7b2l9qX7d+B2CHfc3em681iZYlle2dCu4F2t5czyovK3u1fsvlZhW3FgD2mPZI+0MqiyvUqvakfVp+qk6oEaj5rmvep7t+2d2sfb17/fbX/TAY0DxQc+HhQcvH/I91BrrUFtxWHc4azDz+ui6rq/Z39ff0TtSPGRz0eFR6XHwo911TvU1zeoN5Q2wo2SxrHjccdv/eD1Q3sTq+lQM6O5+AQ4ITnx4sf4H++eDDzZeYp9qukn/Z/2ttBailqh1tzWibakNml7THvf6YDTnR3OHS0/m/989Iz2mZqzymdLz5HOFZybOZ93fvJCxoXxi4kXhzpXdD66tOTSna6wrt7LgZevXvG5cqnbvfv8VZerZ645XTt9nX297Yb9jdYeu56WX+x+aem172296XCz/ZbjrY6+BX3n+l37L972un3ljv+dGwOLBvruLr57/17cPel93v3RB6kPXj/Mejj9aP1j7OOiJwpPKp6qP6391fjXZqm99Oyg12DPs4hnj4a4Qy//lfmvT8MFz6nPK0a0RupHrUfPjPmM3Xqx9MXwy4yX0+OFvyn+tveV0auffnf7vWdiycTwa9HrmT9K3qi+OfrW9m3nZOjk03dp76anit6rvj/2gf2h+2P0x5Hp7E/4T5WfjT93fAn88ngmbWbm3/eE8/sKZW5kc3RyZWFtCmVuZG9iago1IDAgb2JqClsgL0lDQ0Jhc2VkIDggMCBSIF0KZW5kb2JqCjEwIDAgb2JqCjw8IC9UeXBlIC9TdHJ1Y3RUcmVlUm9vdCAvSyA5IDAgUiA+PgplbmRvYmoKOSAwIG9iago8PCAvVHlwZSAvU3RydWN0RWxlbSAvUyAvRG9jdW1lbnQgL1AgMTAgMCBSIC9LIFsgMTEgMCBSIF0gID4+CmVuZG9iagoxMSAwIG9iago8PCAvVHlwZSAvU3RydWN0RWxlbSAvUyAvUCAvUCA5IDAgUiAvUGcgMSAwIFIgL0sgMSAgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9NZWRpYUJveCBbMCAwIDU5NS4yOCA4NDEuODldIC9Db3VudCAxIC9LaWRzIFsgMSAwIFIgXSA+PgplbmRvYmoKMTIgMCBvYmoKPDwgL1R5cGUgL0NhdGFsb2cgL1BhZ2VzIDIgMCBSIC9NYXJrSW5mbyA8PCAvTWFya2VkIHRydWUgPj4gL1N0cnVjdFRyZWVSb290CjEwIDAgUiA+PgplbmRvYmoKNyAwIG9iagpbIDEgMCBSICAvWFlaIDAgODQxLjg5IDAgXQplbmRvYmoKNiAwIG9iago8PCAvVHlwZSAvRm9udCAvU3VidHlwZSAvVHJ1ZVR5cGUgL0Jhc2VGb250IC9BQUFBQUIrSGVsdmV0aWNhTmV1ZSAvRm9udERlc2NyaXB0b3IKMTMgMCBSIC9FbmNvZGluZyAvTWFjUm9tYW5FbmNvZGluZyAvRmlyc3RDaGFyIDMyIC9MYXN0Q2hhciAxMTQgL1dpZHRocyBbIDI3OAoyNTkgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwCjAgMCAwIDcyMiAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDAgOTI2IDAgMCAwIDAgMCAwIDAgMCAwIDAgMCAwIDU5MyA1MzcKMCAwIDAgMCAwIDAgMjIyIDAgMCA1NzQgMCAwIDMzMyBdID4+CmVuZG9iagoxMyAwIG9iago8PCAvVHlwZSAvRm9udERlc2NyaXB0b3IgL0ZvbnROYW1lIC9BQUFBQUIrSGVsdmV0aWNhTmV1ZSAvRmxhZ3MgMzIgL0ZvbnRCQm94ClstOTUxIC00ODEgMTk4NyAxMDc3XSAvSXRhbGljQW5nbGUgMCAvQXNjZW50IDk1MiAvRGVzY2VudCAtMjEzIC9DYXBIZWlnaHQKNzE0IC9TdGVtViA5NSAvTGVhZGluZyAyOCAvWEhlaWdodCA1MTcgL1N0ZW1IIDgwIC9BdmdXaWR0aCA0NDcgL01heFdpZHRoIDIyMjUKL0ZvbnRGaWxlMiAxNCAwIFIgPj4KZW5kb2JqCjE0IDAgb2JqCjw8IC9MZW5ndGgxIDMzMjQgL0xlbmd0aCAxNzYyIC9GaWx0ZXIgL0ZsYXRlRGVjb2RlID4+CnN0cmVhbQp4Aa1XfWhbVRQ/974kbdP0I03Sj6Qf7zVZmjZpszZLatc2ptp0Tcfm3IfkdXS17eJaWe3QOuYfzoIfaBBRQWFDRBB1goxMZKYR3EDwWxh+IIwi/qOI+JdM/9G2/u57z9BucwzZS8+755x73rm/87v3naSLDz6cIRstkUSJmfmpY6RdlmoMH84cX5R1m53BWH/fsSPzhn2RyBQ4cvSR+3Tb8guR+b3ZzNRh3aa/McZm4dBttg2jb3Z+8YRumxFPgaMLM8a85RLs2vmpE8b6tAJbfmBqPqPH274V8ccWHlo07Fcw3nbswYwRz9Kwd+lzG+4MupU6iWs+/d5MME0BzSPmIb+frDdNVg38weySwEXn77xXDPTDj532v1NrLSUfmaIwy4w82jNSfj1IjaXnML9U8pHIsumy5skZZAU8wak2yC6A3jj1UpBayIHAxiBdwMzYZleBTPh4gnlicvLRufrhPFlh4Ckip/jbSeNQ7evbqZybycY/JzumQzvzVLYnfY6x59Q8W38yP0xNy0ArTR7qRKqQLCfnhnPsXhg8BEeHAk0KySM5acvI3rRXlbNyNnU4K4/Is1OHc6Yt2oiJTFYNyznal57DfX9aySVUT1HNqOp25DGJPHgE4VkVGe43MmDUXOFVBJlDO+Wc5N+TvjudWxr25BLDqkdR5GTu4p507uKwR1FVRFmKSIFYlK9jLgFmSwfmS/Us+5ADKdRsVuSExf1K7mI268miEs3jVfKMDAcqFTHSlmSeJfakxVTCq3iEw6t4FeBQh5G7LLRzXzoJJIpAYr2GUhreQGl5EShibYBXrlFacYsorbwZSqtuitLqItJNlNqBuVpQWnN9Sr03ILTIcOI6DC/pDC9dh2HHJoadN2bYVcQNkLVA69IYrrtFDNffDMMNN8Wwu4h0E8MeYHYLhhuLDCc8OSoeWjC8dNWRpf88w/+X8qYNlLMrFGG1aF0SjfGvabf0KvrH3eTjDeSjHyjJ4+Rju2gQ/YUZ3cxGFsrBlmkfPHr3hHlLLumGWUz/OWsGpo1XiWaUavcyY2KURuk4vUu/sSz7ndv4O/wyZjhFiLEv+adoxSWUEf0ZLSWM/gkprUaPuQQJLyNQugJv9TJ6sdDKVghNM5lGrwt7hNMaVw2HWTjMZBIOEx7AKnjADA19/8rWbqbYFYddsbNTa1+xSGTtCD+9+hI/tdrHP0HEGOKX0cMl9OruAgYiCZik6oJWpGSv6ROIkFAgMIcLCGBaCFVv7e51RaJS1OtyRMYWSicbOhcWFthjKytrn2lRuwHlCeQupx0F3PTM5Vpms5bCgorLIRxVmzGWXsIagooSCBOLYj0rQAlIVqznsEdQScTuxX33JDsxObn2NP98NcZ2rKEIUZ9AZ8NKL2LdKooK1rQ8oBCjyF2OnGJvGUorkNg1Xas0tK3disMrGR9HBB/+zYXJF/irR17jz069P/MGfx1LnuV7NYnx9OoZ8Ohb/4NXYk0HheiDPHViERfVasBdqKwT0ooqWwWIlSGPRi7eBPJDYpARyD0Q8VvmOOQpyMuQNyHnIR9DKiaGzPQdlJ8gfAJNCVmtyGpFVqHXQ6839A7oHUARpFZyabsoIkKCYRfcVdSmgavCdjTh+AiGm8SObovzSE8zdzkrube1i5vBtbdL8rZWwtWMqVhvxHWy2jcQDMXbauz+gWD7YJuDPX+AV7Z3dTn71f6m5n51IDbu4mv+0dtaldgOv2+kt1WOJS9jh/6sa3GUBlPTvb3TYyF/cEQUonEncTt22k8HC/iJ0aLBcQKaCwECmuDQCWGoijDaMNrCJOrwFetwg1UR7MaDDGmEzhDsEw+siDchLona6ryVTNTWq8S6UCQqa4Y/xt5a+57X+qOKHG2r27+/PBnriAdqGHucu3oPJqPqkI+3xA/G04tsW3M0UFfXFjsb6WkMD7aGZ9N9gdHp/v7Do4E0Vk4C9Gntnarb+EYZZ5HC+jlOHhAnF2FFDtzgwIF9+SJPAexiOyQA6E7U6oR+685NO7I2r+DcrOBAXQbvv0L4BI5lGZQGSDukD5KCqJA5yCOQZyCnIG9DliGfQSomtH1oKO6D1zhPXvHGXnWe2rRDFOfRbTr1JxtC/YrSH2r4dxyXE4cG4xMJWU5MxAcPJWTGw6ket7snFQ6nut3u7lS4bzrV3p6a7uubSXV0pGbAHxPfGWwXd1AltQnO9RYloc4KsFcR1rgvoLxSDWWZ3kuiAlzUbynxRgcPOEOjkeR4C+fm1b9+7tkVaxznA7fjXw3jO2f9FeoRW3XNZYVHQkcJ0nbaga5/F+2lA1oUoxoAEZdF/HQeEtcdwdHM0eOZxbmZqd0Z/NdD/wBAr7VpCmVuZHN0cmVhbQplbmRvYmoKMTUgMCBvYmoKPDwgL1RpdGxlIChIZWxsbyBXb3JsZCkgL1Byb2R1Y2VyIChtYWNPUyBWZXJzaW9uIDE0LjQuMSBcKEJ1aWxkIDIzRTIyNFwpIFF1YXJ0eiBQREZDb250ZXh0KQovQ3JlYXRvciAoUGFnZXMpIC9DcmVhdGlvbkRhdGUgKEQ6MjAyNDEyMjAwODE1NDhaMDAnMDAnKSAvTW9kRGF0ZSAoRDoyMDI0MTIyMDA4MTU0OFowMCcwMCcpCj4+CmVuZG9iagp4cmVmCjAgMTYKMDAwMDAwMDAwMCA2NTUzNSBmIAowMDAwMDAwMjk2IDAwMDAwIG4gCjAwMDAwMDM0NTEgMDAwMDAgbiAKMDAwMDAwMDAyMiAwMDAwMCBuIAowMDAwMDAwNDA2IDAwMDAwIG4gCjAwMDAwMDMyMTUgMDAwMDAgbiAKMDAwMDAwMzY4NCAwMDAwMCBuIAowMDAwMDAzNjQyIDAwMDAwIG4gCjAwMDAwMDA1MDMgMDAwMDAgbiAKMDAwMDAwMzMwMyAwMDAwMCBuIAowMDAwMDAzMjUwIDAwMDAwIG4gCjAwMDAwMDMzODAgMDAwMDAgbiAKMDAwMDAwMzU0MCAwMDAwMCBuIAowMDAwMDA0MDQzIDAwMDAwIG4gCjAwMDAwMDQzMDkgMDAwMDAgbiAKMDAwMDAwNjE1OSAwMDAwMCBuIAp0cmFpbGVyCjw8IC9TaXplIDE2IC9Sb290IDEyIDAgUiAvSW5mbyAxNSAwIFIgL0lEIFsgPDgxZmRjNzY2OWY1ZmRkZmI1YWIwOTQ5ODdjNDU0ZDU1Pgo8ODFmZGM3NjY5ZjVmZGRmYjVhYjA5NDk4N2M0NTRkNTU+IF0gPj4Kc3RhcnR4cmVmCjYzNjIKJSVFT0YK" # noqa: E501 diff --git a/src/starbridge/search/__init__.py b/src/starbridge/search/__init__.py index 384cd39..8df90b6 100644 --- a/src/starbridge/search/__init__.py +++ b/src/starbridge/search/__init__.py @@ -1,3 +1,5 @@ +"""Search package for Starbridge providing search functionality and related services.""" + from .cli import cli from .service import Service from .settings import Settings diff --git a/src/starbridge/search/service.py b/src/starbridge/search/service.py index d7dbef3..23c4e94 100644 --- a/src/starbridge/search/service.py +++ b/src/starbridge/search/service.py @@ -23,12 +23,19 @@ class Service(MCPBaseService): _bs: BraveSearch def __init__(self) -> None: + """Initialize the search service with Brave Search client.""" super().__init__(Settings) self._bs = BraveSearch(api_key=self._settings.brave_search_api_key) @mcp_tool() - def health(self, context: MCPContext | None = None) -> Health: - """Check health of the search service.""" + def health(self, context: MCPContext | None = None) -> Health: # noqa: ARG002 + """ + Check health of the search service. + + Returns: + Health: The health status of the search service. + + """ if not asyncio.run(self._bs.is_connected()): return Health( status=Health.Status.DOWN, @@ -37,15 +44,21 @@ def health(self, context: MCPContext | None = None) -> Health: return Health(status=Health.Status.UP) @mcp_tool() - def info(self, context: MCPContext | None = None) -> dict: - """Info about search environment.""" + def info(self, context: MCPContext | None = None) -> dict: # noqa: ARG002, PLR6301 + """ + Info about search environment. + + Returns: + dict: Information about the search environment configuration. + + """ return {} @mcp_tool() - async def web( + async def web( # noqa: D417 self, q: str, - context: MCPContext | None = None, + context: MCPContext | None = None, # noqa: ARG002 ) -> WebSearchApiResponse: """ Search the world wide web via Brave Search. diff --git a/src/starbridge/service.py b/src/starbridge/service.py index 6baa8bb..15fdba1 100644 --- a/src/starbridge/service.py +++ b/src/starbridge/service.py @@ -22,13 +22,25 @@ class Service(MCPBaseService): """Service class for core services if starbridge.""" @mcp_tool() - def health(self, context: MCPContext | None = None) -> Health: - """Check the health of the core service of Starbridge.""" + def health(self, context: MCPContext | None = None) -> Health: # noqa: ARG002, PLR6301 + """ + Check the health of the core service of Starbridge. + + Returns: + Health: The health status of the core service + + """ return Health(status=Health.Status.UP) # We are up always @mcp_tool() - def info(self, context: MCPContext | None = None) -> dict: - """Get info about the environment starbridge is running in and all services.""" + def info(self, context: MCPContext | None = None) -> dict: # noqa: ARG002, PLR6301 + """ + Get info about the environment starbridge is running in and all services. + + Returns: + dict: Information about the Starbridge environment and its services + + """ rtn = { "name": __project_name__, "version": __version__, diff --git a/src/starbridge/utils/__init__.py b/src/starbridge/utils/__init__.py index 647cc1a..5531213 100644 --- a/src/starbridge/utils/__init__.py +++ b/src/starbridge/utils/__init__.py @@ -1,3 +1,5 @@ +"""Utility functions and classes for the starbridge package.""" + from .cli import prepare_cli from .console import console from .di import locate_implementations, locate_subclasses diff --git a/src/starbridge/utils/di.py b/src/starbridge/utils/di.py index 8d7af9a..20dec77 100644 --- a/src/starbridge/utils/di.py +++ b/src/starbridge/utils/di.py @@ -1,3 +1,5 @@ +"""Module for dynamic import and discovery of implementations and subclasses.""" + import importlib import pkgutil from inspect import isclass @@ -9,8 +11,14 @@ _subclass_cache = {} -def locate_implementations(_class: Any) -> list[Any]: - """Dynamically discover all Service classes in starbridge packages.""" +def locate_implementations(_class: type[Any]) -> list[Any]: + """ + Dynamically discover all Service classes in starbridge packages. + + Returns: + list[Any]: List of discovered implementations of the given class. + + """ if _class in _implementation_cache: return _implementation_cache[_class] @@ -32,8 +40,14 @@ def locate_implementations(_class: Any) -> list[Any]: return implementations -def locate_subclasses(_class: Any) -> list[Any]: - """Dynamically discover all Service classes in starbridge packages.""" +def locate_subclasses(_class: type[Any]) -> list[type[Any]]: + """ + Dynamically discover all Service classes in starbridge packages. + + Returns: + list[type[Any]]: List of discovered subclasses of the given class. + + """ if _class in _subclass_cache: return _subclass_cache[_class] diff --git a/src/starbridge/web/__init__.py b/src/starbridge/web/__init__.py index c2adb30..95d53d2 100644 --- a/src/starbridge/web/__init__.py +++ b/src/starbridge/web/__init__.py @@ -1,14 +1,16 @@ +"""Web module for interacting with the world wide web.""" + from .cli import cli +from .models import Context, GetResult, LinkTarget, Resource, RobotForbiddenError from .service import Service from .settings import Settings -from .types import Context, GetResult, LinkTarget, Resource, RobotForbiddenException __all__ = [ "Context", "GetResult", "LinkTarget", "Resource", - "RobotForbiddenException", + "RobotForbiddenError", "Service", "Settings", "cli", diff --git a/src/starbridge/web/cli.py b/src/starbridge/web/cli.py index 8694eae..064076b 100644 --- a/src/starbridge/web/cli.py +++ b/src/starbridge/web/cli.py @@ -11,8 +11,8 @@ from starbridge.utils.console import console +from .models import RobotForbiddenError from .service import Service -from .types import RobotForbiddenException cli = typer.Typer(name="web", help="Web operations") @@ -30,7 +30,7 @@ def info() -> None: @cli.command() -def get( +def get( # noqa: PLR0913, PLR0917 url: Annotated[str, typer.Argument(help="URL to fetch")], accept_language: Annotated[ str, @@ -63,7 +63,11 @@ def get( ), ] = False, ) -> None: - """Fetch content from the world wide web via HTTP GET, convert to content type as a best effort, extract links, and provide additional context.""" + """ + Fetch content from the world wide web via HTTP GET. + + Converts to content type as a best effort, extracts links, and provides additional context. + """ try: rtn = asyncio.run( Service().get( @@ -87,7 +91,7 @@ def get( ), ) sys.exit(1) - except RobotForbiddenException as e: + except RobotForbiddenError as e: text = Text() text.append(str(e)) console.print( diff --git a/src/starbridge/web/types.py b/src/starbridge/web/models.py similarity index 67% rename from src/starbridge/web/types.py rename to src/starbridge/web/models.py index f19b7ff..34dea6d 100644 --- a/src/starbridge/web/types.py +++ b/src/starbridge/web/models.py @@ -1,13 +1,17 @@ +"""Data type definitions for web interactions.""" + from typing import Annotated from pydantic import AnyHttpUrl, AnyUrl, BaseModel, Field, model_validator -class RobotForbiddenException(Exception): - pass +class RobotForbiddenError(Exception): + """Exception raised when access to a URL is forbidden by robots.txt.""" class MimeType: + """Constants for commonly used MIME types in web interactions.""" + TEXT_HTML = "text/html" TEXT_MARKDWON = "text/markdown" TEXT_PLAIN = "text/plain" @@ -21,6 +25,8 @@ class MimeType: class Resource(BaseModel): + """A model representing a web resource with its URL, type, and content (text or binary).""" + url: Annotated[AnyHttpUrl, Field(description="Final URL of the resource")] type: Annotated[str, Field(description="MIME type of the resource", min_length=4)] text: Annotated[ @@ -38,6 +44,16 @@ class Resource(BaseModel): @model_validator(mode="after") def check_content_exists(self) -> "Resource": + """ + Validate that either text or blob content exists, but not both. + + Returns: + Resource: The validated resource instance + + Raises: + ValueError: If neither text nor blob is provided, or if both are provided + + """ if self.text is None and self.blob is None: msg = "Either text or blob must be provided" raise ValueError(msg) @@ -48,6 +64,8 @@ def check_content_exists(self) -> "Resource": class LinkTarget(BaseModel): + """A model representing a link target in a web resource with its URL, occurrences and anchor texts.""" + url: Annotated[AnyUrl, Field(description="URL of the link target")] occurrences: Annotated[ int, @@ -64,14 +82,31 @@ class LinkTarget(BaseModel): class Context(BaseModel): + """A model representing additional context information about a web resource or its domain.""" + type: Annotated[str, Field(description="Type of context")] url: Annotated[AnyHttpUrl, Field(description="URL of the context")] text: Annotated[str, Field(description="Content of context in markdown format")] class GetResult(BaseModel): + """ + A model representing the result of a web resource fetch operation. + + Includes the resource, extracted links, and additional context. + """ + def get_context_by_type(self, context_type: str) -> Context | None: - """Get text of additional context of given type.""" + """ + Get text of additional context of given type. + + Args: + context_type: The type of context to retrieve + + Returns: + Context | None: The context of the specified type, or None if not found + + """ if not self.additional_context: return None for ctx in self.additional_context: @@ -80,7 +115,13 @@ def get_context_by_type(self, context_type: str) -> Context | None: return None def get_link_count(self) -> int: - """Get number of extracted links.""" + """ + Get number of extracted links. + + Returns: + int: The number of extracted links + + """ return len(self.extracted_links or []) resource: Annotated[ @@ -91,7 +132,8 @@ def get_link_count(self) -> int: list[LinkTarget] | None, Field( default=None, - description="List of link targets extracted from the resource, if extract_links=True. Sorted by number of occurrences of a URL in the resource", + description="List of link targets extracted from the resource, if extract_links=True. " + "Sorted by number of occurrences of a URL in the resource", ), ] = None additional_context: Annotated[ diff --git a/src/starbridge/web/service.py b/src/starbridge/web/service.py index 084c202..e527ca6 100644 --- a/src/starbridge/web/service.py +++ b/src/starbridge/web/service.py @@ -3,8 +3,8 @@ from starbridge.mcp import MCPBaseService, MCPContext, mcp_tool from starbridge.utils import Health, get_logger +from .models import GetResult from .settings import Settings -from .types import GetResult from .utils import ( extract_links_from_response, get_additional_context_for_url, @@ -22,11 +22,18 @@ class Service(MCPBaseService): _settings: Settings def __init__(self) -> None: + """Initialize the web service with default settings.""" super().__init__(Settings) @mcp_tool() - def health(self, context: MCPContext | None = None) -> Health: - """Check health of the web service.""" + def health(self, context: MCPContext | None = None) -> Health: # noqa: PLR6301, ARG002 + """ + Check health of the web service. + + Returns: + Health: The health status of the web service + + """ if not is_connected(): return Health( status=Health.Status.DOWN, @@ -35,7 +42,7 @@ def health(self, context: MCPContext | None = None) -> Health: return Health(status=Health.Status.UP) @mcp_tool() - def info(self, context: MCPContext | None = None) -> dict: + def info(self, context: MCPContext | None = None) -> dict: # noqa: PLR6301, ARG002 """ Info about web environment. @@ -46,7 +53,7 @@ def info(self, context: MCPContext | None = None) -> dict: return {} @mcp_tool() - async def get( + async def get( # noqa: PLR0913, PLR0917 self, url: str, accept_language: str = "en-US,en;q=0.9,de;q=0.8", @@ -54,49 +61,59 @@ async def get( extract_links: bool = True, additional_context: bool = True, llms_full_txt: bool = False, - context: MCPContext | None = None, + context: MCPContext | None = None, # noqa: ARG002 ) -> GetResult: """ Fetch page from the world wide web via HTTP GET. - Should be called by the assistant when the user asks to fetch/retrieve/load/download a page/content/document from the Internet / the world wide web + Should be called by the assistant when the user asks to fetch/retrieve/load/download + a page/content/document from + the Internet / the world wide web - This includes the case when the user simply pastes a URL without further context - - This includes asks about current news, or e.g. if the user simply prompts the assitant with "What's today on ". + - This includes asks about current news, or e.g. if the user simply prompts the assitant with + "What's today on ". - This includes asks to download a pdf Further tips: - The agent is to disable transform to markdown, extract links, and additional context in error cases only. - - The agent can use this tool to crawl multiple pages. I.e. when asked to crawl a URL use a get call, than look at the top links extracted, follow them, and in the end provide a summary. - + - The agent can use this tool to crawl multiple pages. I.e. when asked to crawl a URL use a get call, than + look at the top links extracted, follow them, and in the end provide a summary. Args: url (str): The URL to fetch content from - accept_language (str, optional): Accept-Language header to send as part of the get request. Defaults to en-US,en;q=0.9,de;q=0.8. + accept_language (str, optional): Accept-Language header to send as part of the get request. + Defaults to en-US,en;q=0.9,de;q=0.8. The assistant can prompt the user for the language preferred, and set this header accordingly. - transform_to_markdown (bool, optional): If set will transform content to markdown if possible. Defaults to true. - If the transformation is not supported, the content will be returned as is + transform_to_markdown (bool, optional): If set will transform content to markdown if possible. + Defaults to true. If the transformation is not supported, the content will be returned as is extract_links (bool, optional): If set will extract links from the content. Defaults to True. Supported for selected content types only - additional_context (bool, optional): If set will include additional context about the URL or it's domain in the response. Defaults to True. + additional_context (bool, optional): If set will include additional context about the URL + or it's domain in the response. Defaults to True. llms_full_txt (bool, optional): Whether to include llms-full.txt in additional context. Defaults to False. context (MCPContext | None, optional): Context object for request tracking. Defaults to None. Returns: - 'resource': The retrieved and possibly transformed resource: + resource: The retrieved and possibly transformed resource: - 'url' (string) the final URL after redirects - - 'type' (content type indicator as defined in http): the type of transformed content, resp. the original content type if no transformation applied - - 'text' (string): the transformed textual content, resp. the original content if no transformation applied + - 'type' (content type indicator as defined in http): the type of transformed content, + resp. the original + content type if no transformation applied + - 'text' (string): the transformed textual content, resp. the original content if no transformation + applied - 'blob' (bytes): the binary content of the resource, if the resource has binary content - 'extracted_links': Optional list of links extracted from the resource, if extract_links=True. Sorted by number of occurrences of a URL in the resource. Each item has: + extracted_links: Optional list of links extracted from the resource, if extract_links=True. + Sorted by number of occurrences of a URL in the resource. Each item has: - 'url' (string) the URL of the link - 'occurrences' (int) the number of occurrences of the link in the resource - 'anchor_texts' (list of strings) the anchor texts of the link - 'additional_context': Optional list of with extra context (only if additional_context=True). Each item has: + additional_context: Optional list of with extra context (only if additional_context=True). Each item has: - 'url' (string) the URL of the context - - 'type' (string) the type of context, e.g. llms_txt for text specifally prepared by a domain for an assistant to read + - 'type' (string) the type of context, e.g. llms_txt for text specifally prepared by a domain for an + assistant to read - 'text' (string) the content of the context in markdown format Raises: - starbridge.web.RobotForbiddenException: If we are not allowed to crawl the URL autonomously + starbridge.web.RobotForbiddenError: If we are not allowed to crawl the URL autonomously requests.exceptions.RequestException: If the HTTP get request failed ValueError: If an invalid format was passed diff --git a/src/starbridge/web/utils.py b/src/starbridge/web/utils.py index 2b0bdfc..cc18001 100644 --- a/src/starbridge/web/utils.py +++ b/src/starbridge/web/utils.py @@ -1,3 +1,6 @@ +"""Utility functions for web-related operations like URL handling, content transformation, and link extraction.""" + +from http import HTTPStatus from urllib.parse import urljoin, urlparse, urlunparse import httpx @@ -13,28 +16,35 @@ from starbridge.utils import get_logger -from .types import ( +from .models import ( HTML_PARSER, Context, LinkTarget, MimeType, Resource, - RobotForbiddenException, + RobotForbiddenError, ) logger = get_logger(__name__) -def is_connected(): +def is_connected() -> bool: + """ + Check if there is an active internet connection by making a HEAD request to Google. + + Returns: + bool: True if connection is established, False otherwise + + """ try: response = requests.head("https://www.google.com", timeout=5) logger.info( "Called head on https://www.google.com/, got status_code: %s", response.status_code, ) - return response.status_code == 200 - except requests.exceptions.RequestException as e: - logger.exception("Failed to connect to www.google.com: %s", e) + return response.status_code == HTTPStatus.OK + except requests.exceptions.RequestException: + logger.exception("Failed to connect to www.google.com: %s") return False @@ -45,7 +55,13 @@ async def get_respectfully( timeout: int, respect_robots_txt: bool = True, ) -> httpx.Response: - """Fetch URL with proper headers and robot.txt checking.""" + """ + Fetch URL with proper headers and robot.txt checking. + + Returns: + httpx.Response: The HTTP response from the requested URL. + + """ async with AsyncClient() as client: if respect_robots_txt: await _ensure_allowed_to_crawl(url=url, user_agent=user_agent) @@ -80,7 +96,15 @@ def _get_robots_txt_url(url: str) -> str: async def _ensure_allowed_to_crawl(url: str, user_agent: str, timeout: int = 5) -> None: """ Ensure allowed to crawl the URL by the user agent according to the robots.txt file. - Raises a RuntimeError if not. + + Args: + url: Website URL to check + user_agent: User agent string to check permissions for + timeout: Request timeout in seconds + + Raises: + RobotForbiddenError: If crawling is not allowed according to the robots.txt file. + """ logger.debug("Checking if allowed to crawl %s", url) robot_txt_url = _get_robots_txt_url(url) @@ -94,16 +118,22 @@ async def _ensure_allowed_to_crawl(url: str, user_agent: str, timeout: int = 5) timeout=timeout, ) except HTTPError as e: - message = f"Failed to fetch robots.txt {robot_txt_url} due to a connection issue, thereby defensively assuming we are not allowed to access the url we want." + message = ( + f"Failed to fetch robots.txt {robot_txt_url} due to a connection issue, " + "thereby defensively assuming we are not allowed to access the url we " + "want." + ) logger.exception(message) - raise RobotForbiddenException(message) from e - if response.status_code in {401, 403}: + raise RobotForbiddenError(message) from e + if response.status_code in {HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN}: message = ( - f"When fetching robots.txt ({robot_txt_url}), received status {response.status_code} so assuming that autonomous fetching is not allowed, the user can try manually fetching by using the fetch prompt", + f"When fetching robots.txt ({robot_txt_url}), received status {response.status_code} " + "so assuming that autonomous fetching is not allowed, the user can try manually " + "fetching by using the fetch prompt" ) logger.error(message) - raise RobotForbiddenException(message) - if 400 <= response.status_code < 500: + raise RobotForbiddenError(message) + if HTTPStatus.BAD_REQUEST <= response.status_code < HTTPStatus.INTERNAL_SERVER_ERROR: return robot_txt = response.text processed_robot_txt = "\n".join(line for line in robot_txt.splitlines() if not line.strip().startswith("#")) @@ -114,15 +144,26 @@ async def _ensure_allowed_to_crawl(url: str, user_agent: str, timeout: int = 5) f"{user_agent}\n" f"{url}\n" f"\n{robot_txt}\n\n" - f"The assistant must let the user know that it failed to view the page. The assistant may provide further guidance based on the above information.\n" - f"The assistant can tell the user that they can try manually fetching the page by using the fetch prompt within their UI.", + f"The assistant must let the user know that it failed to view the page. " + "The assistant may provide further guidance based on the above information.\n" + f"The assistant can tell the user that they can try manually fetching the page " + "by using the fetch prompt within their UI." ) logger.error(message) - raise RobotForbiddenException(message) + raise RobotForbiddenError(message) def _get_normalized_content_type(response: httpx.Response) -> str: - """Get the normalized content type from the response.""" + """ + Get the normalized content type from the response. + + Args: + response: The HTTP response to get the content type from. + + Returns: + str: The normalized MIME type of the response content. + + """ content_type_mapping = { "html": MimeType.TEXT_HTML, "markdown": MimeType.TEXT_MARKDWON, @@ -155,7 +196,16 @@ def _get_normalized_content_type(response: httpx.Response) -> str: def _get_markdown_from_html(html: str) -> str: - """Get markdown from HTML content.""" + """ + Get markdown from HTML content. + + Args: + html: The HTML content to convert + + Returns: + str: The converted markdown content + + """ simplified = simple_json_from_html_string(html, use_readability=False) if simplified["content"]: return markdownify(simplified["content"], heading_style=ATX, strip=["img"]) @@ -165,38 +215,66 @@ def _get_markdown_from_html(html: str) -> str: def _get_markdown_from_pdf(response: httpx.Response) -> str | None: - """Get markdown from PDF content.""" + """ + Get markdown from PDF content. + + Returns: + str | None: Markdown string if conversion is successful, None otherwise + + """ try: rtn = MarkItDown().convert(str(response.url)) return rtn.text_content - except Exception as e: - logger.warning(f"Failed to convert PDF to markdown: {e}") + except Exception: + logger.exception("Failed to convert PDF to markdown") return None def _get_markdown_from_word(response: httpx.Response) -> str | None: - try: - rtn = MarkItDown().convert(str(response.url)) - return rtn.text_content - except Exception as e: - logger.warning(f"Failed to convert PDF to markdown: {e}") - return None + """ + Convert Word document content to markdown. + + Args: + response: HTTP response containing Word document + + Returns: + Markdown string if conversion successful, None otherwise + + """ + rtn = MarkItDown().convert(str(response.url)) + return rtn.text_content def _get_markdown_from_excel(response: httpx.Response) -> str | None: - try: - rtn = MarkItDown().convert(str(response.url)) - return rtn.text_content - except Exception as e: - logger.warning(f"Failed to convert PDF to markdown: {e}") - return None + """ + Convert Excel document content to markdown. + + Args: + response: HTTP response containing Excel document + + Returns: + Markdown string if conversion successful, None otherwise + + """ + rtn = MarkItDown().convert(str(response.url)) + return rtn.text_content def transform_content( response: httpx.Response, transform_to_markdown: bool = True, ) -> Resource: - """Process response according to requested format.""" + """ + Process response according to requested format. + + Args: + response: The HTTP response to process + transform_to_markdown: Whether to attempt converting content to markdown + + Returns: + Resource: Processed content as a Resource object + + """ content_type = _get_normalized_content_type(response) if transform_to_markdown: @@ -253,7 +331,17 @@ def transform_content( def _extract_links_from_html(html: str, url: str) -> list[LinkTarget]: - """Extract links from HTML content.""" + """ + Extract links from HTML content. + + Args: + html: The HTML content to extract links from + url: The base URL for resolving relative links + + Returns: + list[LinkTarget]: List of extracted links with metadata + + """ soup = BeautifulSoup(html, HTML_PARSER) seen_urls: dict[str, LinkTarget] = {} @@ -282,7 +370,16 @@ def _extract_links_from_html(html: str, url: str) -> list[LinkTarget]: def extract_links_from_response( response: httpx.Response, ) -> list[LinkTarget]: - """Extract links from HTML content.""" + """ + Extract links from HTML content. + + Args: + response: The HTTP response to extract links from. + + Returns: + list[LinkTarget]: List of extracted links with their metadata. + + """ match _get_normalized_content_type(response): case MimeType.TEXT_HTML: return _extract_links_from_html(response.text, str(response.url)) @@ -306,10 +403,14 @@ async def get_additional_context_for_url( Get additional context for the url. Args: - url: The URL to get additional context for. + url: The URL to get additional context for + user_agent: User agent string to use for requests + accept_language: Accept-Language header value + timeout: Request timeout in seconds + full: Whether to try fetching llms-full.txt first Returns: - additional context. + List of Context objects with additional information """ rtn = [] @@ -326,7 +427,7 @@ async def get_additional_context_for_url( follow_redirects=True, timeout=timeout, ) - if response.status_code == 200: + if response.status_code == HTTPStatus.OK: rtn.append( Context( type="llms_txt", @@ -335,7 +436,7 @@ async def get_additional_context_for_url( ), ) except HTTPError: - logger.warning(f"Failed to fetch llms-full.txt {llms_full_txt_url}") + logger.exception("Failed to fetch llms-full.txt %s", llms_full_txt_url) if len(rtn) == 0: llms_txt_url = _get_llms_txt_url(url, False) try: @@ -348,7 +449,7 @@ async def get_additional_context_for_url( follow_redirects=True, timeout=timeout, ) - if response.status_code == 200: + if response.status_code == HTTPStatus.OK: rtn.append( Context( type="llms_txt", @@ -357,7 +458,7 @@ async def get_additional_context_for_url( ), ) except HTTPError: - logger.warning(f"Failed to fetch llms.txt {llms_txt_url}") + logger.warning("Failed to fetch llms.txt %s", llms_txt_url) return rtn @@ -367,6 +468,7 @@ def _get_llms_txt_url(url: str, full: bool = True) -> str: Args: url: Website URL to get robots.txt for + full: If True, returns llms-full.txt URL, otherwise returns llms.txt URL Returns: URL of the robots.txt file diff --git a/tests/web/starbridge_web_mcp_test.py b/tests/web/starbridge_web_mcp_test.py index 6ef95eb..3ceee62 100644 --- a/tests/web/starbridge_web_mcp_test.py +++ b/tests/web/starbridge_web_mcp_test.py @@ -22,20 +22,19 @@ def runner(): @pytest.mark.asyncio async def test_web_mcp_tool_get() -> None: """Test server tool get.""" - async with stdio_client(_server_parameters()) as (read, write): - async with ClientSession(read, write) as session: - await session.initialize() - - result = await session.call_tool( - "starbridge_web_get", - { - "url": GET_TEST_URL, - "transform_to_markdown": True, - "extract_links": False, - "additional_context": False, - }, - ) - assert len(result.content) == 1 - content = result.content[0] - assert type(content) is TextContent - assert "\\n\\nstarbridge" in content.text + async with stdio_client(_server_parameters()) as (read, write), ClientSession(read, write) as session: + await session.initialize() + + result = await session.call_tool( + "starbridge_web_get", + { + "url": GET_TEST_URL, + "transform_to_markdown": True, + "extract_links": False, + "additional_context": False, + }, + ) + assert len(result.content) == 1 + content = result.content[0] + assert type(content) is TextContent + assert "\\n\\nstarbridge" in content.text diff --git a/tests/web/starbridge_web_service_test.py b/tests/web/starbridge_web_service_test.py index 1a77b2f..1674d6a 100644 --- a/tests/web/starbridge_web_service_test.py +++ b/tests/web/starbridge_web_service_test.py @@ -3,7 +3,7 @@ import pytest -from starbridge.web import RobotForbiddenException, Service +from starbridge.web import RobotForbiddenError, Service GET_TEST_TEXT_URL = ( "https://github.com/helmut-hoffer-von-ankershoffen/starbridge/raw/refs/heads/main/tests/fixtures/starbridge.txt" @@ -26,7 +26,7 @@ def test_web_service_get_forbidden() -> None: """Check getting content from the web fails if forbidden by robots.txt.""" from starbridge.web import Service - with pytest.raises(RobotForbiddenException): + with pytest.raises(RobotForbiddenError): asyncio.run(Service().get("https://github.com/search/advanced")) diff --git a/tests/web/starbridge_web_utils_test.py b/tests/web/starbridge_web_utils_test.py index 8d3360d..d955183 100644 --- a/tests/web/starbridge_web_utils_test.py +++ b/tests/web/starbridge_web_utils_test.py @@ -5,7 +5,7 @@ from httpx import TimeoutException from starbridge import __project_name__ -from starbridge.web import RobotForbiddenException +from starbridge.web import RobotForbiddenError from starbridge.web.utils import ( _ensure_allowed_to_crawl, get_additional_context_for_url, @@ -22,7 +22,7 @@ def test_web_utils_robots_disallowed_on_timeout() -> None: """Check disallowing on robots.txt timing out.""" - with pytest.raises(RobotForbiddenException): + with pytest.raises(RobotForbiddenError): with patch(HTTPX_ASYNC_CLIENT_GET) as mock_get: mock_get.side_effect = TimeoutException(TIMEOUT_MESSAGE) asyncio.run(_ensure_allowed_to_crawl(GET_TEST_URL, __project_name__)) @@ -30,7 +30,7 @@ def test_web_utils_robots_disallowed_on_timeout() -> None: def test_web_utils_robots_disallowed_on_401() -> None: """Check disallowing on robots.txt forbidden.""" - with pytest.raises(RobotForbiddenException): + with pytest.raises(RobotForbiddenError): with patch(HTTPX_ASYNC_CLIENT_GET) as mock_get: mock_get.return_value.status_code = 401 asyncio.run(_ensure_allowed_to_crawl(GET_TEST_URL, __project_name__))