Add non blocking calls to Ray Serve (#51)

* support non blocking serve

* update quickstart notebook

* update notebook

* update readme

* update installation

* add basic tests for top level function

* Update README.md

* add logo

* update quickstart

* update docs

* wrapper

* ignore import issues on ruff

* update

* Bump version: 0.2.0 → 0.2.1

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.7.1
+    rev: v0.7.2
     hooks:
       # Run the linter.
       - id: ruff
         types_or: [ python, pyi ]
-        args: [ --fix ]
+        args: [ --fix, --ignore=F403,F405 ]
       # Run the formatter.
       - id: ruff-format
         types_or: [ python, pyi ]

README.md

@@ -1,19 +1,19 @@
-[python_badge]: https://img.shields.io/badge/Python-3.10%20|%203.11%20|%203.12-brightgreen?style=for-the-badge
+[python_badge]: https://img.shields.io/badge/Python-3.10+-brightgreen?style=for-the-badge&logo=python&logoColor=white
 [pypi_badge]: https://img.shields.io/pypi/v/xinfer.svg?style=for-the-badge&logo=pypi&logoColor=white&label=PyPI&color=blue
 [downloads_badge]: https://img.shields.io/pepy/dt/xinfer.svg?style=for-the-badge&logo=pypi&logoColor=white&label=Downloads&color=purple
 [license_badge]: https://img.shields.io/badge/License-Apache%202.0-green.svg?style=for-the-badge&logo=apache&logoColor=white
 [transformers_badge]: https://img.shields.io/github/stars/huggingface/transformers?style=for-the-badge&logo=huggingface&label=Transformers%20⭐&color=yellow
 [timm_badge]: https://img.shields.io/github/stars/huggingface/pytorch-image-models?style=for-the-badge&logo=pytorch&label=TIMM%20⭐&color=limegreen
 [ultralytics_badge]: https://img.shields.io/github/stars/ultralytics/ultralytics?style=for-the-badge&logo=udacity&label=Ultralytics%20⭐&color=red
 [vllm_badge]: https://img.shields.io/github/stars/vllm-project/vllm?style=for-the-badge&logo=v&label=vLLM%20⭐&color=purple
-[ollama_badge]: https://img.shields.io/github/stars/ollama/ollama?style=for-the-badge&logo=llama&label=Ollama%20⭐&color=darkgreen
+[ollama_badge]: https://img.shields.io/github/stars/ollama/ollama?style=for-the-badge&logo=ollama&label=Ollama%20⭐&color=darkgreen
 [colab_badge]: https://img.shields.io/badge/Open%20In-Colab-blue?style=for-the-badge&logo=google-colab
 [kaggle_badge]: https://img.shields.io/badge/Open%20In-Kaggle-blue?style=for-the-badge&logo=kaggle
 [back_to_top_badge]: https://img.shields.io/badge/Back_to_Top-↑-blue?style=for-the-badge
 [image_classification_badge]: https://img.shields.io/badge/Image%20Classification-blueviolet?style=for-the-badge
 [object_detection_badge]: https://img.shields.io/badge/Object%20Detection-coral?style=for-the-badge
 [image_to_text_badge]: https://img.shields.io/badge/Image%20to%20Text-gold?style=for-the-badge
-[os_badge]: https://img.shields.io/badge/Tested%20on-Linux%20%7C%20macOS%20%7C%20Windows-indigo?style=for-the-badge
+[os_badge]: https://img.shields.io/badge/Tested%20on-Linux%20%7C%20macOS%20%7C%20Windows-indigo?style=for-the-badge&logo=iterm2&logoColor=white&color=indigo
 
 
 ![Python][python_badge]
@@ -57,7 +57,7 @@ You don't want to spend a weekend learning `TyPorch` just to find out the model
 
 This is where x.infer comes in. 
 
-x.infer is a simple library that allows you to run inference with any computer vision model in just a few lines of code. All in Python.
+x.infer is a simple wrapper that allows you to run inference with any computer vision model in just a few lines of code. All in Python.
 
 Out of the box, x.infer supports the following frameworks:
 
@@ -109,12 +109,12 @@ import xinfer
 
 model = xinfer.create_model("vikhyatk/moondream2")
 
-image = "https://raw.githubusercontent.com/vikhyat/moondream/main/assets/demo-1.jpg"
+image = "https://raw.githubusercontent.com/dnth/x.infer/main/assets/demo/00aa2580828a9009.jpg"
 prompt = "Describe this image. "
 
 model.infer(image, prompt)
 
->>> An animated character with long hair and a serious expression is eating a large burger at a table, with other characters in the background.
+>>> 'A parade with a marching band and a flag-bearing figure passes through a town, with spectators lining the street and a church steeple visible in the background.'
 ```
 
 ## 📦 Installation
@@ -132,6 +132,7 @@ pip install "xinfer[transformers]"
 pip install "xinfer[ultralytics]"
 pip install "xinfer[timm]"
 pip install "xinfer[vllm]"
+pip install "xinfer[ollama]"
 ```
 
 To install all optional dependencies, run:
@@ -158,7 +159,7 @@ xinfer.list_models()
        Available Models                                      
 ┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━┓
 ┃ Implementation ┃ Model ID                                              ┃ Input --> Output     ┃
-┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━��━┩
+┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━┩
 │ timm           │ timm/eva02_large_patch14_448.mim_m38m_ft_in22k_in1k   │ image --> categories │
 │ timm           │ timm/eva02_large_patch14_448.mim_m38m_ft_in1k         │ image --> categories │
 │ timm           │ timm/eva02_large_patch14_448.mim_in22k_ft_in22k_in1k  │ image --> categories │
@@ -252,7 +253,7 @@ curl -X 'POST' \
   -H 'accept: application/json' \
   -H 'Content-Type: application/json' \
   -d '{
-  "image": "https://raw.githubusercontent.com/dnth/x.infer/refs/heads/main/assets/demo/00aa2580828a9009.jpg",
+  "image": "https://raw.githubusercontent.com/dnth/x.infer/main/assets/demo/00aa2580828a9009.jpg",
   "infer_kwargs": {"prompt": "Caption this image"}
 }'
 ```
@@ -268,7 +269,7 @@ headers = {
     "Content-Type": "application/json"
 }
 payload = {
-    "image": "https://raw.githubusercontent.com/dnth/x.infer/refs/heads/main/assets/demo/00aa2580828a9009.jpg",
+    "image": "https://raw.githubusercontent.com/dnth/x.infer/main/assets/demo/00aa2580828a9009.jpg",
     "infer_kwargs": {
         "prompt": "Caption this image"
     }

docs/index.md

@@ -1,16 +1,36 @@
-![Python](https://img.shields.io/badge/Python-3.10%20|%203.11%20|%203.12-brightgreen?style=for-the-badge)
-[![PyPI version](https://img.shields.io/pypi/v/xinfer.svg?style=for-the-badge&logo=pypi&logoColor=white&label=PyPI&color=blue)](https://pypi.org/project/xinfer/)
-[![Downloads](https://img.shields.io/pypi/dm/xinfer.svg?style=for-the-badge&logo=pypi&logoColor=white&label=Downloads&color=purple)](https://pypi.org/project/xinfer/)
-![License](https://img.shields.io/badge/License-Apache%202.0-green.svg?style=for-the-badge&logo=apache&logoColor=white)
+[python_badge]: https://img.shields.io/badge/Python-3.10+-brightgreen?style=for-the-badge&logo=python&logoColor=white
+[pypi_badge]: https://img.shields.io/pypi/v/xinfer.svg?style=for-the-badge&logo=pypi&logoColor=white&label=PyPI&color=blue
+[downloads_badge]: https://img.shields.io/pepy/dt/xinfer.svg?style=for-the-badge&logo=pypi&logoColor=white&label=Downloads&color=purple
+[license_badge]: https://img.shields.io/badge/License-Apache%202.0-green.svg?style=for-the-badge&logo=apache&logoColor=white
+[transformers_badge]: https://img.shields.io/github/stars/huggingface/transformers?style=for-the-badge&logo=huggingface&label=Transformers%20⭐&color=yellow
+[timm_badge]: https://img.shields.io/github/stars/huggingface/pytorch-image-models?style=for-the-badge&logo=pytorch&label=TIMM%20⭐&color=limegreen
+[ultralytics_badge]: https://img.shields.io/github/stars/ultralytics/ultralytics?style=for-the-badge&logo=udacity&label=Ultralytics%20⭐&color=red
+[vllm_badge]: https://img.shields.io/github/stars/vllm-project/vllm?style=for-the-badge&logo=v&label=vLLM%20⭐&color=purple
+[ollama_badge]: https://img.shields.io/github/stars/ollama/ollama?style=for-the-badge&logo=ollama&label=Ollama%20⭐&color=darkgreen
+[colab_badge]: https://img.shields.io/badge/Open%20In-Colab-blue?style=for-the-badge&logo=google-colab
+[kaggle_badge]: https://img.shields.io/badge/Open%20In-Kaggle-blue?style=for-the-badge&logo=kaggle
+[back_to_top_badge]: https://img.shields.io/badge/Back_to_Top-↑-blue?style=for-the-badge
+[image_classification_badge]: https://img.shields.io/badge/Image%20Classification-blueviolet?style=for-the-badge
+[object_detection_badge]: https://img.shields.io/badge/Object%20Detection-coral?style=for-the-badge
+[image_to_text_badge]: https://img.shields.io/badge/Image%20to%20Text-gold?style=for-the-badge
+[os_badge]: https://img.shields.io/badge/Tested%20on-Linux%20%7C%20macOS%20%7C%20Windows-indigo?style=for-the-badge&logo=iterm2&logoColor=white&color=indigo
+
+
+[![Python][python_badge]](https://pypi.org/project/xinfer/)
+[![PyPI version][pypi_badge]](https://pypi.org/project/xinfer/)
+[![Downloads][downloads_badge]](https://pypi.org/project/xinfer/)
+[![License][license_badge]](https://pypi.org/project/xinfer/)
+[![OS Support][os_badge]](https://pypi.org/project/xinfer/)
 
 
 <div align="center">
     <img src="https://raw.githubusercontent.com/dnth/x.infer/refs/heads/main/assets/xinfer.jpg" alt="x.infer" width="500"/>
+    <img src="https://raw.githubusercontent.com/dnth/x.infer/refs/heads/main/assets/code_typing.gif" alt="x.infer" width="500"/>
     <br />
     <br />
     <a href="https://dnth.github.io/x.infer" target="_blank" rel="noopener noreferrer"><strong>Explore the docs »</strong></a>
     <br />
-    <a href="#quickstart" target="_blank" rel="noopener noreferrer">Quickstart</a>
+    <a href="#-quickstart" target="_blank" rel="noopener noreferrer">Quickstart</a>
     ·
     <a href="https://github.com/dnth/x.infer/issues/new?assignees=&labels=Feature+Request&projects=&template=feature_request.md" target="_blank" rel="noopener noreferrer">Feature Request</a>
     ·
@@ -21,18 +41,58 @@
     <a href="https://dicksonneoh.com/" target="_blank" rel="noopener noreferrer">About</a>
 </div>
 
+<div align="center">
+    <br />
+
+</div>
+
+
+## 🤔 Why x.infer?
+So, a new computer vision model just dropped last night. It's called `GPT-54o-mini-vision-pro-max-xxxl`. It's a super cool model, open-source, open-weights, open-data, all the good stuff.
+
+You're excited. You want to try it out. 
+
+But it's written in a new framework, `TyPorch` that you know nothing about.
+You don't want to spend a weekend learning `TyPorch` just to find out the model is not what you expected.
+
+This is where x.infer comes in. 
+
+x.infer is a simple library that allows you to run inference with any computer vision model in just a few lines of code. All in Python.
+
+Out of the box, x.infer supports the following frameworks:
+
+[![Transformers][transformers_badge]](https://github.com/huggingface/transformers)
+[![TIMM][timm_badge]](https://github.com/huggingface/pytorch-image-models)
+[![Ultralytics][ultralytics_badge]](https://github.com/ultralytics/ultralytics)
+[![vLLM][vllm_badge]](https://github.com/vllm-project/vllm)
+[![Ollama][ollama_badge]](https://github.com/ollama/ollama)
+
+Combined, x.infer supports over 1000+ models from all the above frameworks.
+
+Tasks supported:
+
+![Image Classification][image_classification_badge]
+![Object Detection][object_detection_badge]
+![Image to Text][image_to_text_badge]
+
+Run any supported model using the following 4 lines of code:
+
+```python
+import xinfer
 
-## Why x.infer?
-If you'd like to run many models from different libraries without having to rewrite your inference code, x.infer is for you. It has a simple API and is easy to extend. Currently supports Transformers, Ultralytics, and TIMM.
+model = xinfer.create_model("vikhyatk/moondream2")
+model.infer(image, prompt)         # Run single inference
+model.infer_batch(images, prompts) # Run batch inference
+model.launch_gradio()              # Launch Gradio interface
+```
 
-Have a custom model? Create a class that implements the `BaseModel` interface and register it with x.infer. See [Adding New Models](#adding-new-models) for more details.
+Have a custom model? Create a class that implements the `BaseModel` interface and register it with x.infer. See [Add Your Own Model](#add-your-own-model) for more details.
 
-## Key Features
+## 🌟 Key Features
 <div align="center">
-  <img src="https://raw.githubusercontent.com/dnth/x.infer/refs/heads/main/assets/flowchart.gif" alt="x.infer" width="500"/>
+  <img src="https://raw.githubusercontent.com/dnth/x.infer/refs/heads/main/assets/flowchart.gif" alt="x.infer" width="900"/>
 </div>
 
-- **Unified Interface:** Interact with different machine learning models through a single, consistent API.
+- **Unified Interface:** Interact with different computer vision frameworks through a single, consistent API.
 - **Modular Design:** Integrate and swap out models without altering the core framework.
-- **Ease of Use:** Simplifies model loading, input preprocessing, inference execution, and output postprocessing.
 - **Extensibility:** Add support for new models and libraries with minimal code changes.

docs/installation.md

@@ -1,20 +1,22 @@
-# Installation
 
-You must have [PyTorch](https://pytorch.org/get-started/locally/) installed to use x.infer.
+> [!IMPORTANT]
+> You must have [PyTorch](https://pytorch.org/get-started/locally/) installed to use x.infer.
 
 To install the barebones x.infer (without any optional dependencies), run:
 ```bash
 pip install xinfer
 ```
-x.infer can be used with multiple optional libraries. You'll just need to install one or more of the following:
+x.infer can be used with multiple optional dependencies. You'll just need to install one or more of the following:
 
 ```bash
 pip install "xinfer[transformers]"
 pip install "xinfer[ultralytics]"
 pip install "xinfer[timm]"
+pip install "xinfer[vllm]"
+pip install "xinfer[ollama]"
 ```
 
-To install all libraries, run:
+To install all optional dependencies, run:
 ```bash
 pip install "xinfer[all]"
 ```

mkdocs.yml

@@ -23,8 +23,8 @@ theme:
     name: material
     icon:
         repo: fontawesome/brands/github
-    # logo: assets/xinfer.jpg
-    # favicon: assets/favicon.png
+    logo: assets/favicon.png
+    favicon: assets/favicon.png
     features:
         - navigation.instant
         - navigation.tracking

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "xinfer"
-version = "0.2.0"
+version = "0.2.1"
 dynamic = [
     "dependencies",
 ]
@@ -48,7 +48,7 @@ universal = true
 
 
 [tool.bumpversion]
-current_version = "0.2.0"
+current_version = "0.2.1"
 commit = true
 tag = true
 

tests/test_xinfer_top_functions.py

@@ -0,0 +1,14 @@
+import xinfer
+
+
+def test_list_models():
+    # Should not return any errors
+    xinfer.list_models()
+
+
+def test_list_models_interactive():
+    result = xinfer.list_models(interactive=True)
+    assert result is not None
+
+    result = xinfer.list_models(interactive=True, limit=10)
+    assert len(result) == 10 + 2

xinfer/__init__.py

@@ -2,7 +2,7 @@
 
 __author__ = """Dickson Neoh"""
 __email__ = "dickson.neoh@gmail.com"
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 from .core import create_model, list_models
 from .model_registry import ModelInputOutput, register_model

xinfer/core.py

@@ -49,16 +49,6 @@ def list_models(search: str = None, limit: int = 20, interactive: bool = False):
         )
         return
 
-    if interactive:
-        from itables import init_notebook_mode
-
-        logger.info(
-            "Showing interactive table in Jupyter Notebook. Type in the search bar to filter the models."
-        )
-
-        init_notebook_mode(all_interactive=True)
-        return pd.DataFrame(rows)
-
     if len(rows) > limit:
         rows = rows[:limit]
         rows.append(
@@ -72,6 +62,16 @@ def list_models(search: str = None, limit: int = 20, interactive: bool = False):
             f"Showing only top {limit} models. Change the `limit` parameter to see more."
         )
 
+    if interactive:
+        from itables import init_notebook_mode
+
+        logger.info(
+            "Showing interactive table in Jupyter Notebook. Type in the search bar to filter the models."
+        )
+
+        init_notebook_mode(all_interactive=True)
+        return pd.DataFrame(rows)
+
     console = Console()
     table = Table(title="Available Models")
     table.add_column("Implementation", style="cyan")

xinfer/serve.py

@@ -67,6 +67,7 @@ def serve_model(
     deployment_kwargs: dict = None,
     host: str = "127.0.0.1",
     port: int = 8000,
+    blocking: bool = True,
     **model_kwargs,
 ):
     deployment_kwargs = deployment_kwargs or {}
@@ -83,8 +84,12 @@ def serve_model(
     app = deployment.bind(model_id, **model_kwargs)
 
     try:
-        serve.run(app, blocking=True)
+        handle = serve.run(app, blocking=blocking)
+        if not blocking:
+            logger.info(
+                "Running server in non-blocking mode, remember to call serve.shutdown() to stop the server"
+            )
+            return handle  # Return handle without shutting down
     except (KeyboardInterrupt, SystemExit):
         logger.info("Receiving shutdown signal. Cleaning up...")
-    finally:
         serve.shutdown()