From c2ba1b4fdf8d25d57b8715e7c21be0660cf07fb9 Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 22 Nov 2024 13:29:27 +0800 Subject: [PATCH] add docs --- .github/workflows/docs.yml | 25 ++++++++++ README.md | 2 +- docs/index.md | 52 +++++++++++++++++++++ docs/overrides/main.html | 11 +++++ docs/quickstart.md | 93 ++++++++++++++++++++++++++++++++++++++ docs/usage.md | 52 +++++++++++++++++++++ mkdocs.yml | 87 +++++++++++++++++++++++++++++++++++ pyproject.toml | 22 +++++++++ 8 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docs.yml create mode 100644 docs/index.md create mode 100644 docs/overrides/main.html create mode 100644 docs/quickstart.md create mode 100644 docs/usage.md create mode 100644 mkdocs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..3a20dc3 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,25 @@ +name: docs +on: + push: + branches: + - main + - master +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ".[docs]" + - name: PKG-TEST + run: | + python -m unittest discover tests/ + - run: mkdocs gh-deploy --force diff --git a/README.md b/README.md index abadfb5..18b0a4f 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [downloads_badge]: https://img.shields.io/pepy/dt/xretrieval.svg?style=for-the-badge&logo=pypi&logoColor=white&label=Downloads&color=purple [license_badge]: https://img.shields.io/badge/License-Apache%202.0-green.svg?style=for-the-badge&logo=apache&logoColor=white -![Python][python_badge] +[![Python][python_badge]](https://pypi.org/project/xretrieval/) [![PyPI version][pypi_badge]](https://pypi.org/project/xretrieval/) [![Downloads][downloads_badge]](https://pypi.org/project/xretrieval/) ![License][license_badge] diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..aa3f8ea --- /dev/null +++ b/docs/index.md @@ -0,0 +1,52 @@ +[colab_badge]: https://img.shields.io/badge/Open%20In-Colab-blue?style=for-the-badge&logo=google-colab +[kaggle_badge]: https://img.shields.io/badge/Open%20In-Kaggle-blue?style=for-the-badge&logo=kaggle + +[python_badge]: https://img.shields.io/badge/Python-3.10+-brightgreen?style=for-the-badge&logo=python&logoColor=white +[pypi_badge]: https://img.shields.io/pypi/v/xretrieval.svg?style=for-the-badge&logo=pypi&logoColor=white&label=PyPI&color=blue +[downloads_badge]: https://img.shields.io/pepy/dt/xretrieval.svg?style=for-the-badge&logo=pypi&logoColor=white&label=Downloads&color=purple +[license_badge]: https://img.shields.io/badge/License-Apache%202.0-green.svg?style=for-the-badge&logo=apache&logoColor=white + +[![Python][python_badge]](https://pypi.org/project/xretrieval/) +[![PyPI version][pypi_badge]](https://pypi.org/project/xretrieval/) +[![Downloads][downloads_badge]](https://pypi.org/project/xretrieval/) +![License][license_badge] + +
+ x.retrieval +
+
+ Explore the docs » +
+ Quickstart + · + Feature Request + · + Report Bug + · + Discussions + · + About +
+
+
+Evaluate your multimodal retrieval system with any models and datasets. + + +Specific inputs: + +- A dataset +- A model +- A mode (e.g. `image-to-image`) + +Get evaluation metrics: + +- A retrieval results dataframe +- A retrieval metrics dataframe + +## 🌟 Key Features + +- ✅ Supports a wide range of models and datasets. +- ✅ Installation in one line. +- ✅ Run benchmarks with one function call. + + diff --git a/docs/overrides/main.html b/docs/overrides/main.html new file mode 100644 index 0000000..702c96b --- /dev/null +++ b/docs/overrides/main.html @@ -0,0 +1,11 @@ +{% extends "base.html" %} + +{% block content %} +{% if page.nb_url %} + + {% include ".icons/material/download.svg" %} + +{% endif %} + +{{ super() }} +{% endblock content %} diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 0000000..47f1825 --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,93 @@ + +[colab_badge]: https://img.shields.io/badge/Open%20In-Colab-blue?style=for-the-badge&logo=google-colab +[kaggle_badge]: https://img.shields.io/badge/Open%20In-Kaggle-blue?style=for-the-badge&logo=kaggle + +[![Open In Colab][colab_badge]](https://colab.research.google.com/github/dnth/x.retrieval/blob/main/nbs/quickstart.ipynb) +[![Open In Kaggle][kaggle_badge]](https://kaggle.com/kernels/welcome?src=https://github.com/dnth/x.retrieval/blob/main/nbs/quickstart.ipynb) + + +```python +import xretrieval + +metrics, results_df = xretrieval.run_benchmark( + dataset="coco-val-2017", + model_id="transformers/Salesforce/blip2-itm-vit-g", + mode="text-to-text", +) + +``` + +```bash + Retrieval Metrics +┏━━━━━━━━━━━━━━━┳━━━━━━━━┓ +┃ Metric ┃ Score ┃ +┡━━━━━━━━━━━━━━━╇━━━━━━━━┩ +│ MRR │ 0.3032 │ +│ NormalizedDCG │ 0.3497 │ +│ Precision │ 0.2274 │ +│ Recall │ 0.4898 │ +│ HitRate │ 0.4898 │ +│ MAP │ 0.2753 │ +└───────────────┴────────┘ + +``` + +## 📦 Installation + +```bash +pip install xretrieval +``` + +## 🛠️ Usage + +List datasets: + +```python +xretrieval.list_datasets() +``` + +```bash + Available Datasets +┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Dataset Name ┃ Description ┃ +┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ coco-val-2017 │ The COCO Validation Set with 5k images. │ +└───────────────┴─────────────────────────────────────────┘ +``` + +List models: + +```python +xretrieval.list_models() +``` + +```bash + Available Models +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ +┃ Model ID ┃ Model Input ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ +│ transformers/Salesforce/blip2-itm-vit-g │ text-image │ +│ transformers/Salesforce/blip2-itm-vit-g-text │ text │ +│ transformers/Salesforce/blip2-itm-vit-g-image │ image │ +│ sentence-transformers/paraphrase-MiniLM-L3-v2 │ text │ +│ sentence-transformers/paraphrase-albert-small-v2 │ text │ +│ sentence-transformers/multi-qa-distilbert-cos-v1 │ text │ +│ sentence-transformers/all-MiniLM-L12-v2 │ text │ +│ sentence-transformers/all-distilroberta-v1 │ text │ +│ sentence-transformers/multi-qa-mpnet-base-dot-v1 │ text │ +│ sentence-transformers/all-mpnet-base-v2 │ text │ +│ sentence-transformers/multi-qa-MiniLM-L6-cos-v1 │ text │ +│ sentence-transformers/all-MiniLM-L6-v2 │ text │ +│ timm/resnet18.a1_in1k │ image │ +└──────────────────────────────────────────────────┴─────────────┘ +``` + + +Visualize retrieval results: + +```python +xretrieval.visualize_retrieval(results_df) +``` + +![alt text](assets/viz1.png) +![alt text](assets/viz2.png) \ No newline at end of file diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..c8dde1d --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,52 @@ +List datasets: + +```python +import xretrieval +xretrieval.list_datasets() +``` + +```bash + Available Datasets +┏━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +┃ Dataset Name ┃ Description ┃ +┡━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩ +│ coco-val-2017 │ The COCO Validation Set with 5k images. │ +└───────────────┴─────────────────────────────────────────┘ +``` + +List models: + +```python +xretrieval.list_models() +``` + +```bash + Available Models +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┓ +┃ Model ID ┃ Model Input ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━┩ +│ transformers/Salesforce/blip2-itm-vit-g │ text-image │ +│ transformers/Salesforce/blip2-itm-vit-g-text │ text │ +│ transformers/Salesforce/blip2-itm-vit-g-image │ image │ +│ sentence-transformers/paraphrase-MiniLM-L3-v2 │ text │ +│ sentence-transformers/paraphrase-albert-small-v2 │ text │ +│ sentence-transformers/multi-qa-distilbert-cos-v1 │ text │ +│ sentence-transformers/all-MiniLM-L12-v2 │ text │ +│ sentence-transformers/all-distilroberta-v1 │ text │ +│ sentence-transformers/multi-qa-mpnet-base-dot-v1 │ text │ +│ sentence-transformers/all-mpnet-base-v2 │ text │ +│ sentence-transformers/multi-qa-MiniLM-L6-cos-v1 │ text │ +│ sentence-transformers/all-MiniLM-L6-v2 │ text │ +│ timm/resnet18.a1_in1k │ image │ +└──────────────────────────────────────────────────┴─────────────┘ +``` + + +Visualize retrieval results: + +```python +xretrieval.visualize_retrieval(results_df) +``` + +![Visualization 1](https://raw.githubusercontent.com/dnth/x.retrieval/main/assets/viz1.png) +![Visualization 2](https://raw.githubusercontent.com/dnth/x.retrieval/main/assets/viz2.png) diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..fac2164 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,87 @@ +site_name: x.retrieval +site_description: Evaluate your multimodal retrieval system with any models and datasets. +site_author: dnth +site_url: https://dnth.github.io/x.retrieval +repo_url: https://github.com/dnth/x.retrieval + +copyright: "Copyright © 2024 - 2024 Dickson Neoh" + +theme: + palette: + - scheme: default + # primary: blue + # accent: indigo + toggle: + icon: material/toggle-switch-off-outline + name: Switch to dark mode + - scheme: slate + primary: indigo + accent: indigo + toggle: + icon: material/toggle-switch + name: Switch to light mode + name: material + icon: + repo: fontawesome/brands/github + # logo: assets/favicon.png + # favicon: assets/favicon.png + features: + - navigation.instant + - navigation.tracking + - navigation.top + - search.highlight + - search.share + custom_dir: "docs/overrides" + font: + text: Google Sans + code: Regular + +plugins: + - search + - mkdocstrings + - git-revision-date + - git-revision-date-localized: + enable_creation_date: true + type: timeago + # - pdf-export + - mkdocs-jupyter: + include_source: True + ignore_h1_titles: True + execute: True + allow_errors: false + ignore: ["conf.py"] + execute_ignore: ["*ignore.ipynb"] + +markdown_extensions: + - admonition + - abbr + - attr_list + - def_list + - footnotes + - meta + - md_in_html + - pymdownx.superfences + - pymdownx.highlight: + linenums: false + - toc: + permalink: true + +# extra: +# analytics: +# provider: google +# property: UA-XXXXXXXXX-X + +nav: + - Home: index.md + # - Installation: installation.md + - Quickstart: quickstart.md + - Usage: usage.md + # - Contributing: contributing.md + # - FAQ: faq.md + # - Changelog: changelog.md + # - Report Issues: https://github.com/dnth/xinfer/issues + # - Examples: + # - examples/quickstart.ipynb + # - API Reference: + # - xinfer module: xinfer.md + # - common module: common.md diff --git a/pyproject.toml b/pyproject.toml index 720bb0c..3f6acbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,28 @@ dev = [ "flake8>=4.0", ] +docs = [ + "ipykernel", + "livereload", + "nbconvert", + "nbformat", + "sphinx", + "watchdog", + "mkdocs", + "mkdocs-git-revision-date-plugin", + "mkdocs-git-revision-date-localized-plugin", + "mkdocs-jupyter>=0.24.0", + "mkdocs-material>=9.1.3", + "mkdocs-pdf-export-plugin", + "mkdocstrings", + "mkdocstrings-crystal", + "mkdocstrings-python-legacy", + "pygments", + "pymdown-extensions", +] + + + [project.urls] "Homepage" = "https://github.com/dnth/x.retrieval" "Bug Tracker" = "https://github.com/dnth/x.retrieval/issues"