From f2d615b70e85c85e30f7e525b590bbba1436904f Mon Sep 17 00:00:00 2001 From: Mark Saroufim Date: Mon, 25 Sep 2023 16:25:46 -0700 Subject: [PATCH] torchbench is now a library (#1933) Summary: ```python import torchbenchmark.models.densenet121 model, example_inputs = torchbenchmark.models.densenet121.Model(test="eval", device="cuda", batch_size=1).get_module() model(*example_inputs) ``` So making the above example work with `import torchbenchmark` alone is a bit trickier there's a lot of relative imports that need to be fixed in pretty much every single file I also added a simple `test_imports.py` file to make sure this doesn't break anything, I can setup a standalone github action or plug into an existing one (would rather do the latter to avoid another job that needs to run `install.py` again This also solves voznesenskym favorite design pattern from https://github.com/pytorch/pytorch/blob/main/benchmarks/dynamo/torchbench.py ```python for torchbench_dir in ( "./torchbenchmark", "../torchbenchmark", "../torchbench", "../benchmark", "../../torchbenchmark", "../../torchbench", "../../benchmark", ): ``` Pull Request resolved: https://github.com/pytorch/benchmark/pull/1933 Reviewed By: xuzhao9 Differential Revision: D49608012 Pulled By: msaroufim fbshipit-source-id: 382a31f2c5a464608faa730ae15148f59a3e7484 --- .gitignore | 3 ++- README.md | 24 ++++++++++++++++++++++++ setup.py | 16 ++++++++++++++++ test_imports.py | 8 ++++++++ torchbenchmark/__init__.py | 4 ++++ torchbenchmark/canary_models/__init__.py | 1 + torchbenchmark/e2e_models/__init__.py | 1 + torchbenchmark/models/__init__.py | 1 + torchbenchmark/score/__init__.py | 1 + 9 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 setup.py create mode 100644 test_imports.py create mode 100644 torchbenchmark/canary_models/__init__.py create mode 100644 torchbenchmark/e2e_models/__init__.py create mode 100644 torchbenchmark/models/__init__.py create mode 100644 torchbenchmark/score/__init__.py diff --git a/.gitignore b/.gitignore index 30ed2a7324..6b884ee5aa 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,5 @@ old.json te.json logs/ scripts/scribe.py -.userbenchmark/ \ No newline at end of file +.userbenchmark/ +torchbench.egg-info/ \ No newline at end of file diff --git a/README.md b/README.md index d46d962b59..25665edfd2 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,30 @@ cd benchmark python install.py ``` +### Install torchbench as a library + +if you're interested in running torchbench as a library you can + +```bash +python install.py +pip install git+https://www.github.com:pytorch/benchmark.git +``` + +or + +```bash +python install.py +pip install . # add -e for an editable installation +``` + +The above + +```python +import torchbenchmark.models.densenet121 +model, example_inputs = torchbenchmark.models.densenet121.Model(test="eval", device="cuda", batch_size=1).get_module() +model(*example_inputs) +``` + ### Building From Source Note that when building PyTorch from source, torchvision and torchaudio must also be built from source to make sure the C APIs match. diff --git a/setup.py b/setup.py new file mode 100644 index 0000000000..a44e85e758 --- /dev/null +++ b/setup.py @@ -0,0 +1,16 @@ +from setuptools import setup, find_packages + +setup( + name='torchbench', + version='0.1', + description='Benchmarking library for PyTorch', + author='PyTorch Team', + url='https://github.com/pytorch/benchmark', + packages=find_packages(include=['torchbenchmark*', 'userbenchmark*']), + classifiers=[ + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Build Tools', + 'License :: OSI Approved :: BSD 3 License', + 'Programming Language :: Python', + ], +) diff --git a/test_imports.py b/test_imports.py new file mode 100644 index 0000000000..d8cf9b8ffe --- /dev/null +++ b/test_imports.py @@ -0,0 +1,8 @@ +import torchbenchmark.models.densenet121 +model, example_inputs = torchbenchmark.models.densenet121.Model(test="eval", device="cuda", batch_size=1).get_module() +model(*example_inputs) + +import userbenchmark.optim +import torchbenchmark.canary_models +import torchbenchmark.models +import torchbenchmark.score diff --git a/torchbenchmark/__init__.py b/torchbenchmark/__init__.py index 00eeac0443..cdcdc50052 100644 --- a/torchbenchmark/__init__.py +++ b/torchbenchmark/__init__.py @@ -17,6 +17,10 @@ from components._impl.tasks import base as base_task from components._impl.workers import subprocess_worker +from . import models +from . import canary_models +from . import e2e_models +from . import util class ModelNotFoundError(RuntimeError): pass diff --git a/torchbenchmark/canary_models/__init__.py b/torchbenchmark/canary_models/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/torchbenchmark/canary_models/__init__.py @@ -0,0 +1 @@ + diff --git a/torchbenchmark/e2e_models/__init__.py b/torchbenchmark/e2e_models/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/torchbenchmark/e2e_models/__init__.py @@ -0,0 +1 @@ + diff --git a/torchbenchmark/models/__init__.py b/torchbenchmark/models/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/torchbenchmark/models/__init__.py @@ -0,0 +1 @@ + diff --git a/torchbenchmark/score/__init__.py b/torchbenchmark/score/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/torchbenchmark/score/__init__.py @@ -0,0 +1 @@ +