diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000000..be3481754e --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,78 @@ +name: Publish Python distribution to PyPI + +on: + push: + tags: + - '*' + +jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v3 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python distribution to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/lm_eval + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + publish-to-testpypi: + name: Publish Python distribution to TestPyPI + needs: + - build + runs-on: ubuntu-latest + + environment: + name: testpypi + url: https://test.pypi.org/p/lm_eval + + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 2c63804bda..12b4c24763 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -56,7 +56,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e '.[dev,anthropic,sentencepiece]' --extra-index-url https://download.pytorch.org/whl/cpu + pip install -e '.[dev,anthropic,sentencepiece,optimum]' --extra-index-url https://download.pytorch.org/whl/cpu # Install optional git dependencies # pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/CITATION.bib b/CITATION.bib index cad48f31fe..4ec33f1396 100644 --- a/CITATION.bib +++ b/CITATION.bib @@ -1,26 +1,10 @@ -@software{eval-harness, - author = {Gao, Leo and - Tow, Jonathan and - Biderman, Stella and - Black, Sid and - DiPofi, Anthony and - Foster, Charles and - Golding, Laurence and - Hsu, Jeffrey and - McDonell, Kyle and - Muennighoff, Niklas and - Phang, Jason and - Reynolds, Laria and - Tang, Eric and - Thite, Anish and - Wang, Ben and - Wang, Kevin and - Zou, Andy}, +@misc{eval-harness, + author = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy}, title = {A framework for few-shot language model evaluation}, - month = sep, - year = 2021, + month = 12, + year = 2023, publisher = {Zenodo}, - version = {v0.0.1}, - doi = {10.5281/zenodo.5371628}, - url = {https://doi.org/10.5281/zenodo.5371628} + version = {v0.4.0}, + doi = {10.5281/zenodo.10256836}, + url = {https://zenodo.org/records/10256836} } diff --git a/README.md b/README.md index e048399363..5ae743d91b 100644 --- a/README.md +++ b/README.md @@ -46,25 +46,7 @@ cd lm-evaluation-harness pip install -e . ``` -We also provide a number of optional dependencies for extended functionality. Extras can be installed via `pip install -e ".[NAME]"` - -| Name | Use | -|---------------|---------------------------------------| -| anthropic | For using Anthropic's models | -| dev | For linting PRs and contributions | -| gptq | For loading models with GPTQ | -| ifeval | For running the IFEval task | -| mamba | For loading Mamba SSM models | -| math | For running math task answer checking | -| multilingual | For multilingual tokenizers | -| openai | For using OpenAI's models | -| promptsource | For using PromptSource prompts | -| sentencepiece | For using the sentencepiece tokenizer | -| testing | For running library test suite | -| vllm | For loading models with vLLM | -| zeno | For visualizing results with Zeno | -|---------------|---------------------------------------| -| all | Loads all extras (not recommended) | +We also provide a number of optional dependencies for extended functionality. A detailed table is available at the end of this document. ## Basic Usage @@ -109,33 +91,48 @@ The full list of supported arguments are provided [here](./docs/interface.md), a #### Multi-GPU Evaluation with Hugging Face `accelerate` -To parallelize evaluation of HuggingFace models across multiple GPUs, we leverage the [accelerate 🚀](https://github.com/huggingface/accelerate) library as follows: +We support two main ways of using Hugging Face's [accelerate 🚀](https://github.com/huggingface/accelerate) library for multi-GPU evaluation. + +To perform *data-parallel evaluation* (where each GPU loads a **separate full copy** of the model), we leverage the `accelerate` launcher as follows: ``` accelerate launch -m lm_eval --model hf \ --tasks lambada_openai,arc_easy \ --batch_size 16 ``` +(or via `accelerate launch --no-python lm_eval`). + +For cases where your model can fit on a single GPU, this allows you to evaluate on K GPUs K times faster than on one. + +**WARNING**: This setup does not work with FSDP model sharding, so in `accelerate config` FSDP must be disabled, or the NO_SHARD FSDP option must be used. -This will perform *data-parallel evaluation*: that is, placing a **single full copy** of your model onto each available GPU and *splitting batches across GPUs* to evaluate on K GPUs K times faster than on one. +The second way of using `accelerate` for multi-GPU evaluation is when your model is *too large to fit on a single GPU.* + +In this setting, run the library *outside of the `accelerate` launcher*, but passing `parallelize=True` to `--model_args` as follows: + +``` +lm_eval --model hf \ + --tasks lambada_openai,arc_easy \ + --model_args parallelize=True \ + --batch_size 16 +``` -If your model is *is too large to be run on a single one of your GPUs* then you can use `accelerate` with Fully Sharded Data Parallel (FSDP) that splits the weights of the model across your data parallel ranks. To enable this, ensure you select `YES` when asked ```Do you want to use FullyShardedDataParallel?``` when running `accelerate config`. To enable memory-efficient loading, select `YES` when asked `Do you want each individually wrapped FSDP unit to broadcast module parameters from rank 0 at the start?`. This will ensure only the rank 0 process loads the model and then broadcasts the parameters to the other ranks instead of having each rank load all parameters which can lead to large RAM usage spikes around the start of the script that may cause errors. +This means that your model's weights will be split across all available GPUs. -To pass even more advanced keyword arguments to `accelerate`, we allow for the following arguments as well: +For more advanced users or even larger models, we allow for the following arguments when `parallelize=True` as well: - `device_map_option`: How to split model weights across available GPUs. defaults to "auto". - `max_memory_per_gpu`: the max GPU memory to use per GPU in loading the model. - `max_cpu_memory`: the max amount of CPU memory to use when offloading the model weights to RAM. - `offload_folder`: a folder where model weights will be offloaded to disk if needed. -To use `accelerate` with the `lm-eval` command, use -``` -accelerate launch --no_python lm-eval --model ... -``` +These two options (`accelerate launch` and `parallelize=True`) are mutually exclusive. + +**Note: we do not currently support multi-node evaluations natively, and advise using either an externally hosted server to run inference requests against, or creating a custom integration with your distributed framework [as is done for the GPT-NeoX library](https://github.com/EleutherAI/gpt-neox/blob/main/eval_tasks/eval_adapter.py).** ### Tensor + Data Parallel and Optimized Inference with `vLLM` -We also support vLLM for faster inference on [supported model types](https://docs.vllm.ai/en/latest/models/supported_models.html). For single-GPU or multi-GPU — tensor parallel, data parallel, or a combination of both — inference, for example: +We also support vLLM for faster inference on [supported model types](https://docs.vllm.ai/en/latest/models/supported_models.html), especially faster when splitting a model across multiple GPUs. For single-GPU or multi-GPU — tensor parallel, data parallel, or a combination of both — inference, for example: ```bash lm_eval --model vllm \ @@ -160,7 +157,7 @@ lm_eval --model openai-completions \ --tasks lambada_openai,hellaswag ``` -We also support using your own local inference server with an implemented version of the OpenAI ChatCompletions endpoint and passing trained HuggingFace artifacts and tokenizers. +We also support using your own local inference server with servers that mirror the OpenAI Completions and ChatCompletions APIs. ```bash lm_eval --model local-chat-completions --tasks gsm8k --model_args model=facebook/opt-125m,base_url=http://{yourip}:8000/v1 @@ -169,7 +166,7 @@ Note that for externally hosted models, configs such as `--device` and `--batch_ | API or Inference Server | Implemented? | `--model ` name | Models supported: | Request Types: | |---------------------------------------------------------------------------------------------------------------------------|---------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------------------------------------------|------------------------------------------------------------| -| OpenAI Completions | :heavy_check_mark: | `openai-completions` | up to `code-davinci-002` | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| OpenAI Completions | :heavy_check_mark: | `openai-completions`, `local-completions` | All OpenAI Completions API models | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | | OpenAI ChatCompletions | :heavy_check_mark: | `openai-chat-completions`, `local-chat-completions` | [All ChatCompletions API models](https://platform.openai.com/docs/guides/gpt) | `generate_until` (no logprobs) | | Anthropic | :heavy_check_mark: | `anthropic` | [Supported Anthropic Engines](https://docs.anthropic.com/claude/reference/selecting-a-model) | `generate_until` (no logprobs) | | Textsynth | :heavy_check_mark: | `textsynth` | [All supported engines](https://textsynth.com/documentation.html#engines) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | @@ -177,15 +174,23 @@ Note that for externally hosted models, configs such as `--device` and `--batch_ | [Llama.cpp](https://github.com/ggerganov/llama.cpp) (via [llama-cpp-python](https://github.com/abetlen/llama-cpp-python)) | :heavy_check_mark: | `gguf`, `ggml` | [All models supported by llama.cpp](https://github.com/ggerganov/llama.cpp) | `generate_until`, `loglikelihood`, (perplexity evaluation not yet implemented) | | vLLM | :heavy_check_mark: | `vllm` | [Most HF Causal Language Models](https://docs.vllm.ai/en/latest/models/supported_models.html) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | | Mamba | :heavy_check_mark: | `mamba_ssm` | [Mamba architecture Language Models via the `mamba_ssm` package](https://huggingface.co/state-spaces) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | -| Your local inference server! | :heavy_check_mark: | `local-chat-completions` (using `openai-chat-completions` model type) | Any server address that accepts GET requests using HF models and mirror's OpenAI's ChatCompletions interface | `generate_until` | | ... | +| Huggingface Optimum (Causal LMs) | ✔️ | `openvino` | Any decoder-only AutoModelForCausalLM converted with Huggingface Optimum into OpenVINO™ Intermediate Representation (IR) format | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | ... | +| Neuron via AWS Inf2 (Causal LMs) | ✔️ | `neuronx` | Any decoder-only AutoModelForCausalLM supported to run on [huggingface-ami image for inferentia2](https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | ... | +| Your local inference server! | :heavy_check_mark: | `local-completions` or `local-chat-completions` (using `openai-chat-completions` model type) | Any server address that accepts GET requests using HF models and mirror's OpenAI's Completions or ChatCompletions interface | `generate_until` | | ... | -It is on our roadmap to create task variants designed to enable models which do not serve logprobs/loglikelihoods to be compared with generation performance of open-source models. +Models which do not supply logits or logprobs can be used with tasks of type `generate_until` only, while local models, or APIs that supply logprobs/logits of their prompts, can be run on all task types: `generate_until`, `loglikelihood`, `loglikelihood_rolling`, and `multiple_choice`. + +For more information on the different task `output_types` and model request types, see [our documentation](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/model_guide.md#interface). ### Other Frameworks A number of other libraries contain scripts for calling the eval harness through their library. These include [GPT-NeoX](https://github.com/EleutherAI/gpt-neox/blob/main/eval_tasks/eval_adapter.py), [Megatron-DeepSpeed](https://github.com/microsoft/Megatron-DeepSpeed/blob/main/examples/MoE/readme_evalharness.md), and [mesh-transformer-jax](https://github.com/kingoflolz/mesh-transformer-jax/blob/master/eval_harness.py). +To create your own custom integration you can follow instructions from [this tutorial](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md#external-library-usage). + ### Additional Features +> [!Note] +> For tasks unsuitable for direct evaluation — either due risks associated with executing untrusted code or complexities in the evaluation process — the `--predict_only` flag is available to obtain decoded generations for post-hoc evaluation. If you have a Metal compatible Mac, you can run the eval harness using the MPS back-end by replacing `--device cuda:0` with `--device mps` (requires PyTorch version 2.1 or higher). @@ -193,7 +198,7 @@ If you have a Metal compatible Mac, you can run the eval harness using the MPS b > You can inspect what the LM inputs look like by running the following command: > ```bash > python write_out.py \ -> --tasks all_tasks \ +> --tasks \ > --num_fewshot 5 \ > --num_examples 10 \ > --output_base_path /path/to/output/folder @@ -219,11 +224,11 @@ lm_eval --model hf \ --device cuda:0 ``` -[GPTQ](https://github.com/PanQiWei/AutoGPTQ) quantized models can be loaded by specifying their file names in `,gptq=NAME` (or `,gptq=True` for default names) in the `model_args` argument: +[GPTQ](https://github.com/PanQiWei/AutoGPTQ) quantized models can be loaded by specifying their file names in `,autogptq=NAME` (or `,autogptq=True` for default names) in the `model_args` argument: ```bash lm_eval --model hf \ - --model_args pretrained=model-name-or-path,gptq=model.safetensors,gptq_use_triton=True \ + --model_args pretrained=model-name-or-path,autogptq=model.safetensors,gptq_use_triton=True \ --tasks hellaswag ``` @@ -235,6 +240,9 @@ Additionally, one can provide a directory with `--use_cache` to cache the result For a full list of supported arguments, check out the [interface](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md) guide in our documentation! +> [!Tip] +> Running lm-evaluation-harness as an external library and can't find (almost) any tasks available? Run `lm_eval.tasks.initialize_tasks()` to load the library's stock tasks before calling `lm_eval.evaluate()` or `lm_eval.simple_evaluate()` ! + ## Visualizing Results You can use [Zeno](https://zenoml.com) to visualize the results of your eval harness runs. @@ -298,13 +306,41 @@ We try to prioritize agreement with the procedures used by other groups to decre The best way to get support is to open an issue on this repo or join the [EleutherAI Discord server](https://discord.gg/eleutherai). The `#lm-thunderdome` channel is dedicated to developing this project and the `#release-discussion` channel is for receiving support for our releases. If you've used the library and have had a positive (or negative) experience, we'd love to hear from you! +## Optional Extras +Extras dependencies can be installed via `pip install -e ".[NAME]"` + +| Name | Use | +|---------------|---------------------------------------| +| anthropic | For using Anthropic's models | +| dev | For linting PRs and contributions | +| gptq | For loading models with GPTQ | +| hf_transfer | For speeding up HF Hub file downloads | +| ifeval | For running the IFEval task | +| neuronx | For running on AWS inf2 instances | +| mamba | For loading Mamba SSM models | +| math | For running math task answer checking | +| multilingual | For multilingual tokenizers | +| openai | For using OpenAI's models | +| optimum | For running Intel OpenVINO models | +| promptsource | For using PromptSource prompts | +| sentencepiece | For using the sentencepiece tokenizer | +| testing | For running library test suite | +| vllm | For loading models with vLLM | +| zeno | For visualizing results with Zeno | +|---------------|---------------------------------------| +| all | Loads all extras (not recommended) | + ## Cite as ``` -@article{gao2021framework, - title={A framework for few-shot language model evaluation}, - author={Gao, Leo and Tow, Jonathan and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and McDonell, Kyle and Muennighoff, Niklas and others}, - journal={Version v0. 0.1. Sept}, - year={2021} +@misc{eval-harness, + author = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy}, + title = {A framework for few-shot language model evaluation}, + month = 12, + year = 2023, + publisher = {Zenodo}, + version = {v0.4.0}, + doi = {10.5281/zenodo.10256836}, + url = {https://zenodo.org/records/10256836} } ``` diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 0000000000..671f819cd6 --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,81 @@ +# Contributing to LM Evaluation Harness + +Welcome and thank you for your interest in the LM Evaluation Harness! We welcome contributions and feedback and appreciate your time spent with our library, and hope you find it useful! + +We intend LM Evaluation Harness to be a broadly useful and + +## Important Resources + +There are several places information about LM Evaluation Harness is located: + +- Our [documentation pages](https://github.com/EleutherAI/lm-evaluation-harness/tree/main/docs) +- We occasionally use [GitHub Milestones](https://github.com/EleutherAI/lm-evaluation-harness/milestones) to track progress toward specific near-term version releases. +- We maintain a [Project Board](https://github.com/orgs/EleutherAI/projects/25) for tracking current work items and PRs, and for future roadmap items or feature requests. +- Further discussion and support conversations are located in the #lm-thunderdome channel of the [EleutherAI discord](discord.gg/eleutherai). + +## Code Style + +LM Evaluation Harness uses [ruff](https://github.com/astral-sh/ruff) for linting via [pre-commit](https://pre-commit.com/). + +You can install linters and dev tools via + +```pip install lm_eval[dev]``` + +Then, run + +```pre-commit install``` + +in order to ensure linters and other checks will be run upon committing. + +## Testing + +We use [pytest](https://docs.pytest.org/en/latest/) for running unit tests. All library unit tests can be run via: + +``` +python -m pytest --ignore=tests/tests_master --ignore=tests/extra +``` + +## Contributor License Agreement + +We ask that new contributors agree to a Contributor License Agreement affirming that EleutherAI has the rights to use your contribution to our library. +First-time pull requests will have a reply added by @CLAassistant containing instructions for how to confirm this, and we require it before merging your PR. + + +## Contribution Best Practices + +We recommend a few best practices to make your contributions or reported errors easier to assist with. + +**For Pull Requests:** +- PRs should be titled descriptively, and be opened with a brief description of the scope and intent of the new contribution. +- New features should have appropriate documentation added alongside them. +- Aim for code maintainability, and minimize code copying. +- If opening a task, try to share test results on the task using a publicly-available model, and if any public results are available on the task, compare to them. + +**For Feature Requests:** +- Provide a short paragraph's worth of description. What is the feature you are requesting? What is its motivation, and an example use case of it? How does this differ from what is currently supported? + +**For Bug Reports**: +- Provide a short description of the bug. +- Provide a *reproducible example*--what is the command you run with our library that results in this error? Have you tried any other steps to resolve it? +- Provide a *full error traceback* of the error that occurs, if applicable. A one-line error message or small screenshot snippet is unhelpful without the surrounding context. +- Note what version of the codebase you are using, and any specifics of your environment and setup that may be relevant. + +**For Requesting New Tasks**: +- Provide a 1-2 sentence description of what the task is and what it evaluates. +- Provide a link to the paper introducing the task. +- Provide a link to where the dataset can be found. +- Provide a link to a paper containing results on an open-source model on the task, for use in comparisons and implementation validation. +- If applicable, link to any codebase that has implemented the task (especially the original publication's codebase, if existent). + +## How Can I Get Involved? + +To quickly get started, we maintain a list of good first issues, which can be found [on our project board](https://github.com/orgs/EleutherAI/projects/25/views/8) or by [filtering GH Issues](https://github.com/EleutherAI/lm-evaluation-harness/issues?q=is%3Aopen+label%3A%22good+first+issue%22+label%3A%22help+wanted%22). These are typically smaller code changes or self-contained features which can be added without extensive familiarity with library internals, and we recommend new contributors consider taking a stab at one of these first if they are feeling uncertain where to begin. + +There are a number of distinct ways to contribute to LM Evaluation Harness, and all are extremely helpful! A sampling of ways to contribute include: +- **Implementing and verifying new evaluation tasks**: Is there a task you'd like to see LM Evaluation Harness support? Consider opening an issue requesting it, or helping add it! Verifying and cross-checking task implementations with their original versions is also a very valuable form of assistance in ensuring standardized evaluation. +- **Improving documentation** - Improvements to the documentation, or noting pain points / gaps in documentation, are helpful in order for us to improve the user experience of the library and clarity + coverage of documentation. +- **Testing and devops** - We are very grateful for any assistance in adding tests for the library that can be run for new PRs, and other devops workflows. +- **Adding new modeling / inference library integrations** - We hope to support a broad range of commonly-used inference libraries popular among the community, and welcome PRs for new integrations, so long as they are documented properly and maintainable. +- **Proposing or Contributing New Features** - We want LM Evaluation Harness to support a broad range of evaluation usecases. If you have a feature that is not currently supported but desired, feel free to open an issue describing the feature and, if applicable, how you intend to implement it. We would be happy to give feedback on the cleanest way to implement new functionalities and are happy to coordinate with interested contributors via GH discussions or via discord. + +We hope that this has been helpful, and appreciate your interest in contributing! Further questions can be directed to [our Discord](discord.gg/eleutherai). diff --git a/docs/interface.md b/docs/interface.md index 02b3607edb..72ae59e188 100644 --- a/docs/interface.md +++ b/docs/interface.md @@ -44,6 +44,8 @@ This mode supports a number of command-line arguments, the details of which can * `--include_path` : Accepts a path to a folder. If passed, then all YAML files containing `lm-eval`` compatible task configurations will be added to the task registry as available tasks. Used for when one is writing config files for their own task in a folder other than `lm_eval/tasks/` +* `--predict_only`: Generates the model outputs without computing metrics. Use with `--log_samples` to retrieve decoded results. + ## External Library Usage We also support using the library's external API for use within model training loops or other scripts. @@ -59,14 +61,25 @@ import lm_eval my_model = initialize_my_model() # create your model (could be running finetuning with some custom modeling code) ... -lm_obj = Your_LM(model=my_model, batch_size=16) # instantiate an LM subclass that takes your initialized model and can run `Your_LM.loglikelihood()`, `Your_LM.loglikelihood_rolling()`, `Your_LM.generate_until()` - -lm_eval.tasks.initialize_tasks() # register all tasks from the `lm_eval/tasks` subdirectory. Alternatively, can call `lm_eval.tasks.include_path("path/to/my/custom/task/configs")` to only register a set of tasks in a separate directory. - +# instantiate an LM subclass that takes your initialized model and can run +# - `Your_LM.loglikelihood()` +# - `Your_LM.loglikelihood_rolling()` +# - `Your_LM.generate_until()` +lm_obj = Your_LM(model=my_model, batch_size=16) + +# indexes all tasks from the `lm_eval/tasks` subdirectory. +# Alternatively, you can set `TaskManager(include_path="path/to/my/custom/task/configs")` +# to include a set of tasks in a separate directory. +task_manager = lm_eval.tasks.TaskManager() + +# Setting `task_manager` to the one above is optional and should generally be done +# if you want to include tasks from paths other than ones in `lm_eval/tasks`. +# `simple_evaluate` will instantiate its own task_manager is the it is set to None here. results = lm_eval.simple_evaluate( # call simple_evaluate model=lm_obj, tasks=["taskname1", "taskname2"], num_fewshot=0, + task_manager=task_manager, ... ) ``` @@ -82,18 +95,49 @@ As a brief example usage of `evaluate()`: ```python import lm_eval -from my_tasks import MyTask1 # suppose you've defined a custom lm_eval.api.Task subclass in your own external codebase +# suppose you've defined a custom lm_eval.api.Task subclass in your own external codebase +from my_tasks import MyTask1 ... -my_model = initialize_my_model() # create your model (could be running finetuning with some custom modeling code) +# create your model (could be running finetuning with some custom modeling code) +my_model = initialize_my_model() ... -lm_obj = Your_LM(model=my_model, batch_size=16) # instantiate an LM subclass that takes your initialized model and can run `Your_LM.loglikelihood()`, `Your_LM.loglikelihood_rolling()`, `Your_LM.generate_until()` -lm_eval.tasks.initialize_tasks() # register all tasks from the `lm_eval/tasks` subdirectory. Alternatively, can call `lm_eval.tasks.include_path("path/to/my/custom/task/configs")` to only register a set of tasks in a separate directory. +# instantiate an LM subclass that takes your initialized model and can run +# - `Your_LM.loglikelihood()` +# - `Your_LM.loglikelihood_rolling()` +# - `Your_LM.generate_until()` +lm_obj = Your_LM(model=my_model, batch_size=16) + +# The task_manager indexes tasks including ones +# specified by the user through `include_path` +task_manager = lm_eval.tasks.TaskManager( + include_path="/path/to/custom/yaml" + ) + +# To get a task dict for `evaluate` +task_dict = lm_eval.tasks.get_task_dict( + [ + "mmlu", # A stock task + "my_custom_task", # A custom task + { + "task": ..., # A dict that configures a task + "doc_to_text": ..., + }, + MyTask1 # A task object from `lm_eval.task.Task` + ], + task_manager # A task manager that allows lm_eval to + # load the task during evaluation. + # If none is provided, `get_task_dict` + # will instantiated one itself, but this + # only includes the stock tasks so users + # will need to set this if including + # custom paths is required. + ) def evaluate( lm=lm_obj, - task_dict={"mytask1": MyTask1}, + task_dict=task_dict, ... ): ``` diff --git a/docs/new_task_guide.md b/docs/new_task_guide.md index b6be316284..0df7bb3b92 100644 --- a/docs/new_task_guide.md +++ b/docs/new_task_guide.md @@ -46,16 +46,6 @@ dataset_name: ... # the dataset configuration to use. Leave `null` if your datas dataset_kwargs: null # any extra keyword arguments that should be passed to the dataset constructor, e.g. `data_dir`. ``` ------------------------------- -**Tip:** To load a local dataset for evaluation, you can specify data files in the `dataset_kwargs` field, such as the following for JSON files: -``` -dataset_path: json -dataset_name: null -dataset_kwargs: - data_files: /path/to/my/json -``` -------------------------------- - Next, we'd like to tell our task what the dataset's train, validation, and test splits are named, if they exist: ```yaml @@ -99,6 +89,36 @@ Now, in our YAML config file we'll use the `!function` constructor, and tell the process_docs: !function utils.process_docs ``` +### Using Local Datasets + +To load a local dataset for evaluation, you can specify data files in the `dataset_kwargs` field, such as the following for JSON files: + +``` +dataset_path: json +dataset_name: null +dataset_kwargs: + data_files: /path/to/my/json +``` +Or with files already split into separate directories: + +``` +dataset_path: arrow +dataset_kwargs: + data_files: + train: /path/to/arrow/train/data-00000-of-00001.arrow + validation: /path/to/arrow/validation/data-00000-of-00001.arrow +``` + +Alternatively, if you have previously downloaded a dataset from huggingface hub (using `save_to_disk()`) and wish to use the local files, you will need to use `data_dir` under `dataset_kwargs` to point to where the directory is. + +``` +dataset_path: hellaswag +dataset_kwargs: + data_dir: hellaswag_local/ +``` + +You can also set `dataset_path` as a directory path in your local system. This will assume that there is a loading script with the same name as the directory. [See datasets docs](https://huggingface.co/docs/datasets/loading#local-loading-script). + ## Writing a Prompt Template The next thing we need to do is decide what format to use when presenting the data to the LM. This is our **prompt**, where we'll define both an input and output format. @@ -232,7 +252,7 @@ metric_list: ``` `aggregation` and `higher_is_better` can optionally be left out to default to the manually-set defaults if using a natively supported metric, otherwise it must be defined explicitly (for example, when using a custom metric implemented as a function). -For a full list of natively supported metrics and aggregation functions see `docs/advanced_task_guide.md`. All metrics supported in [HuggingFace Evaluate](https://github.com/huggingface/evaluate/tree/main/metrics) can also be used, and will be loaded if a given metric name is not one natively supported in `lm-eval` or `hf_evaluate` is set to `true`. +For a full list of natively supported metrics and aggregation functions see [`docs/task_guide.md`](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/task_guide.md). All metrics supported in [HuggingFace Evaluate](https://github.com/huggingface/evaluate/tree/main/metrics) can also be used, and will be loaded if a given metric name is not one natively supported in `lm-eval` or `hf_evaluate` is set to `true`. ### Optional, More Advanced Setup @@ -245,7 +265,7 @@ As a heuristic check: * Do you expect to compute metrics after applying multiple such processing steps on your model outputs? * Does your task rely on metrics that need a custom implementation? -For more detail on the task system and advanced features, see `docs/advanced_task_guide.md` . If none of the above sound like they apply to your task, it's time to continue onto checking your task performance! +For more detail on the task system and advanced features, see [`docs/task_guide.md`](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/task_guide.md) . If none of the above sound like they apply to your task, it's time to continue onto checking your task performance! ### Task name + groups (registering a task) @@ -270,17 +290,80 @@ This will add your task to the `group1` and `group2` groups, enabling people to If your task is not in the `lm_eval/tasks` folder, you'll need to tell the Eval Harness where to look for YAML files. -You can do this via adding the Python snippet +You can do this via the `--include_path` argument in `__main__.py`. This command will be used to initialize the `TaskManager` object which you can also use for your custom scripts. ```python -from lm_eval.tasks import include_task_folder -include_task_folder("/path/to/yaml/parent/folder") +task_manager = TaskManager(args.verbosity, include_path=args.include_path) ``` -to the top of any Python file that is run or imported when performing evaluation, such as `\_\_main\_\_.py`. Passing `--tasks /path/to/yaml/file` is also accepted. +### Advanced Group Configs + +You can make more complete group config while also tailoring parameters for individual tasks. + +For example, let's build a config for evaluating MMLU and a few natural language inference tasks. For MMLU, we can write the name for the benchmark as a subtask written under `task`. You can configure the parameters such as `num_fewshot`. If the task being configured is a group such as `mmlu` or `super_glue`, the parameter set will be applied to all of the subtasks. + +```yaml +group: nli_and_mmlu +task: + - group: nli_tasks + task: + - cb + - anli_r1 + - rte + - task: mmlu + num_fewshot: 2 +``` +It's also important to note how you can basically insert a group config as a task. Here, to make a group of natural language inference tasks, you simply write like how you would normally write a group config but this time place that as part of a task list under the main group being built. + +### Duplicate Tasks in Group Configs + +There might be cases where you might want to evaluate prompts and how models perform over prompt variations. You can list an existing task (In the example below, `anli_r1`) which varying `doc_to_text` implementation. To differentiate from each variation, we can utilize `task_alias`. LM-Eval will recognize that there are multiple variations of the same tasks and differentiate them. +```yaml +group: flan_held_in +group_alias: Flan (Held-In) +task: + # ANLI R1 + - group: anli_r1_flan + group_alias: ANLI R1 + task: + - task: anli_r1 + task_alias: prompt-0 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nChoose your answer ..." + ... + - task: anli_r1 + task_alias: prompt-1 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nBased on ..." + ... +``` + +### Configuring python classes + +There can occasions when yaml-based tasks cannot accommodate how a task is handled. LM-Eval supports the manually implementing tasks as was previously done before `0.4.x`. To register the task, you can simply make a yaml with the name of the task in `task` and the class object in `class` using the `!function` prefix. + +```yaml +task: squadv2 +class: !function task.SQuAD2 +``` + +This also applies to building group configurations with subtasks that are python classes. + +```yaml +group: scrolls +task: + - task: scrolls_qasper + class: !function task.Qasper + - task: scrolls_quality + class: !function task.QuALITY + - task: scrolls_narrativeqa + class: !function task.NarrativeQA + ... +``` + ## Beautifying Table Display To avoid conflict, each task needs to be registered with a unique name. Because of this, slight variations of task are still counted as unique tasks and need to be named uniquely. This could be done by appending an additional naming that may refer to the variation such as in MMLU where the template used to evaluated for flan are differentiated from the default by the prefix `mmlu_flan_*`. Printing the full task names can easily clutter the results table at the end of the evaluation especially when you have a long list of tasks or are using a benchmark that comprises of many tasks. To make it more legible, you can use `task_alias` and `group_alias` to provide an alternative task name and group name that will be printed. diff --git a/docs/task_guide.md b/docs/task_guide.md index 4a0fab5af7..8665ffa13c 100644 --- a/docs/task_guide.md +++ b/docs/task_guide.md @@ -4,7 +4,7 @@ The `lm-evaluation-harness` is meant to be an extensible and flexible framework These YAML configuration files, along with the current codebase commit hash, are intended to be shareable such that providing the YAML config enables another researcher to precisely replicate the evaluation setup used by another, in the case that the prompt or setup differs from standard `lm-eval` task implementations. -While adding a standard evaluation task on a new dataset can be occasionally as simple as swapping out a Hugging Face dataset path in an existing file, more specialized evaluation setups. Here we'll provide a crash course on the more advanced logic implementable in YAML form available to users. +While adding a standard evaluation task on a new dataset can be occasionally as simple as swapping out a Hugging Face dataset path in an existing file, more specialized evaluation setups also exist. Here we'll provide a crash course on the more advanced logic implementable in YAML form available to users. If your intended task relies on features beyond what are described in this guide, we'd love to hear about it! Feel free to open an issue describing the scenario on Github, create a PR to the project with a proposed implementation, or ask in the `#lm-thunderdome` channel on the EleutherAI discord. @@ -50,7 +50,7 @@ Scoring details: - **doc_to_decontamination_query** (`str`, *optional*) — Query for decontamination if `should_decontaminate` is True. If `should_decontaminate` is True but `doc_to_decontamination_query` is `None`, `doc_to_decontamination_query` will follow `doc_to_text`. Other: -- **metadata** (`Union[str, list]`, *optional*) — An optional field where arbitrary metadata can be passed. A good example would be `version` that is used to denote the version of the yaml config. +- **metadata** (`dict`, *optional*) — An optional field where arbitrary metadata can be passed. Most tasks should include a `version` key in this field that is used to denote the version of the yaml config. Other special metadata keys are: `num_fewshot`, to override the printed `n-shot` table column for a task. ## Filters @@ -301,6 +301,23 @@ task: - hendrycksTest* ``` +It is also possible to list an existing task in your benchmark configuration with some adjustments. For example, a few tasks from mmlu is included `multimedqa`. There, the `task_alias` and `group_alias` (See [here](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/new_task_guide.md#beautifying-table-display) for more details) are modified to suit the benchmark. + +```yaml +group: multimedqa +task: + - pubmedqa + - medmcqa + - medqa_4options + - task: mmlu_anatomy + task_alias: "anatomy (mmlu)" + group_alias: null + - task: mmlu_clinical_knowledge + task_alias: "clinical_knowledge (mmlu)" + group_alias: null + ... +``` + Alternatively, benchmarks can have tasks that are customizable for each task. They can be defined like how a yaml task is usually set. ```yaml @@ -363,4 +380,4 @@ task: ignore_punctuation: true ``` -Calling the benchmark is done the same way we would call any task with `--tasks`. Benchmarks can be added in `lm_eval/benchmarks/` +Calling the benchmark is done the same way we would call any task with `--tasks`. Benchmarks can be added in `lm_eval/tasks/benchmarks/` diff --git a/lm_eval/__main__.py b/lm_eval/__main__.py index 37fdabc6df..40e4237a30 100644 --- a/lm_eval/__main__.py +++ b/lm_eval/__main__.py @@ -10,8 +10,7 @@ import numpy as np from lm_eval import evaluator, utils -from lm_eval.api.registry import ALL_TASKS -from lm_eval.tasks import include_path, initialize_tasks +from lm_eval.tasks import TaskManager, include_path, initialize_tasks from lm_eval.utils import make_table @@ -143,6 +142,13 @@ def parse_eval_args() -> argparse.Namespace: metavar="CRITICAL|ERROR|WARNING|INFO|DEBUG", help="Controls the reported logging error level. Set to DEBUG when testing + adding new task configurations for comprehensive log output.", ) + parser.add_argument( + "--predict_only", + "-x", + action="store_true", + default=False, + help="Use with --log_samples. Only model outputs will be saved and metrics will not be evaluated.", + ) return parser.parse_args() @@ -156,7 +162,13 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: eval_logger.info(f"Verbosity set to {args.verbosity}") os.environ["TOKENIZERS_PARALLELISM"] = "false" + if args.predict_only: + args.log_samples = True + if (args.log_samples or args.predict_only) and not args.output_path: + assert args.output_path, "Specify --output_path" + initialize_tasks(args.verbosity) + task_manager = TaskManager(args.verbosity, include_path=args.include_path) if args.limit: eval_logger.warning( @@ -168,10 +180,11 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: include_path(args.include_path) if args.tasks is None: - task_names = ALL_TASKS + eval_logger.error("Need to specify task to evaluate.") + sys.exit() elif args.tasks == "list": eval_logger.info( - "Available Tasks:\n - {}".format("\n - ".join(sorted(ALL_TASKS))) + "Available Tasks:\n - {}".format("\n - ".join(task_manager.all_tasks)) ) sys.exit() else: @@ -184,16 +197,14 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: config = utils.load_yaml_config(yaml_file) task_names.append(config) else: - tasks_list = args.tasks.split(",") - task_names = utils.pattern_match(tasks_list, ALL_TASKS) - for task in [task for task in tasks_list if task not in task_names]: + task_list = args.tasks.split(",") + task_names = task_manager.match_tasks(task_list) + for task in [task for task in task_list if task not in task_names]: if os.path.isfile(task): config = utils.load_yaml_config(task) task_names.append(config) task_missing = [ - task - for task in tasks_list - if task not in task_names and "*" not in task + task for task in task_list if task not in task_names and "*" not in task ] # we don't want errors if a wildcard ("*") task name was used if task_missing: @@ -223,10 +234,9 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: else: path.mkdir(parents=True, exist_ok=True) output_path_file = path.joinpath("results.json") - elif args.log_samples and not args.output_path: - assert args.output_path, "Specify --output_path" eval_logger.info(f"Selected Tasks: {task_names}") + eval_logger.info("Loading selected tasks...") results = evaluator.simple_evaluate( model=args.model, @@ -243,6 +253,8 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: write_out=args.write_out, log_samples=args.log_samples, gen_kwargs=args.gen_kwargs, + task_manager=task_manager, + predict_only=args.predict_only, ) if results is not None: @@ -257,7 +269,7 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: batch_sizes = ",".join(map(str, results["config"]["batch_sizes"])) if args.output_path: - output_path_file.open("w").write(dumped) + output_path_file.open("w", encoding="utf-8").write(dumped) if args.log_samples: for task_name, config in results["configs"].items(): diff --git a/lm_eval/api/filter.py b/lm_eval/api/filter.py index bc26a1a637..8d9db68217 100644 --- a/lm_eval/api/filter.py +++ b/lm_eval/api/filter.py @@ -1,12 +1,11 @@ +from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import List - -from datasets import Dataset +from typing import Callable, Iterable, List, Union from lm_eval.api.instance import Instance -class Filter: +class Filter(ABC): """ Filter classes operate on a per-task level. They take all model outputs (`instance.resps` for all `task.instances`) @@ -15,12 +14,13 @@ class Filter: """ - def __init__(self, *args, **kwargs) -> None: + def __init__(self, **kwargs) -> None: """ Can define custom behavior here, if an individual instantiation of a Filter class should have state. """ - def apply(self, resps, docs): + @abstractmethod + def apply(self, resps: Union[List, Iterable], docs: List[dict]) -> Iterable: """ Defines the operation to perform on a list of the `inst.resps` properties of `Instance` objects. Should return the list of (filtered) response lists *in the same order as they were input*, e.g. @@ -40,15 +40,15 @@ class FilterEnsemble: """ name: str - filters: List[Filter] + filters: List[Callable[[], Filter]] + + def apply(self, instances: List[Instance]) -> None: + resps, docs = zip(*((inst.resps, inst.doc) for inst in instances)) + resps, docs = list(resps), list(docs) - def apply(self, instances: List[Instance], docs: List[Dataset]) -> None: - resps = [ - inst.resps for inst in instances - ] # operate just on the model responses for f in self.filters: # apply filters in sequence - resps = f.apply(resps, docs) + resps = f().apply(resps, docs) # add the end results after filtering to filtered_requests of their respective source instances. # has key `self.name`: each FilterEnsemble applied in a given run should use a different name. diff --git a/lm_eval/api/instance.py b/lm_eval/api/instance.py index 7d3c23aa11..b5913a3c63 100644 --- a/lm_eval/api/instance.py +++ b/lm_eval/api/instance.py @@ -4,7 +4,12 @@ @dataclass class Instance: - request_type: Literal["loglikelihood", "loglikelihood_rolling", "generate_until"] + request_type: Literal[ + "loglikelihood", + "loglikelihood_rolling", + "generate_until", + "multiple_choice", + ] doc: dict arguments: tuple idx: int diff --git a/lm_eval/api/metrics.py b/lm_eval/api/metrics.py index 85a944c888..5b71a9ee94 100644 --- a/lm_eval/api/metrics.py +++ b/lm_eval/api/metrics.py @@ -2,6 +2,7 @@ import math import random from collections.abc import Iterable +from typing import List import evaluate import numpy as np @@ -15,6 +16,11 @@ # Register Aggregations First +@register_aggregation("bypass") +def bypass_agg(arr): + return 999 + + @register_aggregation("mean") def mean(arr): return sum(arr) / len(arr) @@ -207,6 +213,16 @@ def mean_stderr(arr): return sample_stddev(arr) / math.sqrt(len(arr)) +@register_metric( + metric="bypass", + higher_is_better=True, + output_type=["loglikelihood", "multiple_choice", "generate_until"], + aggregation="bypass", +) +def bypass(items): + return None + + @register_metric( metric="mcc", higher_is_better=True, @@ -410,3 +426,64 @@ def stderr_for_metric(metric, bootstrap_iters): stderr = {mean: mean_stderr, acc_all: acc_all_stderr} return stderr.get(metric, None) + + +def pooled_sample_stderr(stderrs: List[float], sizes: List[int]): + # Used to aggregate bootstrapped stderrs across subtasks in a group, + # when we are weighting by the size of each subtask. + # + + assert len(stderrs) == len(sizes) + + # formula source: https://en.wikipedia.org/wiki/Pooled_variance + # this empirically matches running `stderr_for_metric` on all instances + # from the subtasks concatenated with each other. + pooled_sample_var = ( + sum([(size - 1) * stderr**2 for size, stderr in zip(sizes, stderrs)]) + ) / (sum(sizes) - len(sizes)) + + return np.sqrt(pooled_sample_var) + + +def combined_sample_stderr(stderrs: List[float], sizes: List[int], metrics=None): + assert ( + metrics is not None + ), "Need to pass a list of each subtask's metric for this stderr aggregation" + assert len(stderrs) == len(sizes) and len(sizes) == len(metrics) + + # See https://github.com/EleutherAI/lm-evaluation-harness/pull/1390 for more documentation. + # This formula depends on sample means. + # removed because it seems to give erroneously huge stderrs for groupings of tasks + # and does not seem to match up with bootstrap-calculated stderrs for groups. + + ### don't use this unless a statistician has told you it's the right thing to do ### + + # accumulators: we'll aggregate pairwise N - 1 times + variance = stderrs[0] ** 2 + curr_size = sizes[0] + curr_score = metrics[0] + + for stderr, size, score in zip(stderrs[1:], sizes[1:], metrics[1:]): + curr_score = ((curr_score * curr_size) + (score * size)) / ( + curr_size + size + ) # NOTE: this assumes our aggregation fn is "mean" + + variance = ((curr_size - 1) * variance + (size - 1) * (stderr**2)) / ( + curr_size + size - 1 + ) + curr_size * size / ((curr_size + size) * (curr_size + size - 1)) * ( + curr_score - score + ) ** 2 + + return np.sqrt(variance) + + +def aggregate_subtask_metrics(metrics, sizes, weight_by_size=True): + # A helper function that is used to aggregate + # subtask scores cross-task. + # TODO: does not hold for non-mean aggregations + if weight_by_size: + sizes = [1] * len(sizes) + + assert len(metrics) == len(sizes) + + return sum([metric * size for metric, size in zip(metrics, sizes)]) / sum(sizes) diff --git a/lm_eval/api/registry.py b/lm_eval/api/registry.py index 5fb9c011fc..3c4031195f 100644 --- a/lm_eval/api/registry.py +++ b/lm_eval/api/registry.py @@ -1,4 +1,5 @@ import logging +from typing import Callable, Dict import evaluate @@ -75,7 +76,7 @@ def decorate(fn): OUTPUT_TYPE_REGISTRY = {} METRIC_REGISTRY = {} METRIC_AGGREGATION_REGISTRY = {} -AGGREGATION_REGISTRY = {} +AGGREGATION_REGISTRY: Dict[str, Callable[[], Dict[str, Callable]]] = {} HIGHER_IS_BETTER_REGISTRY = {} DEFAULT_METRIC_REGISTRY = { @@ -118,7 +119,7 @@ def decorate(fn): return decorate -def get_metric(name, hf_evaluate_metric=False): +def get_metric(name: str, hf_evaluate_metric=False) -> Callable: if not hf_evaluate_metric: if name in METRIC_REGISTRY: return METRIC_REGISTRY[name] @@ -136,7 +137,7 @@ def get_metric(name, hf_evaluate_metric=False): ) -def register_aggregation(name): +def register_aggregation(name: str): def decorate(fn): assert ( name not in AGGREGATION_REGISTRY @@ -148,25 +149,21 @@ def decorate(fn): return decorate -def get_aggregation(name): +def get_aggregation(name: str) -> Callable[[], Dict[str, Callable]]: try: return AGGREGATION_REGISTRY[name] except KeyError: - eval_logger.warning( - "{} not a registered aggregation metric!".format(name), - ) + eval_logger.warning(f"{name} not a registered aggregation metric!") -def get_metric_aggregation(name): +def get_metric_aggregation(name: str) -> Callable[[], Dict[str, Callable]]: try: return METRIC_AGGREGATION_REGISTRY[name] except KeyError: - eval_logger.warning( - "{} metric is not assigned a default aggregation!".format(name), - ) + eval_logger.warning(f"{name} metric is not assigned a default aggregation!") -def is_higher_better(metric_name): +def is_higher_better(metric_name) -> bool: try: return HIGHER_IS_BETTER_REGISTRY[metric_name] except KeyError: diff --git a/lm_eval/api/task.py b/lm_eval/api/task.py index 6fe2c69800..7204a3d56a 100644 --- a/lm_eval/api/task.py +++ b/lm_eval/api/task.py @@ -5,6 +5,7 @@ import re from collections.abc import Callable from dataclasses import asdict, dataclass +from inspect import getsource from typing import Any, List, Literal, Tuple, Union import datasets @@ -37,7 +38,6 @@ "generate_until", ] - eval_logger = logging.getLogger("lm-eval") @@ -74,16 +74,18 @@ class TaskConfig(dict): num_fewshot: int = None # scoring options metric_list: list = None - output_type: str = "generate_until" + output_type: Literal[ + "loglikelihood", + "loglikelihood_rolling", + "generate_until", + "multiple_choice", + ] = "generate_until" generation_kwargs: dict = None repeats: int = 1 filter_list: Union[str, list] = None should_decontaminate: bool = False doc_to_decontamination_query: str = None - - metadata: Union[ - str, list - ] = None # by default, not used in the code. allows for users to pass arbitrary info to tasks + metadata: dict = None # by default, not used in the code. allows for users to pass arbitrary info to tasks def __post_init__(self) -> None: if self.generation_kwargs is not None: @@ -110,15 +112,13 @@ def __post_init__(self) -> None: "do_sample": False, } - # TODO: how to make TaskConfigs be de- and re-serializable, even when using the !function constructor? - def __getitem__(self, item): return getattr(self, item) def __setitem__(self, item, value): return setattr(self, item, value) - def to_dict(self): + def to_dict(self, keep_callable: bool = False) -> dict: """dumps the current config as a dictionary object, as a printable format. null fields will not be printed. Used for dumping results alongside full task configuration @@ -133,11 +133,34 @@ def to_dict(self): for k, v in list(cfg_dict.items()): if v is None: cfg_dict.pop(k) - elif isinstance(v, Callable): - # TODO: this should handle Promptsource template objects as a separate case? - cfg_dict[k] = str(v) + elif k == "metric_list": + for metric_dict in v: + for metric_key, metric_value in metric_dict.items(): + if callable(metric_value): + metric_dict[metric_key] = self.serialize_function( + metric_value, keep_callable=keep_callable + ) + cfg_dict[k] = v + elif callable(v): + cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable) return cfg_dict + def serialize_function( + self, value: Union[Callable, str], keep_callable=False + ) -> Union[Callable, str]: + """Serializes a given function or string. + + If 'keep_callable' is True, the original callable is returned. + Otherwise, attempts to return the source code of the callable using 'getsource'. + """ + if keep_callable: + return value + else: + try: + return getsource(value) + except (TypeError, OSError): + return str(value) + class Task(abc.ABC): """A task represents an entire benchmark including its dataset, problems, @@ -282,7 +305,7 @@ def fewshot_docs(self): return self.validation_docs() else: eval_logger.warning( - "has_training_docs and has_validation_docs are False" + f"[Task: {self.config.task}] has_training_docs and has_validation_docs are False" ", using test_docs as fewshot_docs but this is not recommended." ) return self.test_docs() @@ -334,7 +357,7 @@ def build_all_requests(self, limit=None, rank=None, world_size=None) -> None: else: assert False, f"Task dataset (path={self.DATASET_PATH}, name={self.DATASET_NAME}) must have valid or test docs!" - eval_logger.info(f"Building contexts for task on rank {rank}...") + eval_logger.info(f"Building contexts for {self.config.task} on rank {rank}...") instances = [] for doc_id, doc in utils.create_iterator( @@ -413,6 +436,9 @@ def higher_is_better(self): """ pass + def get_config(self, key: str) -> Any: + return getattr(self._config, key, None) + @classmethod def count_bytes(cls, doc): """Used for byte-level perplexity metrics in rolling loglikelihood""" @@ -485,23 +511,60 @@ def fewshot_context( return description + labeled_examples + example def apply_filters(self): + """Iterates over FilterEnsembles and applies them to instances""" if hasattr(self, "_filters"): for f in self._filters: - f.apply(self._instances, None) + f.apply(self._instances) else: eval_logger.warning("No filter defined, passing through instances") return self._instances def dump_config(self) -> dict: - """Returns a dictionary representing the task's config. - - :returns: str - The fewshot context. - """ + """Returns the config as a dictionary.""" # TODO: this should only return the overrides applied to a non-YAML task's configuration. # (num_fewshot) return self.config.to_dict() + def set_config(self, key: str, value: Any, update: bool = False) -> None: + """Set or update the configuration for a given key.""" + if key is None: + raise ValueError("Key must be provided.") + + if update: + current_value = getattr(self._config, key, {}) + if not isinstance(current_value, dict): + raise TypeError( + f"Expected a dict for key '{key}', got {type(current_value).__name__} instead." + ) + current_value.update(value) + else: + setattr(self._config, key, value) + + def override_metric(self, metric_name: str) -> None: + """ + Override the default metrics used for evaluation with custom metrics. + + Parameters: + - metric_name (str): The name of the custom metric to override. Should be registered in api.metrics. + """ + ( + self._metric_fn_list, + self._aggregation_list, + self._metric_fn_kwargs, + self._higher_is_better, + ) = ({}, {}, {}, {}) + self._metric_fn_list[metric_name] = get_metric(metric_name) + self._aggregation_list[metric_name] = get_metric_aggregation(metric_name) + self._higher_is_better[metric_name] = is_higher_better(metric_name) + self._metric_fn_kwargs[metric_name] = {} + if not isinstance(self, ConfigurableTask): + self.process_results = lambda x, y: {metric_name: get_metric(metric_name)} + self.aggregation = lambda: { + metric_name: get_metric_aggregation(metric_name) + } + setattr(self._config, "metric_list", [{"metric": metric_name}]) + setattr(self._config, "process_results", None) + class ConfigurableTask(Task): VERSION = "Yaml" @@ -598,7 +661,7 @@ def __init__( INV_AGG_REGISTRY = {v: k for k, v in AGGREGATION_REGISTRY.items()} metric_agg = get_metric_aggregation(metric_name) eval_logger.warning( - f"[Task: {self._config.task}] metric {metric_name} is defined, but aggregation is not. " + f"[Task: {self.config.task}] metric {metric_name} is defined, but aggregation is not. " f"using default " f"aggregation={INV_AGG_REGISTRY[metric_agg]}" ) @@ -610,7 +673,7 @@ def __init__( ] else: eval_logger.warning( - f"[Task: {self._config.task}] metric {metric_name} is defined, but higher_is_better is not. " + f"[Task: {self.config.task}] metric {metric_name} is defined, but higher_is_better is not. " f"using default " f"higher_is_better={is_higher_better(metric_name)}" ) @@ -623,16 +686,15 @@ def __init__( if self.config.filter_list is not None: self._filters = [] for filter_config in self.config.filter_list: - for filter_pipeline in filter_config: - filter_name = filter_config["name"] - filter_functions = filter_config["filter"] - components = [] - for function in filter_functions: - kwargs = { - key: function[key] for key in function if key != "function" - } - components.append([function["function"], kwargs]) - filter_pipeline = build_filter_ensemble(filter_name, components) + filter_name = filter_config["name"] + filter_functions = filter_config["filter"] + components = [] + for function in filter_functions: + kwargs = { + key: function[key] for key in function if key != "function" + } + components.append([function["function"], kwargs]) + filter_pipeline = build_filter_ensemble(filter_name, components) self._filters.append(filter_pipeline) else: self._filters = [build_filter_ensemble("none", [["take_first", None]])] @@ -808,9 +870,10 @@ def fewshot_context(self, doc, num_fewshot): return labeled_examples + str(example) def apply_filters(self): + """Iterates over FilterEnsembles and applies them to instances""" if hasattr(self, "_filters"): for f in self._filters: - f.apply(self._instances, self.task_docs) + f.apply(self._instances) else: eval_logger.warning("No filter defined, passing through instances") return self._instances @@ -1188,12 +1251,15 @@ def process_results(self, doc, results): return result_dict - def aggregation(self): + def aggregation(self) -> dict: return self._aggregation_list - def higher_is_better(self): + def higher_is_better(self) -> dict: return self._higher_is_better + def get_config(self, key: str) -> Any: + return getattr(self._config, key, None) + class MultipleChoiceTask(Task): OUTPUT_TYPE: str = "loglikelihood" diff --git a/lm_eval/decontamination/archiver.py b/lm_eval/decontamination/archiver.py index e6bff33f0c..fa8a715f78 100644 --- a/lm_eval/decontamination/archiver.py +++ b/lm_eval/decontamination/archiver.py @@ -30,7 +30,9 @@ def __init__(self, file_path: str, compression_level: int = 3) -> None: self.cctx = zstandard.ZstdCompressor(level=compression_level) self.compressor = self.cctx.stream_writer(self.fh) - def add_data(self, data, meta={}) -> None: + def add_data(self, data, meta=None) -> None: + if meta is None: + meta = {} self.compressor.write( json.dumps({"text": data, "meta": meta}, default=json_serial).encode( "UTF-8" @@ -108,7 +110,7 @@ def __init__(self, file_path) -> None: def read_tqdm(self, update_frequency: int = 10000): current_file_position = 0 line_counter = 0 - with open(self.file_path, "r") as fh, tqdm.tqdm( + with open(self.file_path, "r", encoding="utf-8") as fh, tqdm.tqdm( total=os.path.getsize(self.file_path), dynamic_ncols=True, unit="byte", diff --git a/lm_eval/decontamination/decontaminate.py b/lm_eval/decontamination/decontaminate.py index f5b4157c67..3874eb58be 100644 --- a/lm_eval/decontamination/decontaminate.py +++ b/lm_eval/decontamination/decontaminate.py @@ -38,7 +38,7 @@ def get_train_overlap(docs_by_task_set: dict, ngrams_path: str, limit: int) -> d # return get_train_overlap_stub(docs, ngrams_path, ngrams_n_size) info_dict_path = os.path.join(ngrams_path, "info.json") - info_dict = json.load(open(info_dict_path, "r")) + info_dict = json.load(open(info_dict_path, "r", encoding="utf-8")) ngrams_n_size = info_dict["ngram_size"] janitor = Janitor() diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index 5d277a6bf7..13da52cdf9 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -1,43 +1,45 @@ -import random -import itertools import collections - -import torch +import itertools +import logging +import random +from typing import Optional, Union import numpy as np +import torch -import lm_eval.api -import lm_eval.tasks -import lm_eval.models import lm_eval.api.metrics import lm_eval.api.registry - +import lm_eval.models +from lm_eval.tasks import TaskManager, get_task_dict from lm_eval.utils import ( + eval_logger, + get_git_commit_hash, positional_deprecated, run_task_tests, - get_git_commit_hash, simple_parse_args_string, - eval_logger, ) @positional_deprecated def simple_evaluate( model, - model_args=None, - tasks=[], - num_fewshot=None, - batch_size=None, - max_batch_size=None, - device=None, - use_cache=None, - limit=None, + model_args: Optional[str] = None, + tasks=None, + num_fewshot: Optional[int] = None, + batch_size: Optional[int] = None, + max_batch_size: Optional[int] = None, + device: Optional[str] = None, + use_cache: Optional[str] = None, + limit: Optional[Union[int, float]] = None, bootstrap_iters: int = 100000, check_integrity: bool = False, decontamination_ngrams_path=None, write_out: bool = False, log_samples: bool = True, gen_kwargs: str = None, + task_manager: TaskManager = None, + verbosity: str = "INFO", + predict_only: bool = False, ): """Instantiate and evaluate a model on a list of tasks. @@ -46,7 +48,7 @@ def simple_evaluate( :param model_args: Optional[str] String arguments for each model class, see LM.create_from_arg_string. Ignored if `model` argument is a LM object. - :param tasks: list[Union[str, Task]] + :param tasks: list[Union[str, dict, Task]] List of task names or Task objects. Task objects will be taken to have name task.EVAL_HARNESS_NAME if defined and type(task).__name__ otherwise. :param num_fewshot: int Number of examples in few-shot context @@ -71,6 +73,9 @@ def simple_evaluate( :param gen_kwargs: str String arguments for model generation Ignored for all tasks with loglikelihood output_type + :param predict_only: bool + If true only model outputs will be generated and returned. Metrics will not be evaluated + :return Dictionary of results """ @@ -80,6 +85,10 @@ def simple_evaluate( 1234 ) # TODO: this may affect training runs that are run with evaluation mid-run. + eval_logger.setLevel(getattr(logging, f"{verbosity}")) + + if tasks is None: + tasks = [] assert ( tasks != [] ), "No tasks specified, or no tasks found. Please verify the task names." @@ -87,7 +96,7 @@ def simple_evaluate( if gen_kwargs is not None: gen_kwargs = simple_parse_args_string(gen_kwargs) eval_logger.warning( - "generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks." + "generation_kwargs specified through cli, these settings will update set parameters in yaml tasks. Ensure 'do_sample=True' for non-greedy decoding!" ) if gen_kwargs == "": gen_kwargs = None @@ -119,30 +128,45 @@ def simple_evaluate( + ".db", ) - task_dict = lm_eval.tasks.get_task_dict(tasks) + if task_manager is None: + task_manager = TaskManager(verbosity) + + eval_logger.info( + "get_task_dict has been updated to accept an optional argument, `task_manager`" + "Read more here:https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md#external-library-usage" + ) + task_dict = get_task_dict(tasks, task_manager) for task_name in task_dict.keys(): task_obj = task_dict[task_name] - if type(task_obj) == tuple: - group, task_obj = task_obj + if isinstance(task_obj, tuple): + _, task_obj = task_obj if task_obj is None: continue - config = task_obj._config - if config["output_type"] == "generate_until" and gen_kwargs is not None: - config["generation_kwargs"].update(gen_kwargs) + if task_obj.get_config("output_type") == "generate_until": + if gen_kwargs is not None: + task_obj.set_config( + key="generation_kwargs", value=gen_kwargs, update=True + ) + + if predict_only: + log_samples = True + eval_logger.info( + f"Processing {task_name} in output-only mode. Metrics will not be calculated!" + ) + # we have to change the class properties post-hoc. This is pretty hacky. + task_obj.override_metric(metric_name="bypass") if num_fewshot is not None: - if config["num_fewshot"] == 0: + if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0: eval_logger.info( f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored." ) else: - default_num_fewshot = config["num_fewshot"] eval_logger.warning( f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}" ) - - task_obj._config["num_fewshot"] = num_fewshot + task_obj.set_config(key="num_fewshot", value=num_fewshot) if check_integrity: run_task_tests(task_list=tasks) @@ -155,14 +179,20 @@ def simple_evaluate( decontamination_ngrams_path=decontamination_ngrams_path, write_out=write_out, log_samples=log_samples, + verbosity=verbosity, ) if lm.rank == 0: + if isinstance(model, str): + model_name = model + elif hasattr(model, "config") and hasattr(model.config, "_name_or_path"): + model_name = model.config._name_or_path + else: + model_name = type(model).__name__ + # add info about the model and few shot config results["config"] = { - "model": model - if isinstance(model, str) - else model.model.config._name_or_path, + "model": model_name, "model_args": model_args, "batch_size": batch_size, "batch_sizes": list(lm.batch_sizes.values()) @@ -187,11 +217,12 @@ def simple_evaluate( def evaluate( lm, task_dict, - limit=None, - bootstrap_iters: int = 100000, + limit: Optional[int] = None, + bootstrap_iters: Optional[int] = 100000, decontamination_ngrams_path=None, write_out: bool = False, log_samples: bool = True, + verbosity: str = "INFO", ): """Instantiate and evaluate a model on a list of tasks. @@ -211,8 +242,17 @@ def evaluate( Dictionary of results """ + eval_logger.setLevel(getattr(logging, f"{verbosity}")) # decontaminate = decontamination_ngrams_path is not None + for task_name, task in task_dict.items(): + if isinstance(task, tuple): + _, task = task + if not log_samples: + assert ( + "bypass" not in getattr(task, "_metric_fn_list", {}).keys() + ), f"log_samples must be True for 'bypass' only tasks: {task_name}" + # stores the final result for each task, for each metric/filter pair. results = collections.defaultdict(dict) # Tracks each task's version. @@ -237,7 +277,7 @@ def evaluate( # get lists of each type of request for task_name, task in task_dict.items(): - if type(task) == tuple: + if isinstance(task, tuple): group_name, task = task task_hierarchy[group_name].append(task_name) versions[group_name] = "N/A" @@ -252,10 +292,9 @@ def evaluate( versions[task_name] = task.VERSION configs[task_name] = dict(task.dump_config()) - if "num_fewshot" in configs[task_name]: - n_shot = configs[task_name]["num_fewshot"] - else: - n_shot = 0 + # Number of few-shots for printing. + if (n_shot := configs[task_name].get("num_fewshot")) == 0: + n_shot = configs[task_name].get("metadata", {}).get("num_fewshot", 0) num_fewshot[task_name] = n_shot if "task_alias" in configs[task_name]: @@ -311,7 +350,7 @@ def evaluate( ### Run LM on inputs, get all outputs ### # execute each type of request for reqtype, reqs in requests.items(): - eval_logger.info("Running {} requests".format(reqtype)) + eval_logger.info(f"Running {reqtype} requests") # create `K` copies of each request `req` based off `K = req.repeats` cloned_reqs = [] for req in reqs: @@ -334,7 +373,7 @@ def evaluate( ### Postprocess outputs ### # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately) for task_name, task in task_dict.items(): - if type(task) == tuple: + if isinstance(task, tuple): group, task = task if task is None: continue @@ -345,7 +384,7 @@ def evaluate( # unpack results and sort back in order and return control to Task for task_name, task in task_dict.items(): - if type(task) == tuple: + if isinstance(task, tuple): group, task = task if task is None: continue @@ -396,10 +435,10 @@ def evaluate( vals_torch = collections.defaultdict(list) for (task_name, key, metric), items in vals.items(): numitem = 0 - if type(items[0]) == tuple: + if isinstance(items[0], tuple): numitem = len(items[0]) - if isinstance(items[0], (str, list)): + if isinstance(items[0], (str, list, tuple)): # handle the string case gathered_items = [None] * lm.accelerator.num_processes torch.distributed.all_gather_object(gathered_items, items) @@ -435,93 +474,70 @@ def evaluate( vals = vals_torch if lm.rank == 0: - ### Aggregate results over all datapoints ### # aggregate results ; run bootstrap CIs for (task_name, key, metric), items in vals.items(): task = task_dict[task_name] - metric_key = metric + "," + key - - if type(task) == tuple: - group_name, task = task - else: - group_name = None + group_name, task = task if isinstance(task, tuple) else (None, task) + metric_key = f"{metric},{key}" agg_fn = task.aggregation()[metric] + results[task_name][metric_key] = agg_fn(items) results[task_name]["samples"] = len(items) # hotfix: bleu, chrf, ter seem to be really expensive to bootstrap # so we run them less iterations. still looking for a cleaner way to do this if bootstrap_iters > 0: - stderr = lm_eval.api.metrics.stderr_for_metric( - metric=task.aggregation()[metric], + stderr_fn = lm_eval.api.metrics.stderr_for_metric( + metric=agg_fn, bootstrap_iters=min(bootstrap_iters, 100) if metric in ["bleu", "chrf", "ter"] else bootstrap_iters, ) - if stderr is not None and len(items) > 1: - results[task_name][metric + "_stderr" + "," + key] = stderr(items) - else: - results[task_name][metric + "_stderr" + "," + key] = "N/A" + results[task_name][f"{metric}_stderr,{key}"] = ( + stderr_fn(items) if (stderr_fn and len(items) > 1) else "N/A" + ) if bool(results): for group, task_list in reversed(task_hierarchy.items()): - if task_list == []: - total_size = results[group]["samples"] - else: - total_size = 0 - - for task in task_list: - metrics = results[task].copy() - - if "alias" in metrics: - metrics.pop("alias") - - current_size = metrics.pop("samples") - # TODO: There should be a way for users - # to toggle between weighted and - # unweighted averaging - # For unweighted averaging, use: - # current_size = 1 - - all_stderr = [] - for metric in [ - key for key in metrics.keys() if "_stderr" not in key - ]: - stderr = "_stderr,".join(metric.split(",")) - stderr_score = results[task][stderr] - var_score = stderr_score**2 - metric_score = results[task][metric] - - all_stderr.append(stderr) - - if metric in results[group]: - results[group][metric] = ( - results[group][metric] * total_size - + metric_score * current_size - ) / (total_size + current_size) - # $$s_z^2 = \frac{(n-1) s_x^2 + (m-1) s_y^2}{n+m-1} + \frac{nm(\bar x - \bar y)^2}{(n+m)(n+m-1)}.$$ - results[group][stderr] = ( - (total_size - 1) * results[group][stderr] - + (current_size - 1) * var_score - ) / ( - total_size + current_size - 1 - ) + total_size * current_size / ( - (total_size + current_size) - * (total_size + current_size - 1) - ) * (results[group][metric] - metric_score) ** 2 - else: - results[group][metric] = metric_score - results[group][stderr] = var_score - - total_size += current_size - - for stderr in all_stderr: - results[group][stderr] = np.sqrt(results[group][stderr]) - - results[group]["samples"] = total_size + if len(task_list) == 0: + # task_hierarchy entries are either + # `group_name: [subtask1, subtask2, ...]` + # or `task_name: []`. + # we only want to operate on groups here. + continue + for metric in [ + key + for key in results[task_list[0]].keys() + if "_stderr" not in key and key not in ["alias", "samples"] + ]: # TODO: what if tasks don't all share the same metrics + stderr = "_stderr,".join(metric.split(",")) + + # gather metrics, sizes, and stderrs from subtasks + metrics = [ + results[task][metric] for task in task_list + ] # TODO: copy? + stderrs = [results[task][stderr] for task in task_list] + sizes = [results[task]["samples"] for task in task_list] + + # compute group's pooled metric and stderr + results[group][ + metric + ] = lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes) + # TODO: calculate grouped metric using aggregation fn + if "N/A" in stderrs: + results[group][stderr] = "N/A" + else: + results[group][ + stderr + ] = lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes) + # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility + # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line: + # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics) + + results[group]["samples"] = sum(sizes) def print_tasks(task_hierarchy, results, tab=0): results_agg = collections.defaultdict(dict) @@ -596,8 +612,10 @@ def print_tasks(task_hierarchy, results, tab=0): groups_agg = {**groups_agg, **_groups_agg} for group_name, task_list in task_hierarchy.items(): - if task_list != []: - num_fewshot[group_name] = num_fewshot[task_list[0]] + if task_list: + num_fewshot[group_name] = num_fewshot[ + task_list[0] + ] # TODO: validate this results_dict = { "results": dict(results_agg.items()), diff --git a/lm_eval/filters/__init__.py b/lm_eval/filters/__init__.py index 76eb78467e..271f8c1ee8 100644 --- a/lm_eval/filters/__init__.py +++ b/lm_eval/filters/__init__.py @@ -1,3 +1,6 @@ +from typing import List, Union +from functools import partial + from lm_eval.api.filter import FilterEnsemble from . import selection from . import extraction @@ -20,24 +23,25 @@ } -def get_filter(filter_name): +def get_filter(filter_name: str) -> Union[type, str]: if filter_name in FILTER_REGISTRY: return FILTER_REGISTRY[filter_name] else: return filter_name -def build_filter_ensemble(filter_name, components): +def build_filter_ensemble( + filter_name: str, components: List[List[str]] +) -> FilterEnsemble: """ Create a filtering pipeline. """ filters = [] for function, kwargs in components: if kwargs is None: - f = get_filter(function)() - else: - # create a filter given its name in the registry - f = get_filter(function)(**kwargs) # TODO: pass kwargs to filters properly + kwargs = {} + # create a filter given its name in the registry + f = partial(get_filter(function), **kwargs) # add the filter as a pipeline step filters.append(f) diff --git a/lm_eval/filters/selection.py b/lm_eval/filters/selection.py index 6aaddbbbe8..01001fa377 100644 --- a/lm_eval/filters/selection.py +++ b/lm_eval/filters/selection.py @@ -17,12 +17,14 @@ def apply(self, resps, docs): class TakeKFilter(Filter): - def __init__(self, *args, **kwargs) -> None: + def __init__(self, **kwargs) -> None: self.k = kwargs.pop("k") - super().__init__(*args, **kwargs) + super().__init__(**kwargs) def apply(self, resps, docs): + # need resp to be subscriptable to check below + resps = list(resps) # check we have at least k responses per doc, else we can't take the first k assert ( len(resps[0]) >= self.k diff --git a/lm_eval/filters/transformation.py b/lm_eval/filters/transformation.py index f254b0db1b..41d03df7e1 100644 --- a/lm_eval/filters/transformation.py +++ b/lm_eval/filters/transformation.py @@ -24,7 +24,7 @@ def filter_set(inst): class MapFilter(Filter): - def __init__(self, mapping_dict: dict = {}, default_value=None) -> None: + def __init__(self, mapping_dict: dict = None, default_value=None) -> None: """ Initializes the MapFilter with a given mapping dictionary and default value. @@ -37,6 +37,8 @@ def __init__(self, mapping_dict: dict = {}, default_value=None) -> None: Example: mapper = MapFilter({'A': 1, 'B': 2}, default_value=0) """ + if mapping_dict is None: + mapping_dict = {} assert isinstance( mapping_dict, dict ), "Provided mapping_dict is not a dictionary" diff --git a/lm_eval/models/__init__.py b/lm_eval/models/__init__.py index f994bdebf5..64f338e6ba 100644 --- a/lm_eval/models/__init__.py +++ b/lm_eval/models/__init__.py @@ -6,5 +6,17 @@ from . import gguf from . import vllm_causallms from . import mamba_lm - +from . import optimum_lm +from . import neuron_optimum # TODO: implement __all__ + + +import os + +try: + # enabling faster model download + import hf_transfer + + os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1" +except ImportError: + pass diff --git a/lm_eval/models/huggingface.py b/lm_eval/models/huggingface.py index 0dc786d9dd..1526e45dcc 100644 --- a/lm_eval/models/huggingface.py +++ b/lm_eval/models/huggingface.py @@ -1,12 +1,18 @@ import copy import os +from datetime import timedelta from pathlib import Path from typing import List, Literal, Optional, Tuple, Union import torch import torch.nn.functional as F import transformers -from accelerate import Accelerator, DistributedType, find_executable_batch_size +from accelerate import ( + Accelerator, + DistributedType, + InitProcessGroupKwargs, + find_executable_batch_size, +) from packaging import version from peft import PeftModel from peft import __version__ as PEFT_VERSION @@ -108,8 +114,8 @@ def __init__( assert not parallelize, "`parallelize=True` is not compatible with passing pre-initialized model to `pretrained`" self._model = pretrained self._device = self._model.device - self._config = self._model.config + gpus = 0 if tokenizer: assert isinstance( @@ -132,7 +138,10 @@ def __init__( assert isinstance(batch_size, (int, str)) gpus = torch.cuda.device_count() - accelerator = Accelerator() + accelerator_kwargs = InitProcessGroupKwargs(timeout=timedelta(weeks=52)) + accelerator = Accelerator(kwargs_handlers=[accelerator_kwargs]) + if accelerator.num_processes > 1: + self.accelerator = accelerator if not (parallelize or accelerator.num_processes > 1): # use user-passed device @@ -198,19 +207,21 @@ def __init__( ) # access self._model through self.model property outside this method - self.model.eval() - self.model.tie_weights() + if isinstance(self.model, torch.nn.Module): + self.model.eval() + self.model.tie_weights() if isinstance(pretrained, str) and (gpus >= 1 or str(self.device) == "mps"): - if not (parallelize or autogptq or ("device_map" in kwargs)): + # TODO: can remove this whole snippet except in the mps case, perhaps? + if not (parallelize or autogptq or hasattr(self, "accelerator")): # place model onto device requested manually, # if not using HF Accelerate or device_map # or any other option that preloads model onto device try: self.model.to(self.device) except ValueError: - eval_logger.info( - "Failed to place model onto specified device. This may be because the model is quantized via `bitsandbytes`. If the desired GPU is being used, this message is safe to ignore." + eval_logger.debug( + "Failed to place model onto specified device. This may be because the model is quantized via `bitsandbytes` or `device_map` is provided. If the desired GPU is being used, this message is safe to ignore." ) self._create_tokenizer( @@ -235,6 +246,16 @@ def __init__( if self.config.model_type == "qwen": # Qwen's trust_remote_code tokenizer does not allow for adding special tokens self.tokenizer.pad_token = "<|endoftext|>" + elif ( + self.tokenizer.__class__.__name__ == "RWKVWorldTokenizer" + or self.tokenizer.__class__.__name__ == "Rwkv5Tokenizer" + ): + # The RWKV world tokenizer, does not allow for adding special tokens / setting the pad token (which is set as 0) + # The additional tokenizer name check is needed, as there exists rwkv4 models with neox tokenizer + # --- + # Note that the world tokenizer class name, might change in the future for the final huggingface merge + # https://github.com/huggingface/transformers/pull/26963 + assert self.tokenizer.pad_token_id == 0 else: self.tokenizer.add_special_tokens({"pad_token": "<|pad|>"}) @@ -358,7 +379,7 @@ def world_size(self): def _get_backend( self, - config: transformers.AutoConfig, + config: Union[transformers.PretrainedConfig, transformers.AutoConfig], backend: Optional[Literal["default", "causal", "seq2seq"]] = "default", trust_remote_code: Optional[bool] = False, ) -> None: @@ -456,12 +477,24 @@ def _create_model( if parallelize: model_kwargs.update( _get_accelerate_args( - device_map_option, + device_map_option, # TODO: phase out device_map_option? max_memory_per_gpu, max_cpu_memory, offload_folder, ) ) + elif "device_map" not in model_kwargs: + # set a device_map to initialize model on the right GPU. + # this is needed because it seems that the default behavior + # for quantized models now seems to be device_map="auto" + # which breaks data-parallel mode. + if hasattr(self, "accelerator"): + model_kwargs.update( + {"device_map": {"": f"cuda:{self.accelerator.local_process_index}"}} + ) + else: + model_kwargs.update({"device_map": {"": str(self.device)}}) + if not autogptq: if model_kwargs.get("load_in_4bit", None): assert ( @@ -587,12 +620,17 @@ def forward_batch(batch_size): (batch_size, max_length), device=self.device ).long() for _ in range(5): - out = F.log_softmax(self._model_call(test_batch, **call_kwargs), dim=-1) - out = out # Identity process so that it passes pre-commit + out = F.log_softmax(self._model_call(test_batch, **call_kwargs), dim=-1) # noqa: F841 return batch_size - batch_size = forward_batch() + try: + batch_size = forward_batch() + except RuntimeError as e: + if "No executable batch size found" in str(e): + batch_size = 1 + else: + raise if self.world_size > 1: # if multi-GPU, always take minimum over all selected batch sizes @@ -690,10 +728,19 @@ def _model_call(self, inps, attn_mask=None, labels=None): return self.model(inps).logits def _model_generate(self, context, max_length, stop, **generation_kwargs): - # we require users to pass do_sample=True explicitly - # for non-greedy gen. This should be reevaluated when considering beam search. - if "do_sample" not in generation_kwargs: - generation_kwargs["do_sample"] = False + # temperature = 0.0 if not set + # if do_sample is false and temp==0.0: + # remove temperature, as do_sample=False takes care of this + # and we don't want a warning from HF + generation_kwargs["temperature"] = generation_kwargs.get("temperature", 0.0) + do_sample = generation_kwargs.get("do_sample", None) + + # The temperature has to be a strictly positive float -- if it is 0.0, use greedy decoding strategies + if generation_kwargs.get("temperature") == 0.0 and do_sample is None: + generation_kwargs["do_sample"] = do_sample = False + + if do_sample is False and generation_kwargs.get("temperature") == 0.0: + generation_kwargs.pop("temperature") # build stopping criteria stopping_criteria = stop_sequences_criteria( self.tokenizer, stop, context.shape[1], context.shape[0] @@ -1030,6 +1077,7 @@ def _collate(x): return -len(toks), x[0] pbar = tqdm(total=len(requests), disable=(self.rank != 0)) + adaptive_batch_size = None if self.batch_size == "auto": # using rolling window with maximum context print("Passed argument batch_size = auto. Detecting largest batch size") @@ -1074,7 +1122,7 @@ def _collate(x): ) else: raise ValueError( - f"Expected `kwargs` to be of type `dict` but got {kwargs}" + f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}" ) if not until: until = [self.tok_decode(self.eot_token_id)] diff --git a/lm_eval/models/mamba_lm.py b/lm_eval/models/mamba_lm.py index fc7769fd59..27a7a1f460 100644 --- a/lm_eval/models/mamba_lm.py +++ b/lm_eval/models/mamba_lm.py @@ -42,7 +42,7 @@ def __init__( The HFLM arguments - `backend`, `revision`, `subfolder`, `tokenizer`, `truncation`, `max_length`, + `backend`, `tokenizer`, `truncation`, `max_length`, `device`, `dtype`, `batch_size`, `max_batch_size`, `trust_remote_code`, `use_fast_tokenizer` Are all supported by Mamba where they do not conflict @@ -98,7 +98,6 @@ def _create_model( pretrained, device=self._device, dtype=torch.float16 if dtype == "auto" else utils.get_dtype(dtype), - **kwargs, ) def _model_generate(self, context, max_length, stop, **generation_kwargs): diff --git a/lm_eval/models/neuron_optimum.py b/lm_eval/models/neuron_optimum.py new file mode 100644 index 0000000000..b2dca589c8 --- /dev/null +++ b/lm_eval/models/neuron_optimum.py @@ -0,0 +1,753 @@ +import copy +import json +import logging +import subprocess +from collections import defaultdict +from typing import List, Optional, Union + +import torch +import torch.nn.functional as F +import transformers +from packaging import version +from tqdm import tqdm +from transformers import GenerationConfig +from transformers.generation import StoppingCriteriaList + +from lm_eval import utils +from lm_eval.api.model import LM +from lm_eval.api.registry import register_model +from lm_eval.utils import stop_sequences_criteria + + +try: + NEURON_AVAILABLE = True + from optimum.neuron import NeuronModelForCausalLM + from optimum.neuron.generation import TokenSelector + from optimum.neuron.version import __version__ as optimum_neuron_version +except ImportError: + NeuronModelForCausalLM = object + NEURON_AVAILABLE = False + + +logger = logging.getLogger(__name__) + + +def get_nc_count() -> Union[int, None]: + """Returns the number of neuron cores on the current instance.""" + try: + cmd = "neuron-ls --json-output" + result = subprocess.run(cmd, shell=True, capture_output=True) + print(f"inferring nc_count from `neuron-ls` {result.stdout}") + json_output = json.loads(result.stdout) + count = sum([x["nc_count"] for x in json_output]) + print(f"nc_count={count}") + return count + except Exception: + return None + + +def wrap_constant_batch_size(func): + def _decorator(self, input_ids): + """input_ids a 2D array with batch_size on dim=0 + + makes sure the func runs with self.batch_size + """ + # access a from TestSample + batch_size = input_ids.shape[0] + + if batch_size < self.batch_size: + # handle the event of input_ids.shape[0] != batch_size + # Neuron cores expect constant batch_size + input_ids = torch.concat( + ( + input_ids, + # add missing_batch_size dummy + torch.zeros( + [self.batch_size - batch_size, *input_ids.size()[1:]], + dtype=input_ids.dtype, + device=input_ids.device, + ), + ), + dim=0, + ) + elif batch_size > self.batch_size: + raise ValueError( + f"The specified batch_size ({batch_size}) exceeds the model static batch size ({self.batch_size})" + ) + # return the forward pass that requires constant batch size + return func(self, input_ids)[:batch_size] + + return _decorator + + +class CustomNeuronModelForCausalLM(NeuronModelForCausalLM): + """NeuronModelForCausalLM with `stopping_criteria` in `generate`""" + + def generate( + self, + input_ids: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + stopping_criteria: Optional["StoppingCriteriaList"] = None, + generation_config: Optional["GenerationConfig"] = None, + **kwargs, + ) -> torch.LongTensor: + r""" + A streamlined generate() method overriding the transformers.GenerationMixin.generate() method. + + This method uses the same logits processors/warpers and stopping criteria as the transformers library + `generate()` method but restricts the generation to greedy search and sampling. + + It does not support transformers `generate()` advanced options. + + Please refer to https://huggingface.co/docs/transformers/en/main_classes/text_generation#transformers.GenerationMixin.generate + for details on generation configuration. + + Parameters: + input_ids (`torch.Tensor` of shape `(batch_size, sequence_length)`): + The sequence used as a prompt for the generation. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. + generation_config (`~transformers.generation.GenerationConfig`, *optional*): + The generation configuration to be used as base parametrization for the generation call. `**kwargs` + passed to generate matching the attributes of `generation_config` will override them. If + `generation_config` is not provided, default will be used, which had the following loading + priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model + configuration. Please note that unspecified parameters will inherit [`~transformers.generation.GenerationConfig`]'s + default values, whose documentation should be checked to parameterize generation. + + Returns: + `torch.Tensor`: A `torch.FloatTensor`. + """ + # The actual generation configuration is a combination of config and parameters + generation_config = copy.deepcopy( + self.generation_config if generation_config is None else generation_config + ) + model_kwargs = generation_config.update( + **kwargs + ) # All unused kwargs must be model kwargs + # Check model kwargs are actually used by either prepare_inputs_for_generation or forward + self._validate_model_kwargs(model_kwargs) + + # Instantiate a TokenSelector for the specified configuration + selector = TokenSelector.create( + input_ids, generation_config, self, self.max_length + ) + selector.stopping_criteria.append(stopping_criteria) + # Verify that the inputs are compatible with the model static input dimensions + batch_size, sequence_length = input_ids.shape + if sequence_length > self.max_length: + raise ValueError( + f"The input sequence length ({sequence_length}) exceeds the model static sequence length ({self.max_length})" + ) + padded_input_ids = input_ids + padded_attention_mask = attention_mask + if batch_size > self.batch_size: + raise ValueError( + f"The specified batch_size ({batch_size}) exceeds the model static batch size ({self.batch_size})" + ) + elif batch_size < self.batch_size: + logger.warning( + "Inputs will be padded to match the model static batch size. This will increase latency." + ) + padding_shape = [self.batch_size - batch_size, sequence_length] + padding = torch.full( + padding_shape, fill_value=self.config.eos_token_id, dtype=torch.int64 + ) + padded_input_ids = torch.cat([input_ids, padding]) + if attention_mask is not None: + padding = torch.zeros(padding_shape, dtype=torch.int64) + padded_attention_mask = torch.cat([attention_mask, padding]) + # Drop the current generation context and clear the Key/Value cache + self.reset_generation() + + output_ids = self.generate_tokens( + padded_input_ids, + selector, + batch_size, + attention_mask=padded_attention_mask, + **model_kwargs, + ) + return output_ids[:batch_size, :] + + +@register_model("neuronx") +class NEURON_HF(LM): + """ + Enables usage with on AWS Neuron + using the HuggingFace Transformers + Transformers neuronx library. + Tested with neuron 2.17.0 + """ + + _DEFAULT_MAX_LENGTH = 2048 + + def __init__( + self, + pretrained: Optional[str] = "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + revision: Optional[str] = "main", + tp_degree: Optional[int] = None, + subfolder: Optional[str] = None, + tokenizer: Optional[str] = None, + truncation: Optional[bool] = False, + max_length: Optional[int] = None, + dtype: Optional[Union[str, torch.dtype]] = "auto", + batch_size: Optional[int] = 1, + low_cpu_mem_usage: Optional[bool] = True, + trust_remote_code: Optional[bool] = False, + use_fast_tokenizer: Optional[bool] = True, + # arguments used for splitting a model across GPUs naively. + # only used if `parallelize=True`. + ) -> None: + if not NEURON_AVAILABLE: + raise Exception( + "Tried to load neuron model, but neuron is not installed ", + "please install neuron via pip install transformers-neuron ", + "also make sure you are running on an AWS inf2 instance", + ) + if version.parse(optimum_neuron_version) != version.parse("0.0.17"): + logger.warning( + '`optimum-neuron` model requires `pip install "optimum[neuronx]>=0.0.17" ' + "preferably using the Hugging Face Neuron Deep Learning AMI (Ubuntu 22.04) " + "https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2 " + f"You are using optimum-neuron={optimum_neuron_version}" + ) + super().__init__() + + assert isinstance(pretrained, str) + assert isinstance(batch_size, (int, str)) + + self.batch_size_per_gpu = int(batch_size) + batch_size = int(batch_size) + if tp_degree is None: + # execute `neuron-ls --json-output | jq '.[0].nc_count'`` + # to get the number of neuron cores on your instance + tp_degree = get_nc_count() + + assert isinstance(tp_degree, int), ( + f"model_args must include tp_degree. tp_degree must be set to an integer," + f" but is tp_degree=`{tp_degree}` with type=`{type(tp_degree)}`." + "Set it to number of neuron cores on your instance." + " For inf2.xlarge and inf2.8xlarge, set it to `2`." + " For inf2.24xlarge, set it to `12`." + " For inf2.48xlarge, set it to `24`." + ) + + # TODO: update this to be less of a hack once subfolder is fixed in HF + revision = revision + ("/" + subfolder if subfolder is not None else "") + + self._config = transformers.AutoConfig.from_pretrained( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + ) + torch_dtype = utils.get_dtype(dtype) + + assert torch_dtype in [ + torch.float16, + torch.bfloat16, + ], "Only float16 and bfloat16 are supported" + + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + pretrained if tokenizer is None else tokenizer, + revision=revision, + trust_remote_code=trust_remote_code, + use_fast=use_fast_tokenizer, + ) + + # Neuron specific code + if torch_dtype == torch.float16: + self.amp_dtype = "f16" + elif torch_dtype == torch.bfloat16: + self.amp_dtype = "bf16" + elif torch_dtype == torch.float32: + self.amp_dtype = "f32" + else: + raise NotImplementedError("Only float16 and bfloat16 are implemented.") + + compiler_args = {"num_cores": tp_degree, "auto_cast_type": self.amp_dtype} + input_shapes = { + "batch_size": batch_size, + "sequence_length": self._DEFAULT_MAX_LENGTH, + } + + print( + f"{'='*20} \n loading model to neuron with" + f" {compiler_args}, {input_shapes}..." + ) + self.model = CustomNeuronModelForCausalLM.from_pretrained( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + low_cpu_mem_usage=low_cpu_mem_usage, + export=True, + **compiler_args, + **input_shapes, + ) + print(f"SUCCESS: neuron model compiled. \n {'='*20}") + + self.truncation = truncation + + self.vocab_size = self.tokenizer.vocab_size + self.tokenizer.pad_token_id = self.tokenizer.eos_token_id + + self._max_length = max_length + + self.batch_schedule = 1 + self.batch_sizes = {} + + @property + def config(self): + # return the associated transformers.AutoConfig for the given pretrained model. + return self._config + + @property + def eot_token_id(self): + # we use EOT because end of *text* is more accurate for what we're doing than end of *sentence* + return self.tokenizer.eos_token_id + + @property + def max_length(self): + if self._max_length: # if max length manually set, return it + return self._max_length + seqlen_config_attrs = ("n_positions", "max_position_embeddings", "n_ctx") + for attr in seqlen_config_attrs: + if hasattr(self.model.config, attr): + return getattr(self.model.config, attr) + if hasattr(self.tokenizer, "model_max_length"): + if self.tokenizer.model_max_length == 1000000000000000019884624838656: + return self._DEFAULT_MAX_LENGTH + return self.tokenizer.model_max_length + return self._DEFAULT_MAX_LENGTH + + @property + def max_gen_toks(self) -> int: + return 256 + + @property + def batch_size(self): + return self.batch_size_per_gpu + + @property + def device(self): + """device are neuron cores, but the created tensors are on CPU.""" + return "cpu" + + @property + def rank(self): + return 0 + + @property + def world_size(self): + return 1 + + def tok_encode(self, string: str, left_truncate_len=None, add_special_tokens=None): + """ """ + if add_special_tokens is None: + add_special_tokens = False + + encoding = self.tokenizer.encode(string, add_special_tokens=add_special_tokens) + + # left-truncate the encoded context to be at most `left_truncate_len` tokens long + if left_truncate_len: + encoding = encoding[-left_truncate_len:] + + return encoding + + def tok_batch_encode( + self, + strings: List[str], + padding_side: str = "left", + left_truncate_len: int = None, + truncation: bool = False, + ): + # encode a batch of strings. converts to tensors and pads automatically, unlike tok_encode. + old_padding_side = self.tokenizer.padding_side + self.tokenizer.padding_side = padding_side + + add_special_tokens = False + + encoding = self.tokenizer( + strings, + truncation=truncation, + padding="longest", + return_tensors="pt", + add_special_tokens=add_special_tokens, + ) + if left_truncate_len: + encoding["input_ids"] = encoding["input_ids"][:, -left_truncate_len:] + encoding["attention_mask"] = encoding["attention_mask"][ + :, -left_truncate_len: + ] + self.tokenizer.padding_side = old_padding_side + + return encoding["input_ids"], encoding["attention_mask"] + + def tok_decode(self, tokens): + return self.tokenizer.decode(tokens) + + @wrap_constant_batch_size + def _model_call(self, input_ids: torch.Tensor): + """ + get logits for the entire sequence + + :param input_ids: torch.Tensor + A torch tensor of shape [batch, sequence_cont] + the size of sequence may vary from call to call + :return + A torch tensor of shape [batch, sequence, vocab] with the + logits returned from the model's decoder-lm head + """ + _, sequence_length = input_ids.shape + + with torch.inference_mode(): + cache_ids = torch.arange(0, sequence_length, dtype=torch.int32).split(1) + input_ids_split = input_ids.split(1, dim=1) + + return torch.concat( + [ + self.model.forward( + input_ids=input_id, cache_ids=cache_id, return_dict=False + )[0] + for input_id, cache_id in zip(input_ids_split, cache_ids) + ], + dim=1, + ) + + def _model_generate(self, context, max_length, stop, **generation_kwargs): + # we require users to pass do_sample=True explicitly + # for non-greedy gen. This should be reevaluated when considering beam search. + + with torch.inference_mode(): + if "do_sample" not in generation_kwargs.keys(): + generation_kwargs["do_sample"] = False + + stopping_criteria = stop_sequences_criteria( + self.tokenizer, + stop + [self.tokenizer.decode([self.config.eos_token_id])], + 1, + context.shape[0], + ) + + return self.model.generate( + input_ids=context, + max_length=max_length, + stopping_criteria=stopping_criteria, + pad_token_id=self.eot_token_id, + use_cache=True, + **generation_kwargs, + ) + + def _select_cont_toks(self, logits, contlen=None, inplen=None): + assert ( + contlen and inplen + ), "Must pass input len and cont. len to select scored logits for causal LM" + # discard right-padding. + # also discard the input/context tokens. we'll only score continuations. + logits = logits[inplen - contlen : inplen] + + return logits + + def _encode_pair(self, context, continuation): + n_spaces = len(context) - len(context.rstrip()) + if n_spaces > 0: + continuation = context[-n_spaces:] + continuation + context = context[:-n_spaces] + + whole_enc = self.tok_encode(context + continuation, add_special_tokens=False) + context_enc = self.tok_encode(context, add_special_tokens=False) + + # whole_enc = self.tok_encode(context + continuation) + # context_enc = self.tok_encode(context, add_special_tokens=False) + context_enc_len = len(context_enc) + continuation_enc = whole_enc[context_enc_len:] + return context_enc, continuation_enc + + def loglikelihood(self, requests): + new_reqs = [] + for context, continuation in [req.args for req in requests]: + if context == "": + # end of text as context + context_enc, continuation_enc = ( + [self.eot_token_id], + self.tok_encode(continuation), + ) + else: + context_enc, continuation_enc = self._encode_pair(context, continuation) + + new_reqs.append(((context, continuation), context_enc, continuation_enc)) + + return self._loglikelihood_tokens(new_reqs) + + def loglikelihood_rolling(self, requests): + loglikelihoods = [] + + adaptive_batch_size = None + + for (string,) in tqdm([req.args for req in requests], disable=(self.rank != 0)): + rolling_token_windows = list( + map( + utils.make_disjoint_window, + utils.get_rolling_token_windows( + token_list=self.tok_encode(string), + prefix_token=self.eot_token_id, + max_seq_len=self.max_length, + context_len=1, + ), + ) + ) + + # TODO: Right now, we pass single EOT token to the Encoder and the full context to the decoder, in seq2seq case + rolling_token_windows = [(None,) + x for x in rolling_token_windows] + + pad_amnt = 0 + if self.world_size > 1: + # We pad out the external document-level iterator so the inner iterator doesn't hang + mytensor = torch.tensor(len(rolling_token_windows), device=self.device) + gathered = ( + self.accelerator.gather(mytensor).cpu().detach().numpy().tolist() + ) + + pad_amnt = max(gathered) - gathered[self.rank] + if pad_amnt > 0: + rolling_token_windows += pad_amnt * [rolling_token_windows[0]] + + string_nll = self._loglikelihood_tokens( + rolling_token_windows, + disable_tqdm=True, + override_bs=adaptive_batch_size, + ) + + if (self.world_size > 1) and (pad_amnt > 0): + string_nll = [x[0] for x in string_nll[:-pad_amnt]] + else: + # discard is_greedy + string_nll = [x[0] for x in string_nll] + + string_nll = sum(string_nll) + loglikelihoods.append(string_nll) + + return loglikelihoods + + def _loglikelihood_tokens( + self, requests, disable_tqdm: bool = False, override_bs=None + ): + # TODO: implement some kind of efficient-request-middleware that lumps together requests with the same context + res = [] + + def _collate(x): + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + + toks = x[1] + x[2] + return -len(toks), tuple(toks) + + re_ord = utils.Reorderer(requests, _collate) + + n_reordered_requests = len(re_ord.get_reordered()) # noqa + # automatic (variable) batch size detection for vectorization + # pull longest context sample from request + + chunks = utils.chunks( + re_ord.get_reordered(), + n=self.batch_size, + fn=None, + ) + + for chunk in tqdm(chunks, disable=(disable_tqdm or (self.rank != 0))): + inps = [] + cont_toks_list = [] + inplens = [] + + conts = [] # noqa + encoder_attns = [] # noqa + + padding_len_inp = None + padding_len_cont = None # noqa + # because vectorizing is annoying, we first convert each (context, continuation) pair to padded + # tensors, then we pack them together into a batch, call the model, and then pick it all apart + # again because vectorizing is annoying + + for _, context_enc, continuation_enc in chunk: + # sanity check + assert len(context_enc) > 0 + assert len(continuation_enc) > 0 + assert len(continuation_enc) <= self.max_length + + # how this all works (illustrated on a causal decoder-only setup): + # CTX CONT + # inp 0 1 2 3|4 5 6 7 8 9 <- last token is deleted by inp[:, :-1] + # model \ \ + # logits 1 2 3|4 5 6 7 8 9 <- the ctx half gets tossed out by the + # cont_toks 4 5 6 7 8 9 [:, -len(continuation_enc):, :self.vocab_size] slice + + # when too long to fit in context, truncate from the left + inp = torch.tensor( + (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1], + dtype=torch.long, + device=self.device, + ) + (inplen,) = inp.shape + + padding_len_inp = ( + max(padding_len_inp, inplen) + if padding_len_inp is not None + else inplen + ) + + inps.append(inp) # [1, inp_length] + cont_toks_list.append(continuation_enc) + inplens.append(inplen) + + # create encoder attn mask and batched conts, if seq2seq + call_kwargs = {} + batched_inps = utils.pad_and_concat( + padding_len_inp, inps, padding_side="right" + ) # [batch, padding_len_inp] + + multi_logits = F.log_softmax( + self._model_call(batched_inps, **call_kwargs), dim=-1 + ) # [batch, padding_length (inp or cont), vocab] + + for (cache_key, _, _), logits, inplen, cont_toks in zip( + chunk, multi_logits, inplens, cont_toks_list + ): + # Slice to original seq length + contlen = len(cont_toks) + # take only logits in the continuation + # (discard context toks if decoder-only ; discard right-padding) + # also discards + checks for "virtual tokens" in the causal LM's input window + # from prompt/prefix tuning tokens, if applicable + ctx_len = inplen + (logits.shape[0] - padding_len_inp) + logits = self._select_cont_toks(logits, contlen=contlen, inplen=ctx_len) + logits = logits.unsqueeze(0) # [1, seq, vocab] + + # Check if per-token argmax is exactly equal to continuation + greedy_tokens = logits.argmax(dim=-1) + cont_toks = torch.tensor( + cont_toks, dtype=torch.long, device=self.device + ).unsqueeze(0) # [1, seq] + max_equal = (greedy_tokens == cont_toks).all() + + # Obtain log-probs at the corresponding continuation token indices + # last_token_slice = logits[:, -1, :].squeeze(0).tolist() + logits = torch.gather(logits, 2, cont_toks.unsqueeze(-1)).squeeze( + -1 + ) # [1, seq] + + # Answer: (log prob, is-exact-match) + answer = (float(logits.sum()), bool(max_equal)) + + res.append(answer) + + self.cache_hook.add_partial("loglikelihood", cache_key, answer) + + return re_ord.get_original(res) + + def generate_until(self, requests): + res = defaultdict(list) + re_ords = {} + + def _collate(x): + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(x[0]) + return -len(toks), x[0] + + # we group requests by their generation_kwargs, + # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling + # in the same batch. + grouper = utils.Grouper(requests, lambda x: str(x.args[1])) + for key, reqs in grouper.get_grouped().items(): + # within each set of reqs for given kwargs, we reorder by token length, descending. + re_ords[key] = utils.Reorderer([req.args for req in reqs], _collate) + + pbar = tqdm(total=len(requests), disable=(self.rank != 0)) + + # for each different set of kwargs, we execute all requests, by batch. + for key, re_ord in re_ords.items(): + chunks = utils.chunks(re_ord.get_reordered(), n=self.batch_size) + for chunk in tqdm(chunks, disable=self.rank != 0): + contexts, all_gen_kwargs = zip(*chunk) + # we assume all gen kwargs in the batch are the same + # this is safe to assume because the `grouper` object ensures it. + gen_kwargs = all_gen_kwargs[0] + # unpack our keyword arguments. + until = None + if isinstance(gen_kwargs, dict): + kwargs = copy.deepcopy(gen_kwargs) # edge case for repeats > 1 + if "until" in kwargs.keys(): + until = kwargs.pop("until") + if isinstance(until, str): + until = [kwargs] + elif not isinstance(until, list): + raise ValueError( + f"Expected `kwargs['until']` to be of type Union[str,list] but got {until}" + ) + else: + raise ValueError( + f"Expected `kwargs` to be of type `dict` but got {kwargs}" + ) + if not until: + until = [self.tok_decode(self.eot_token_id)] + if "max_gen_toks" in kwargs.keys(): + max_gen_toks = kwargs.pop("max_gen_toks") + else: + max_gen_toks = self.max_gen_toks + # first stop sequence is used to halt generation upon encountering + primary_until = [until[0]] + + max_ctx_len = self.max_length - max_gen_toks + + # encode, pad, and truncate contexts for this batch + context_enc, attn_masks = self.tok_batch_encode( + contexts, + left_truncate_len=max_ctx_len, + truncation=self.truncation, + ) + context_enc = context_enc.to(self.device) + attn_masks = attn_masks.to(self.device) + + if "max_length" not in kwargs: + kwargs["max_length"] = context_enc.shape[1] + max_gen_toks + + # perform batched generation + cont = self._model_generate( + context=context_enc, + attention_mask=attn_masks, + stop=primary_until, + **kwargs, + ) + + cont_toks_list = cont.tolist() + for cont_toks, context in zip(cont_toks_list, contexts): + # discard context + left-padding toks if using causal decoder-only LM + cont_toks = cont_toks[context_enc.shape[1] :] + + s = self.tok_decode(cont_toks) + + # use secondary stop seqs to cut off should-have-been-stopped content post-hoc + for term in until: + if len(term) > 0: + # ignore '' separator, + # for seq2seq case where self.tok_decode(self.eot_token_id) = '' + s = s.split(term)[0] + + res[key].append(s) + + self.cache_hook.add_partial( + "generate_until", (context, gen_kwargs), s + ) + pbar.update(1) + # reorder this group of results back to original unsorted form + res[key] = re_ord.get_original(res[key]) + + pbar.close() + + return grouper.get_original(res) diff --git a/lm_eval/models/openai_completions.py b/lm_eval/models/openai_completions.py index c001e5caa2..51eaf49a9f 100644 --- a/lm_eval/models/openai_completions.py +++ b/lm_eval/models/openai_completions.py @@ -2,14 +2,14 @@ import os from collections import defaultdict from importlib.util import find_spec -from typing import List, Optional, Tuple +from typing import List, Literal, Optional, Tuple from tqdm import tqdm from lm_eval import utils from lm_eval.api.model import LM from lm_eval.api.registry import register_model -from lm_eval.utils import retry_on_specific_exceptions +from lm_eval.utils import eval_logger, retry_on_specific_exceptions def get_result(response, ctxlen: int) -> Tuple[float, bool]: @@ -40,7 +40,7 @@ def get_result(response, ctxlen: int) -> Tuple[float, bool]: return continuation_logprobs, is_greedy -def oa_completion(**kwargs): +def oa_completion(client, chat: bool = False, **kwargs): """Query OpenAI API for completion. Retry with back-off until they respond @@ -64,19 +64,24 @@ def _exception_callback(e: Exception, sleep_time: float) -> None: on_exception_callback=_exception_callback, ) def completion(): - return openai.completions.create(**kwargs) + if chat: + return client.chat.completions.create(**kwargs) + else: + return client.completions.create(**kwargs) return completion() -@register_model("openai-completions") +@register_model("openai-completions", "local-completions") class OpenaiCompletionsLM(LM): - REQ_CHUNK_SIZE = 20 _DEFAULT_MAX_LENGTH = 2048 def __init__( self, model: str, + base_url: str = None, + tokenizer: Optional[str] = None, + tokenizer_backend: Literal["tiktoken", "huggingface"] = "tiktoken", truncate: bool = False, max_gen_toks: int = 256, batch_size: int = 1, @@ -101,15 +106,44 @@ def __init__( please install these via `pip install lm-eval[openai]` or `pip install -e .[openai]`", ) self.model = model - self.tokenizer = tiktoken.encoding_for_model(self.model) - self.vocab_size = self.tokenizer.n_vocab + self.base_url = base_url + self.tokenizer_backend = tokenizer_backend self.truncate = truncate - self.end_of_text_token_id = self.tokenizer.eot_token + self._batch_size = batch_size self._max_gen_toks = max_gen_toks self._max_length = max_length + # if we have a local model, use HF tokenizer over tiktoken + if self.tokenizer_backend == "huggingface": + import transformers # noqa: E401 + + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + tokenizer if tokenizer else self.model + ) + self.vocab_size = self.tokenizer.vocab + self.end_of_text_token_id = self.tokenizer.eos_token + elif self.tokenizer_backend == "tiktoken": + if self.base_url: + eval_logger.warning( + f"Passed `base_url={self.base_url}` but using Tiktoken tokenizer backend. " + "Pass `tokenizer_backend=huggingface` and provide the HF tokenizer name if your model does not use Tiktoken." + ) + + self.tokenizer = tiktoken.encoding_for_model(self.model) + self.vocab_size = self.tokenizer.n_vocab + self.end_of_text_token_id = self.tokenizer.eot_token + else: + raise ValueError( + f"Expected tokenizer_backend to be one of ['tiktoken', 'huggingface'] but got {self.tokenizer_backend}" + ) + # Read from environment variable OPENAI_API_KEY + # Set to EMPTY for local openai.api_key = os.environ["OPENAI_API_KEY"] + if self.base_url: + self.client = openai.OpenAI(base_url=self.base_url) + else: + self.client = openai.OpenAI() @property def eot_token_id(self): @@ -127,9 +161,8 @@ def max_gen_toks(self) -> int: return self._max_gen_toks @property - def batch_size(self): - # Isn't used because we override _loglikelihood_tokens - raise NotImplementedError() + def batch_size(self) -> int: + return self._batch_size @property def device(self): @@ -186,7 +219,7 @@ def _collate(x): re_ord = utils.Reorderer(requests, _collate) for chunk in tqdm( - list(utils.chunks(re_ord.get_reordered(), self.REQ_CHUNK_SIZE)), + list(utils.chunks(re_ord.get_reordered(), self.batch_size)), disable=disable_tqdm, ): inps = [] @@ -203,6 +236,7 @@ def _collate(x): ctxlens.append(ctxlen) response = oa_completion( + client=self.client, model=self.model, prompt=inps, echo=True, @@ -251,7 +285,7 @@ def sameuntil_chunks(xs, size): # todo: more intelligent batching for heterogeneous `until` for chunk, request_args in tqdm( - list(sameuntil_chunks(re_ord.get_reordered(), self.REQ_CHUNK_SIZE)) + list(sameuntil_chunks(re_ord.get_reordered(), self.batch_size)) ): inps = [] self._max_gen_toks = request_args.pop("max_gen_toks", self.max_gen_toks) @@ -265,6 +299,7 @@ def sameuntil_chunks(xs, size): request_args["temperature"] = request_args.get("temperature", 0) response = oa_completion( + client=self.client, model=self.model, prompt=inps, max_tokens=self.max_gen_toks, @@ -329,35 +364,6 @@ def loglikelihood_rolling(self, requests) -> List[float]: return loglikelihoods -def oa_chat_completion(client, **kwargs): - """Query OpenAI API for chat completion. - - Retry with back-off until they respond - """ - if not find_spec("openai") or not find_spec("tiktoken"): - raise Exception( - "attempted to use 'openai' LM type, but package `openai` or `tiktoken` are not installed. " - "Please install these via `pip install lm-eval[openai]` or `pip install -e .[openai]`" - ) - else: - import openai - - def _exception_callback(e: Exception, sleep_time: float) -> None: - import traceback - - traceback.print_exc() - - @retry_on_specific_exceptions( - on_exceptions=[openai.OpenAIError], - max_retries=None, # retry forever, consider changing - on_exception_callback=_exception_callback, - ) - def completion(): - return client.chat.completions.create(**kwargs) - - return completion() - - @register_model("openai-chat-completions", "local-chat-completions") class OpenaiChatCompletionsLM(LM): def __init__( @@ -460,8 +466,12 @@ def generate_until(self, requests) -> List[str]: f"Expected repr(kwargs) to be of type repr(dict) but got {kwargs}" ) - response = oa_chat_completion( - client=self.client, messages=inps, model=self.model, **kwargs + response = oa_completion( + client=self.client, + chat=True, + messages=inps, + model=self.model, + **kwargs, ) for resp, (context, args_) in zip(response.choices, chunk): diff --git a/lm_eval/models/optimum_lm.py b/lm_eval/models/optimum_lm.py new file mode 100644 index 0000000000..0c1189b3fc --- /dev/null +++ b/lm_eval/models/optimum_lm.py @@ -0,0 +1,69 @@ +from importlib.util import find_spec +from pathlib import Path + +from lm_eval.api.registry import register_model +from lm_eval.models.huggingface import HFLM + + +@register_model("openvino") +class OptimumLM(HFLM): + """ + Optimum Intel provides a simple interface to optimize Transformer models and convert them to \ + OpenVINO™ Intermediate Representation (IR) format to accelerate end-to-end pipelines on \ + Intel® architectures using OpenVINO™ runtime. + """ + + def __init__( + self, + device="cpu", + **kwargs, + ) -> None: + if "backend" in kwargs: + # optimum currently only supports causal models + assert ( + kwargs["backend"] == "causal" + ), "Currently, only OVModelForCausalLM is supported." + + self.openvino_device = device + + super().__init__( + device=self.openvino_device, + backend=kwargs.get("backend", "causal"), + **kwargs, + ) + + def _create_model( + self, + pretrained: str, + revision="main", + dtype="auto", + trust_remote_code=False, + **kwargs, + ) -> None: + if not find_spec("optimum"): + raise Exception( + "package `optimum` is not installed. Please install it via `pip install optimum[openvino]`" + ) + else: + from optimum.intel.openvino import OVModelForCausalLM + + model_kwargs = kwargs if kwargs else {} + model_file = Path(pretrained) / "openvino_model.xml" + if model_file.exists(): + export = False + else: + export = True + kwargs["ov_config"] = { + "PERFORMANCE_HINT": "LATENCY", + "NUM_STREAMS": "1", + "CACHE_DIR": "", + } + + self._model = OVModelForCausalLM.from_pretrained( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + export=export, + device=self.openvino_device.upper(), + **model_kwargs, + ) diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index 6912428ed1..79aae35874 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -170,18 +170,12 @@ def _model_generate( stop: Optional[List[str]] = None, **kwargs, ): - if "do_sample" in kwargs.keys(): - kwargs.pop("do_sample") if generate: - # hf defaults - kwargs["skip_special_tokens"] = kwargs.get("skip_special_tokens", False) - kwargs["spaces_between_special_tokens"] = kwargs.get( - "spaces_between_special_tokens", False - ) + kwargs = self.modify_gen_kwargs(kwargs) sampling_params = SamplingParams(max_tokens=max_tokens, stop=stop, **kwargs) else: sampling_params = SamplingParams( - temperature=0, prompt_logprobs=2, max_tokens=1 + temperature=0, prompt_logprobs=1, max_tokens=1 ) if self.data_parallel_size > 1: requests = [list(x) for x in divide(requests, self.data_parallel_size)] @@ -438,3 +432,16 @@ def _parse_logprobs(tokens: List, outputs, ctxlen: int) -> Tuple[float, bool]: break return continuation_logprobs, is_greedy + + @staticmethod + def modify_gen_kwargs(kwargs: dict) -> dict: + # sampling_params + do_sample = kwargs.pop("do_sample", None) + if do_sample is False or "temperature" not in kwargs: + kwargs["temperature"] = 0.0 + # hf defaults + kwargs["skip_special_tokens"] = kwargs.get("skip_special_tokens", False) + kwargs["spaces_between_special_tokens"] = kwargs.get( + "spaces_between_special_tokens", False + ) + return kwargs diff --git a/lm_eval/prompts/__init__.py b/lm_eval/prompts/__init__.py index d8b62e7deb..c505113a3d 100644 --- a/lm_eval/prompts/__init__.py +++ b/lm_eval/prompts/__init__.py @@ -117,7 +117,7 @@ def apply(self, doc): # TODO need a way to process doc_to_choice if "doc_to_choice" in self.prompt_string: - raise "Not yet implemented to accept doc_to_choice" + raise Exception("Not yet implemented to accept doc_to_choice") text_string = utils.apply_template(doc_to_text, doc) target_string = utils.apply_template(doc_to_target, doc) diff --git a/lm_eval/tasks/__init__.py b/lm_eval/tasks/__init__.py index 93109de422..20d87c082e 100644 --- a/lm_eval/tasks/__init__.py +++ b/lm_eval/tasks/__init__.py @@ -1,214 +1,378 @@ import os -import yaml +import abc +import collections + +from functools import partial from typing import List, Union, Dict from lm_eval import utils -from lm_eval import prompts -from lm_eval.api.task import TaskConfig, Task, ConfigurableTask -from lm_eval.api.registry import ( - register_task, - register_group, - TASK_REGISTRY, - GROUP_REGISTRY, - ALL_TASKS, -) +from lm_eval.api.task import Task, ConfigurableTask import logging -# import python tasks -from .squadv2.task import SQuAD2 -from .scrolls.task import ( - QuALITY, - NarrativeQA, - ContractNLI, - GovReport, - SummScreenFD, - QMSum, -) - -eval_logger = utils.eval_logger - - -def register_configurable_task(config: Dict[str, str]) -> int: - SubClass = type( - config["task"] + "ConfigurableTask", - (ConfigurableTask,), - {"CONFIG": TaskConfig(**config)}, - ) - if "task" in config: - task_name = "{}".format(config["task"]) - register_task(task_name)(SubClass) +class TaskManager: + """TaskManager indexes all tasks from the default `lm_eval/tasks/` + and an optional directory if provided. - if "group" in config: - if config["group"] == config["task"]: - raise ValueError("task and group name cannot be the same") - elif type(config["group"]) == str: - group_name = [config["group"]] - else: - group_name = config["group"] + """ + def __init__( + self, + verbosity="INFO", + include_path=None + ) -> None: + + self.verbosity = verbosity + self.include_path = include_path + self.logger = utils.eval_logger + self.logger.setLevel(getattr(logging, f"{verbosity}")) + + self._task_index = self.initialize_tasks( + include_path=include_path + ) + self._all_tasks = sorted(list(self._task_index.keys())) - for group in group_name: - register_group(group)(SubClass) + self.task_group_map = collections.defaultdict(list) - return 0 + def initialize_tasks(self, include_path: str = None): + """Creates an dictionary of tasks index. + + :param include_path: str = None + An additional path to be searched for tasks + + :return + Dictionary of task names as key and task metadata + """ + all_paths = [os.path.dirname(os.path.abspath(__file__)) + "/"] + if include_path is not None: + if isinstance(include_path, str): + include_path = [include_path] + all_paths.extend(include_path) + task_index = {} + for task_dir in all_paths: + tasks = self._get_task_and_group(task_dir) + task_index = {**tasks, **task_index} -def register_configurable_group(config: Dict[str, str], yaml_path: str = None) -> int: - group = config["group"] - all_task_list = config["task"] - config_list = [task for task in all_task_list if type(task) != str] - task_list = [task for task in all_task_list if type(task) == str] - - for task_config in config_list: - task_config = utils.load_yaml_config(yaml_path, task_config) - var_configs = check_prompt_config( - { - **task_config, - **{"group": group}, - }, - yaml_path=os.path.dirname(yaml_path), + return task_index + + @property + def all_tasks(self): + return self._all_tasks + + @property + def task_index(self): + return self._task_index + + def match_tasks(self, task_list): + return utils.pattern_match( + task_list, self.all_tasks ) - for config in var_configs: - register_configurable_task(config) - - task_names = utils.pattern_match(task_list, ALL_TASKS) - for task in task_names: - if (task in TASK_REGISTRY) or (task in GROUP_REGISTRY): - if group in GROUP_REGISTRY: - GROUP_REGISTRY[group].append(task) + + def _name_is_registered(self, name): + if name in self.all_tasks: + return True + return False + + def _name_is_task(self, name): + if self._name_is_registered(name) and ("task" in self.task_index[name]["type"]): + return True + return False + + def _name_is_group(self, name): + if self._name_is_registered(name) and (self.task_index[name]["type"] == "group"): + return True + return False + + def _name_is_python_task(self, name): + if self._name_is_registered(name) and (self.task_index[name]["type"] == "python_task"): + return True + return False + + def _config_is_task(self, config): + if ("task" in config) and isinstance(config["task"], str): + return True + return False + + def _config_is_group(self, config): + if ("task" in config) and isinstance(config["task"], list): + return True + return False + + def _config_is_python_task(self, config): + if "class" in config: + return True + return False + + def _get_yaml_path(self, name): + assert name in self.task_index + return self.task_index[name]["yaml_path"] + + def _get_config(self, name): + assert name in self.task_index + yaml_path = self._get_yaml_path(name) + if yaml_path == -1: + return {} + else: + return utils.load_yaml_config(yaml_path, mode="full") + + def _get_tasklist(self, name): + assert self._name_is_task(name) == False + return self.task_index[name]["task"] + + def _process_alias(self, config, group=None): + # If the group is not the same as the original + # group which the group alias was intended for, + # Set the group_alias to None instead. + if ("group_alias" in config) and ("group" in config) and group is not None: + if config["group"] != group: + config["group_alias"] = None + return config + + def _load_individual_task_or_group( + self, + name_or_config: Union[str, dict] = None, + parent_name: str = None, + update_config: dict = None, + yaml_path: str = None, + ) -> ConfigurableTask: + def load_task(config, task, group=None, yaml_path=None): + if "include" in config: + assert yaml_path is not None + config.update( + utils.load_yaml_config( + yaml_path, + yaml_config={"include": config.pop("include")}, + mode="full", + ) + ) + if self._config_is_python_task(config): + task_object = config["class"]() + else: + config = self._process_alias(config, group=group) + task_object = ConfigurableTask(config=config) + if group is not None: + task_object = (group, task_object) + return {task: task_object} + + if isinstance(name_or_config, str): + if update_config is not None: + # Process name_or_config as a dict instead + name_or_config = {"task": name_or_config, **update_config} + elif self._name_is_task(name_or_config): + task_config = self._get_config(name_or_config) + return load_task(task_config, task=name_or_config, group=parent_name) else: - GROUP_REGISTRY[group] = [task] - ALL_TASKS.add(group) + group_name = name_or_config + subtask_list = self._get_tasklist(name_or_config) + if subtask_list == -1: + group_config = self._get_config(name_or_config) + subtask_list = group_config["task"] + + # This checks if we're at the root. + if parent_name is None: + group_config = self._get_config(name_or_config) + if set(group_config.keys()) > set(["task", "group"]): + update_config = { + k:v for k,v in group_config.items() if k not in ["task", "group"] + } + yaml_path = self._get_yaml_path(group_name) - return 0 + if (update_config is not None) and ("group_alias" in update_config): + group_name = update_config["group_alias"] + update_config.pop("group_alias") + if isinstance(name_or_config, dict): -def check_prompt_config( - config: Dict[str, str], yaml_path: str = None -) -> List[Dict[str, str]]: - all_configs = [] - if "use_prompt" in config: - prompt_list = prompts.load_prompt_list( - use_prompt=config["use_prompt"], - dataset_name=config["dataset_path"], - subset_name=config["dataset_name"] if "dataset_name" in config else None, - yaml_path=yaml_path, - ) - for idx, prompt_variation in enumerate(prompt_list): - all_configs.append( - { - **config, - **{"use_prompt": prompt_variation}, - **{ - "task": "_".join( - [ - config["task"] - if "task" in config - else get_task_name_from_config(config), - prompt_variation.split("/")[-1] - if ".yaml" in prompt_variation - else prompt_variation, - ] - ) - }, - **{"output_type": "generate_until"}, + if update_config is not None: + name_or_config={ + **name_or_config, + **update_config, } - ) - else: - all_configs.append(config) - return all_configs - -def get_task_name_from_config(task_config: Dict[str, str]) -> str: - if "dataset_name" in task_config: - return "{dataset_path}_{dataset_name}".format(**task_config) - else: - return "{dataset_path}".format(**task_config) + if self._config_is_task(name_or_config): + name = name_or_config["task"] + # If the name is registered as a group + # if self._name_is_task(name) is False: + if self._name_is_group(name): + group_name = name + update_config = {k:v for k,v in name_or_config.items() if k != "task"} + subtask_list = self._get_tasklist(name) + if subtask_list == -1: + subtask_list = self._get_config(name)["task"] + else: + if self._name_is_registered(name): + base_task_config = self._get_config(name) + + # Check if this is a duplicate. + if parent_name is not None: + name_or_config["group"] = parent_name + num_duplicate = len(list(filter(lambda x: x.startswith(name), self.task_group_map[parent_name]))) + if num_duplicate > 0: + name = f"{name}-{num_duplicate}" + self.task_group_map[parent_name].append(name) + + task_config={ + **base_task_config, + **name_or_config, + } + else: + task_config = name_or_config + return load_task(task_config, task=name, group=parent_name, yaml_path=yaml_path) + else: + group_name = name_or_config["group"] + subtask_list = name_or_config["task"] + # update_config = {k:v for k,v in name_or_config.items() if k != "task"} + if set(name_or_config.keys()) > set(["task", "group"]): + update_config = { + k:v for k,v in name_or_config.items() if k not in ["task", "group"] + } + all_subtasks = {} + if (parent_name is not None): + all_subtasks = {group_name: (parent_name, None)} -def include_task_folder(task_dir: str, register_task: bool = True) -> None: - """ - Calling this function - """ + fn = partial(self._load_individual_task_or_group, parent_name=group_name, update_config=update_config, yaml_path=yaml_path) + all_subtasks = {**all_subtasks, **dict(collections.ChainMap(*map(fn, subtask_list)))} + return all_subtasks - # Track whether any tasks failed during loading - import_fail = False - for root, subdirs, file_list in os.walk(task_dir): - # if (subdirs == [] or subdirs == ["__pycache__"]) and (len(file_list) > 0): - for f in file_list: - if f.endswith(".yaml"): - yaml_path = os.path.join(root, f) - try: - config = utils.load_yaml_config(yaml_path) - - if "task" not in config: - continue - - all_configs = check_prompt_config( - config, yaml_path=os.path.dirname(yaml_path) - ) - for config in all_configs: - if register_task: - if type(config["task"]) == str: - register_configurable_task(config) - else: - if type(config["task"]) == list: - register_configurable_group(config, yaml_path) - - # Log this silently and show it only when - # the user defines the appropriate verbosity. - except (ImportError, ModuleNotFoundError) as e: - import_fail = True - eval_logger.debug( - f"{yaml_path}: {e}. Config will not be added to registry." - ) - except Exception as error: - import traceback - - eval_logger.warning( - "Unexpected error loading config in\n" - f" {yaml_path}\n" - " Config will not be added to registry\n" - f" Error: {error}\n" - f" Traceback: {traceback.format_exc()}" - ) - if import_fail: - eval_logger.warning( - "Some tasks could not be loaded due to missing dependencies." - " Run with `--verbosity DEBUG` for full details." - ) - return 0 + def load_task_or_group(self, task_list: Union[str, list] = None) -> dict: + """Loads a dictionary of task objects from a list + :param task_list: Union[str, list] = None + Single string or list of string of task names to be loaded -def include_path(task_dir): - include_task_folder(task_dir) - # Register Benchmarks after all tasks have been added - include_task_folder(task_dir, register_task=False) - return 0 + :return + Dictionary of task objects + """ + if isinstance(task_list, str): + task_list = [task_list] + all_loaded_tasks = dict( + collections.ChainMap( + *map( + self._load_individual_task_or_group, + task_list + ) + ) + ) + return all_loaded_tasks + + def load_config(self, config: Dict): + return self._load_individual_task_or_group(config) + + def _get_task_and_group(self, task_dir: str): + """Creates an dictionary of tasks index with the following metadata, + - `type`, that can be either `task`, `python_task`, or `group`. + `task` refer to regular task configs, `python_task` are special + yaml files that only consists of `task` and `class` parameters. + `group` are group configs. + - `yaml_path`, path to the yaml file. If the entry is a `group` that + was configured through a task config, the yaml_path will be -1 + and all subtasks will be listed in `task` (see below) + - `task`, reserved for entries with `type` as `group`. This will list + all subtasks. When a group config is created (as opposed to task + config having `group` parameter set), this will be set to -1 to + avoid recursive indexing. The whole list of subtasks will be loaded + at evaluation. + + :param task_dir: str + A directory to check for tasks + + :return + Dictionary of task names as key and task metadata + """ + tasks_and_groups = collections.defaultdict() + for root, _, file_list in os.walk(task_dir): + for f in file_list: + if f.endswith(".yaml"): + yaml_path = os.path.join(root, f) + config = utils.load_yaml_config(yaml_path, mode="simple") + if self._config_is_python_task(config): + # This is a python class config + tasks_and_groups[config["task"]] = { + "type": "python_task", + "yaml_path": yaml_path, + } + elif self._config_is_group(config): + # This is a group config + tasks_and_groups[config["group"]] = { + "type": "group", + "task": -1, # This signals that + # we don't need to know + # the task list for indexing + # as it can be loaded + # when called. + "yaml_path": yaml_path, + } -def initialize_tasks(verbosity="INFO"): - eval_logger.setLevel(getattr(logging, f"{verbosity}")) + # # Registered the level 1 tasks from a group config + # for config in config["task"]: + # if isinstance(config, dict) and self._config_is_task(config): + # task = config["task"] + # tasks_and_groups[task] = { + # "type": "task", + # "yaml_path": yaml_path, + # } + + elif self._config_is_task(config): + # This is a task config + task = config["task"] + tasks_and_groups[task] = { + "type": "task", + "yaml_path": yaml_path, + } - task_dir = os.path.dirname(os.path.abspath(__file__)) + "/" - include_path(task_dir) + if "group" in config: + groups = config["group"] + if isinstance(config["group"], str): + groups = [groups] + + for group in groups: + if group not in tasks_and_groups: + tasks_and_groups[group] = { + "type": "group", + "task": [task], + "yaml_path": -1, + } + else: + tasks_and_groups[group]["task"].append(task) + else: + self.logger.debug(f"File {f} in {root} could not be loaded") + + return tasks_and_groups +def include_path(task_dir): + logger = utils.eval_logger + logger.setLevel(getattr(logging, "INFO")) + logger.info( + "To still use tasks loaded from args.include_path," + "see an example of the new TaskManager API in https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/interface.md#external-library-usage" + ) + return 0 -def get_task(task_name, config): - try: - return TASK_REGISTRY[task_name](config=config) - except KeyError: - eval_logger.info("Available tasks:") - eval_logger.info(list(TASK_REGISTRY) + list(GROUP_REGISTRY)) - raise KeyError(f"Missing task {task_name}") +def initialize_tasks(verbosity="INFO"): + logger = utils.eval_logger + logger.setLevel(getattr(logging, f"{verbosity}")) + logger.info( + "lm_eval.tasks.initialize_tasks() is deprecated and no longer necessary. " + "It will be removed in v0.4.2 release. " + "TaskManager will instead be used." + ) + return 0 +def get_task_name_from_config(task_config: Dict[str, str]) -> str: + if "task" in task_config: + return task_config["task"] + if "dataset_name" in task_config: + return "{dataset_path}_{dataset_name}".format(**task_config) + else: + return "{dataset_path}".format(**task_config) def get_task_name_from_object(task_object): - for name, class_ in TASK_REGISTRY.items(): - if class_ is task_object: - return name + if hasattr(task_object, "config"): + return task_object._config["task"] # TODO: scrap this # this gives a mechanism for non-registered tasks to have a custom name anyways when reporting @@ -218,54 +382,40 @@ def get_task_name_from_object(task_object): else type(task_object).__name__ ) +def get_task_dict(task_name_list: List[Union[str, Dict, Task]], task_manager: TaskManager = None): + """Creates a dictionary of task objects from either a name of task, config, or prepared Task object. -# TODO: pass num_fewshot and other cmdline overrides in a better way -def get_task_dict(task_name_list: List[Union[str, Dict, Task]], **kwargs): - config = {**kwargs} + :param task_name_list: List[Union[str, Dict, Task]] + Name of model or LM object, see lm_eval.models.get_model + :param task_manager: TaskManager = None + A TaskManager object that stores indexed tasks. If not set, + task_manager will load one. This should be set by the user + if there are additional paths that want to be included + via `include_path` - task_name_from_registry_dict = {} + :return + Dictionary of task objects + """ + task_name_from_string_dict = {} task_name_from_config_dict = {} task_name_from_object_dict = {} - if type(task_name_list) != list: + if isinstance(task_name_list, str): task_name_list = [task_name_list] - for task_element in task_name_list: - if isinstance(task_element, str): - if task_element in GROUP_REGISTRY: - group_name = task_element - for task_name in GROUP_REGISTRY[task_element]: - if task_name not in task_name_from_registry_dict: - task_obj = get_task_dict(task_name) - if task_name in task_obj.keys(): - task_dict = { - task_name: (group_name, task_obj[task_name]), - } - else: - task_dict = { - task_name: (group_name, None), - **task_obj, - } + string_task_name_list = [task for task in task_name_list if isinstance(task, str)] + others_task_name_list = [task for task in task_name_list if ~isinstance(task, str)] + if len(string_task_name_list) > 0: + if task_manager is None: + task_manager = TaskManager() - task_name_from_registry_dict = { - **task_name_from_registry_dict, - **task_dict, - } - else: - task_name = task_element - if task_name not in task_name_from_registry_dict: - task_name_from_registry_dict = { - **task_name_from_registry_dict, - task_name: get_task(task_name=task_element, config=config), - } + task_name_from_string_dict = task_manager.load_task_or_group(string_task_name_list) - elif isinstance(task_element, dict): - task_element.update(config) + for task_element in others_task_name_list: + if isinstance(task_element, dict): task_name_from_config_dict = { **task_name_from_config_dict, - get_task_name_from_config(task_element): ConfigurableTask( - config=task_element - ), + **task_manager.load_config(config=task_element), } elif isinstance(task_element, Task): @@ -274,11 +424,11 @@ def get_task_dict(task_name_list: List[Union[str, Dict, Task]], **kwargs): get_task_name_from_object(task_element): task_element, } - assert set(task_name_from_registry_dict.keys()).isdisjoint( + assert set(task_name_from_string_dict.keys()).isdisjoint( set(task_name_from_object_dict.keys()) ) return { - **task_name_from_registry_dict, + **task_name_from_string_dict, **task_name_from_config_dict, **task_name_from_object_dict, } diff --git a/lm_eval/tasks/arc/arc_easy.yaml b/lm_eval/tasks/arc/arc_easy.yaml index e551863b7e..3eda1d893a 100644 --- a/lm_eval/tasks/arc/arc_easy.yaml +++ b/lm_eval/tasks/arc/arc_easy.yaml @@ -1,7 +1,7 @@ group: - ai2_arc task: arc_easy -dataset_path: ai2_arc +dataset_path: allenai/ai2_arc dataset_name: ARC-Easy output_type: multiple_choice training_split: train diff --git a/lm_eval/tasks/bbh/_generate_configs.py b/lm_eval/tasks/bbh/_generate_configs.py index 18a55c705a..0d085a1d0a 100644 --- a/lm_eval/tasks/bbh/_generate_configs.py +++ b/lm_eval/tasks/bbh/_generate_configs.py @@ -28,7 +28,7 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our other YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) base_doc_to_text = "Q: {{input}}\nA:" @@ -70,7 +70,7 @@ def parse_args(): file_save_path = args.save_prefix_path + f"/{task}.yaml" utils.eval_logger.info(f"Saving yaml for subset {task} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, diff --git a/lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml b/lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml index ff61bdbd6d..4608ea6e51 100644 --- a/lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml +++ b/lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml @@ -28,3 +28,4 @@ filter_list: num_fewshot: 0 metadata: version: 2.0 + num_fewshot: 3 # controls what is printed in n-shot diff --git a/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml b/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml index 43967866f5..f84641eee2 100644 --- a/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml +++ b/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml @@ -19,3 +19,4 @@ generation_kwargs: num_fewshot: 0 metadata: version: 1.0 + num_fewshot: 3 # will be printed in results table diff --git a/lm_eval/tasks/belebele/_generate_configs.py b/lm_eval/tasks/belebele/_generate_configs.py index 54274a419e..fd96034afb 100644 --- a/lm_eval/tasks/belebele/_generate_configs.py +++ b/lm_eval/tasks/belebele/_generate_configs.py @@ -27,13 +27,13 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our other YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) if args.cot_prompt_path is not None: import json - with open(args.cot_prompt_path) as f: + with open(args.cot_prompt_path, encoding="utf-8") as f: cot_file = json.load(f) def query(): @@ -42,7 +42,7 @@ def query(): print(query()) languages = [split["split"] for split in query()] - for lang in tqdm(languages): + for lang in tqdm([lang for lang in languages if "default" not in lang]): yaml_dict = { "include": base_yaml_name, "task": f"belebele_{args.task_prefix}_{lang}" @@ -54,7 +54,7 @@ def query(): file_save_path = args.save_prefix_path + f"_{lang}.yaml" logging.info(f"Saving yaml for subset {lang} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, diff --git a/lm_eval/tasks/belebele/belebele_default.yaml b/lm_eval/tasks/belebele/belebele_default.yaml deleted file mode 100644 index d47f8d8af2..0000000000 --- a/lm_eval/tasks/belebele/belebele_default.yaml +++ /dev/null @@ -1,4 +0,0 @@ -"fewshot_split": "default" -"include": "_default_template_yaml" -"task": "belebele_default" -"test_split": "default" diff --git a/lm_eval/tasks/benchmarks/flan/yaml_templates/held_in_template_yaml b/lm_eval/tasks/benchmarks/flan/_held_in_template_yaml similarity index 85% rename from lm_eval/tasks/benchmarks/flan/yaml_templates/held_in_template_yaml rename to lm_eval/tasks/benchmarks/flan/_held_in_template_yaml index d3d9f52633..c19b47cdae 100644 --- a/lm_eval/tasks/benchmarks/flan/yaml_templates/held_in_template_yaml +++ b/lm_eval/tasks/benchmarks/flan/_held_in_template_yaml @@ -1,5 +1,6 @@ output_type: generate_until -validation_split: validation +test_split: null +doc_to_choice: null metric_list: - metric: exact_match aggregation: mean diff --git a/lm_eval/tasks/benchmarks/flan/flan_anli.yaml b/lm_eval/tasks/benchmarks/flan/flan_anli.yaml deleted file mode 100644 index 21278e1f69..0000000000 --- a/lm_eval/tasks/benchmarks/flan/flan_anli.yaml +++ /dev/null @@ -1,17 +0,0 @@ -group: flan_anli -task: - - include: yaml_templates/held_in_template_yaml - task: anli_r1 - dataset_path: anli - use_prompt: prompt_templates/anli.yaml:* - validation_split: dev_r1 - - include: yaml_templates/held_in_template_yaml - task: anli_r2 - dataset_path: anli - use_prompt: prompt_templates/anli.yaml:* - validation_split: dev_r2 - - include: yaml_templates/held_in_template_yaml - task: anli_r3 - dataset_path: anli - use_prompt: prompt_templates/anli.yaml:* - validation_split: dev_r3 diff --git a/lm_eval/tasks/benchmarks/flan/flan_arc.yaml b/lm_eval/tasks/benchmarks/flan/flan_arc.yaml deleted file mode 100644 index 4e73b7ce53..0000000000 --- a/lm_eval/tasks/benchmarks/flan/flan_arc.yaml +++ /dev/null @@ -1,14 +0,0 @@ -group: flan_arc -task: - - include: yaml_templates/held_in_template_yaml - task: arc_easy - dataset_path: ai2_arc - dataset_name: ARC-Easy - use_prompt: prompt_templates/arc.yaml:* - validation_split: validation - - include: yaml_templates/held_in_template_yaml - task: arc_challenge - dataset_path: ai2_arc - dataset_name: ARC-Challenge - use_prompt: prompt_templates/arc.yaml:* - validation_split: validation diff --git a/lm_eval/tasks/benchmarks/flan/flan_boolq.yaml b/lm_eval/tasks/benchmarks/flan/flan_boolq.yaml deleted file mode 100644 index 8fe36cd53c..0000000000 --- a/lm_eval/tasks/benchmarks/flan/flan_boolq.yaml +++ /dev/null @@ -1,7 +0,0 @@ -group: flan_boolq -task: - - include: yaml_templates/held_in_template_yaml - dataset_path: super_glue - dataset_name: boolq - use_prompt: prompt_templates/boolq.yaml:* - validation_split: validation diff --git a/lm_eval/tasks/benchmarks/flan/flan_cot.yaml b/lm_eval/tasks/benchmarks/flan/flan_cot.yaml deleted file mode 100644 index 10102d2461..0000000000 --- a/lm_eval/tasks/benchmarks/flan/flan_cot.yaml +++ /dev/null @@ -1,11 +0,0 @@ -group: flan_cot -task: - - include: yaml_templates/cot_template_yaml - dataset_path: gsmk - dataset_name: boolq - use_prompt: promptsource:* - validation_split: validation - - include: yaml_templates/cot_template_yaml - dataset_path: EleutherAI/asdiv - use_prompt: promptsource:* - validation_split: validation diff --git a/lm_eval/tasks/benchmarks/flan/flan_held_in.yaml b/lm_eval/tasks/benchmarks/flan/flan_held_in.yaml index 5465b58c24..5796713506 100644 --- a/lm_eval/tasks/benchmarks/flan/flan_held_in.yaml +++ b/lm_eval/tasks/benchmarks/flan/flan_held_in.yaml @@ -1,6 +1,331 @@ group: flan_held_in +group_alias: Flan (Held-In) task: - - flan_boolq - - flan_rte - - flan_anli - - flan_arc + # ANLI R1 + - group: anli_r1_flan + group_alias: ANLI R1 + task: + - task: anli_r1 + task_alias: prompt-0 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r1 + task_alias: prompt-1 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r1 + task_alias: prompt-2 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r1 + task_alias: prompt-3 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r1 + task_alias: prompt-4 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r1 + task_alias: prompt-5 + include: _held_in_template_yaml + doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r1 + task_alias: prompt-6 + include: _held_in_template_yaml + doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r1 + task_alias: prompt-7 + include: _held_in_template_yaml + doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r1 + task_alias: prompt-8 + include: _held_in_template_yaml + doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + # ANLI R2 + - group: anli_r2_flan + group_alias: ANLI R2 + task: + - task: anli_r2 + task_alias: prompt-0 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r2 + task_alias: prompt-1 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r2 + task_alias: prompt-2 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r2 + task_alias: prompt-3 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r2 + task_alias: prompt-4 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r2 + task_alias: prompt-5 + include: _held_in_template_yaml + doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r2 + task_alias: prompt-6 + include: _held_in_template_yaml + doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r2 + task_alias: prompt-7 + include: _held_in_template_yaml + doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r2 + task_alias: prompt-8 + include: _held_in_template_yaml + doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + # ANLI R3 + - group: anli_r3_flan + group_alias: ANLI R3 + task: + - task: anli_r3 + task_alias: prompt-0 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r3 + task_alias: prompt-1 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r3 + task_alias: prompt-2 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r3 + task_alias: prompt-3 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r3 + task_alias: prompt-4 + include: _held_in_template_yaml + doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r3 + task_alias: prompt-5 + include: _held_in_template_yaml + doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r3 + task_alias: prompt-6 + include: _held_in_template_yaml + doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r3 + task_alias: prompt-7 + include: _held_in_template_yaml + doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + - task: anli_r3 + task_alias: prompt-8 + include: _held_in_template_yaml + doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" + doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" + # Arc Easy + - group: arc_easy_flan + group_alias: Arc Easy + task: + - task: arc_easy + task_alias: prompt-0 + include: _held_in_template_yaml + doc_to_text: "{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_easy + task_alias: prompt-1 + include: _held_in_template_yaml + doc_to_text: "Question: {{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}\nAnswer:" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_easy + task_alias: prompt-2 + include: _held_in_template_yaml + doc_to_text: "Question: {{question}}\n\nWhat is the correct answer to the question from the following choices?\nOPTIONS:\n- {{choices.text|join('\n- ')}}" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_easy + task_alias: prompt-3 + include: _held_in_template_yaml + doc_to_text: "Q: {{question}}\nWhat is the correct answer to this question?\nOPTIONS:\n- {{choices.text|join('\n- ')}}...A:" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_easy + task_alias: prompt-4 + include: _held_in_template_yaml + doc_to_text: "Choose your answer?\n\n{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_easy + task_alias: prompt-5 + include: _held_in_template_yaml + doc_to_text: "Answer the question\n\n{{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_easy + task_alias: prompt-6 + include: _held_in_template_yaml + doc_to_text: "{{question}}\n\nPick the answer from these options\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + # Arc Challenge + - group: arc_challenge_flan + group_alias: Arc Challenge + task: + - task: arc_challenge + task_alias: prompt-0 + include: _held_in_template_yaml + doc_to_text: "{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_challenge + task_alias: prompt-1 + include: _held_in_template_yaml + doc_to_text: "Question: {{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}\nAnswer:" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_challenge + task_alias: prompt-2 + include: _held_in_template_yaml + doc_to_text: "Question: {{question}}\n\nWhat is the correct answer to the question from the following choices?\nOPTIONS:\n- {{choices.text|join('\n- ')}}" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_challenge + task_alias: prompt-3 + include: _held_in_template_yaml + doc_to_text: "Q: {{question}}\nWhat is the correct answer to this question?\nOPTIONS:\n- {{choices.text|join('\n- ')}}...A:" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_challenge + task_alias: prompt-4 + include: _held_in_template_yaml + doc_to_text: "Choose your answer?\n\n{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_challenge + task_alias: prompt-5 + include: _held_in_template_yaml + doc_to_text: "Answer the question\n\n{{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + - task: arc_challenge + task_alias: prompt-6 + include: _held_in_template_yaml + doc_to_text: "{{question}}\n\nPick the answer from these options\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" + doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" + # BoolQ + - group: boolq_flan + group_alias: BoolQ + task: + - task: boolq + task_alias: prompt-0 + include: _held_in_template_yaml + doc_to_text: "{{passage}}\n\nCan we conclude that {{question}}?\n\nOPTIONS:\n- no\n- yes" + doc_to_target: "{{['no', 'yes'][label]}}" + - task: boolq + task_alias: prompt-1 + include: _held_in_template_yaml + doc_to_text: "{{passage}}\n\nIs it true that {{question}}?\n\nOPTIONS:\n- no\n- yes" + doc_to_target: "{{['no', 'yes'][label]}}" + - task: boolq + task_alias: prompt-2 + include: _held_in_template_yaml + doc_to_text: "{{passage}}\n\n{{question}}?\n\nOPTIONS:\n- no\n- yes" + doc_to_target: "{{['no', 'yes'][label]}}" + - task: boolq + task_alias: prompt-3 + include: _held_in_template_yaml + doc_to_text: "Text: {{passage}}\n\nQuestion: {{question}}?\n\nOPTIONS:\n- no\n- yes" + doc_to_target: "{{['no', 'yes'][label]}}" + - task: boolq + task_alias: prompt-4 + include: _held_in_template_yaml + doc_to_text: "{{passage}}\n\nWhat's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes" + doc_to_target: "{{['no', 'yes'][label]}}" + - task: boolq + task_alias: prompt-5 + include: _held_in_template_yaml + doc_to_text: "{{passage}}\nBased on the above text what's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes" + doc_to_target: "{{['no', 'yes'][label]}}" + - task: boolq + task_alias: prompt-6 + include: _held_in_template_yaml + doc_to_text: "{{passage}}\nAnswer this question making sure that the answer is supposed by the text: {{question}}?\n\nOPTIONS:\n- no\n- yes" + doc_to_target: "{{['no', 'yes'][label]}}" + - task: boolq + task_alias: prompt-7 + include: _held_in_template_yaml + doc_to_text: "{{passage}}\n\nIs the following statement correct based on the text\n\n{{question}}\n\nOPTIONS:\n- no\n- yes" + doc_to_target: "{{['no', 'yes'][label]}}" + - task: boolq + task_alias: prompt-8 + include: _held_in_template_yaml + doc_to_text: "{{passage}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes" + doc_to_target: "{{['no', 'yes'][label]}}" + - task: boolq + task_alias: prompt-9 + include: _held_in_template_yaml + doc_to_text: "Is it true that {{question}} based on the following text?\n\n{{passage}}\n\nOPTIONS:\n- no\n- yes" + doc_to_target: "{{['no', 'yes'][label]}}" + # RTE + - group: rte_flan + group_alias: RTE + task: + - task: rte + task_alias: prompt-0 + include: _held_in_template_yaml + doc_to_text: "{{sentence1}}\n\nQuestion with options: Based on the paragraph above can we conclude that \"{{sentence2}}\"?\n\nOPTIONS:\n- yes\n- no" + doc_to_target: "{{['yes', 'no'][label]}}" + - task: rte + task_alias: prompt-1 + include: _held_in_template_yaml + doc_to_text: "{{sentence1}}\n\nBased on that paragraph can we conclude that the sentence below is true?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no" + doc_to_target: "{{['yes', 'no'][label]}}" + - task: rte + task_alias: prompt-2 + include: _held_in_template_yaml + doc_to_text: "{{sentence1}}\n\nQ with options: Can we draw the following conclusion?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no" + doc_to_target: "{{['yes', 'no'][label]}}" + - task: rte + task_alias: prompt-3 + include: _held_in_template_yaml + doc_to_text: "{{sentence1}}\nDoes this next sentence follow, given the preceding text?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no" + doc_to_target: "{{['yes', 'no'][label]}}" + - task: rte + task_alias: prompt-4 + include: _held_in_template_yaml + doc_to_text: "{{sentence1}}\nOPTIONS:\n- yes\n- no\nQuestion: Can we infer the following?\n{{sentence2}}" + doc_to_target: "{{['yes', 'no'][label]}}" + - task: rte + task_alias: prompt-5 + include: _held_in_template_yaml + doc_to_text: "Read the following paragraph and determine if the hypothesis is true. Select from options at the end:\n\n{{sentence1}}\n\nHypothesis: {{sentence2}}\nOPTIONS:\n- yes\n- no\nThe answer is" + doc_to_target: "{{['yes', 'no'][label]}}" + - task: rte + task_alias: prompt-6 + include: _held_in_template_yaml + doc_to_text: "Read the text and determine if the sentence is true:\n\n{{sentence1}}\n\nSentence: {{sentence2}}\nOPTIONS:\n- yes\n- no\nA:" + doc_to_target: "{{['yes', 'no'][label]}}" + - task: rte + task_alias: prompt-7 + include: _held_in_template_yaml + doc_to_text: "Question with options: can we draw the following hypothesis from the context? \n\nContext:\n\n{{sentence1}}\n\nHypothesis: {{sentence2}}\nOPTIONS:\n- yes\n- no\nA:" + doc_to_target: "{{['yes', 'no'][label]}}" + - task: rte + task_alias: prompt-8 + include: _held_in_template_yaml + doc_to_text: "Determine if the sentence is true based on the text below. Choose from options.\n{{sentence2}}\n\n{{sentence1}}\nOPTIONS:\n- yes\n- no" + doc_to_target: "{{['yes', 'no'][label]}}" diff --git a/lm_eval/tasks/benchmarks/flan/flan_held_in_yaml b/lm_eval/tasks/benchmarks/flan/flan_held_in_yaml deleted file mode 100644 index a31a942e6a..0000000000 --- a/lm_eval/tasks/benchmarks/flan/flan_held_in_yaml +++ /dev/null @@ -1,39 +0,0 @@ -group: flan_held_in -task: - - include: flan/yaml_templates/held_in_template_yaml - dataset_path: super_glue - dataset_name: boolq - use_prompt: flan/prompt_templates/boolq.yaml:* - validation_split: validation - - include: flan/yaml_templates/held_in_template_yaml - dataset_path: super_glue - dataset_name: rte - use_prompt: flan/prompt_templates/rte.yaml:* - validation_split: validation - - include: flan/yaml_templates/held_in_template_yaml - task: anli_r1 - dataset_path: anli - use_prompt: flan/prompt_templates/anli.yaml:* - validation_split: dev_r1 - - include: flan/yaml_templates/held_in_template_yaml - task: anli_r2 - dataset_path: anli - use_prompt: flan/prompt_templates/anli.yaml:* - validation_split: dev_r2 - - include: flan/yaml_templates/held_in_template_yaml - task: anli_r3 - dataset_path: anli - use_prompt: flan/prompt_templates/anli.yaml:* - validation_split: dev_r3 - - include: flan/yaml_templates/held_in_template_yaml - task: arc_easy - dataset_path: ai2_arc - dataset_name: ARC-Easy - use_prompt: flan/prompt_templates/arc.yaml:* - validation_split: validation - - include: flan/yaml_templates/held_in_template_yaml - task: arc_challenge - dataset_path: ai2_arc - dataset_name: ARC-Challenge - use_prompt: flan/prompt_templates/arc.yaml:* - validation_split: validation diff --git a/lm_eval/tasks/benchmarks/flan/flan_held_out.yaml b/lm_eval/tasks/benchmarks/flan/flan_held_out.yaml index e1429cdac7..cf806b8821 100644 --- a/lm_eval/tasks/benchmarks/flan/flan_held_out.yaml +++ b/lm_eval/tasks/benchmarks/flan/flan_held_out.yaml @@ -1,10 +1,10 @@ group: flan_held_out task: # BBH - - bbh_flan_zeroshot - - bbh_flan_fewshot - - bbh_flan_cot_fewshot - - bbh_flan_cot_zeroshot + - bbh_zeroshot + - bbh_fewshot + - bbh_cot_fewshot + - bbh_cot_zeroshot # MMLU - mmlu - mmlu_flan_n_shot_generative diff --git a/lm_eval/tasks/benchmarks/flan/flan_rte.yaml b/lm_eval/tasks/benchmarks/flan/flan_rte.yaml deleted file mode 100644 index a4f407361d..0000000000 --- a/lm_eval/tasks/benchmarks/flan/flan_rte.yaml +++ /dev/null @@ -1,7 +0,0 @@ -group: flan_rte -task: - - include: yaml_templates/held_in_template_yaml - dataset_path: super_glue - dataset_name: rte - use_prompt: prompt_templates/rte.yaml:* - validation_split: validation diff --git a/lm_eval/tasks/benchmarks/flan/prompt_templates/anli.yaml b/lm_eval/tasks/benchmarks/flan/prompt_templates/anli.yaml deleted file mode 100644 index 6ff7884013..0000000000 --- a/lm_eval/tasks/benchmarks/flan/prompt_templates/anli.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Flan Prompt Templates -prompts: - "template-0": - doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is" - doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - "template-1": - doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" - doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - "template-2": - doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" - doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - "template-3": - doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" - doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - "template-4": - doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:" - doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - "template-5": - doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n" - doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - "template-6": - doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" - doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - "template-7": - doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" - doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - "template-8": - doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" - doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" diff --git a/lm_eval/tasks/benchmarks/flan/prompt_templates/arc.yaml b/lm_eval/tasks/benchmarks/flan/prompt_templates/arc.yaml deleted file mode 100644 index 4ee34e6592..0000000000 --- a/lm_eval/tasks/benchmarks/flan/prompt_templates/arc.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Flan Prompt Templates -prompts: - "template-0": - doc_to_text: "{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" - doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - "template-1": - doc_to_text: "Question: {{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}\nAnswer:" - doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - "template-2": - doc_to_text: "Question: {{question}}\n\nWhat is the correct answer to the question from the following choices?\nOPTIONS:\n- {{choices.text|join('\n- ')}}" - doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - "template-3": - doc_to_text: "Q: {{question}}\nWhat is the correct answer to this question?\nOPTIONS:\n- {{choices.text|join('\n- ')}}...A:" - doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - "template-4": - doc_to_text: "Choose your answer?\n\n{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" - doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - "template-5": - doc_to_text: "Answer the question\n\n{{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}" - doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - "template-6": - doc_to_text: "{{question}}\n\nPick the answer from these options\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" - doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" diff --git a/lm_eval/tasks/benchmarks/flan/prompt_templates/boolq.yaml b/lm_eval/tasks/benchmarks/flan/prompt_templates/boolq.yaml deleted file mode 100644 index f8c8ebfca8..0000000000 --- a/lm_eval/tasks/benchmarks/flan/prompt_templates/boolq.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Flan Prompt Templates -prompts: - "template-0": - doc_to_text: "{{passage}}\n\nCan we conclude that {{question}}?\n\nOPTIONS:\n- no\n- yes" - doc_to_target: "{{['no', 'yes'][label]}}" - "template-1": - doc_to_text: "{{passage}}\n\nIs it true that {{question}}?\n\nOPTIONS:\n- no\n- yes" - doc_to_target: "{{['no', 'yes'][label]}}" - "template-2": - doc_to_text: "{{passage}}\n\n{{question}}?\n\nOPTIONS:\n- no\n- yes" - doc_to_target: "{{['no', 'yes'][label]}}" - "template-3": - doc_to_text: "Text: {{passage}}\n\nQuestion: {{question}}?\n\nOPTIONS:\n- no\n- yes" - doc_to_target: "{{['no', 'yes'][label]}}" - "template-4": - doc_to_text: "{{passage}}\n\nWhat's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes" - doc_to_target: "{{['no', 'yes'][label]}}" - "template-5": - doc_to_text: "{{passage}}\nBased on the above text what's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes" - doc_to_target: "{{['no', 'yes'][label]}}" - "template-6": - doc_to_text: "{{passage}}\nAnswer this question making sure that the answer is supposed by the text: {{question}}?\n\nOPTIONS:\n- no\n- yes" - doc_to_target: "{{['no', 'yes'][label]}}" - "template-7": - doc_to_text: "{{passage}}\n\nIs the following statement correct based on the text\n\n{{question}}\n\nOPTIONS:\n- no\n- yes" - doc_to_target: "{{['no', 'yes'][label]}}" - "template-8": - # doc_to_text: "{{title}}\n\n{{passage}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes" - doc_to_text: "{{passage}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes" - doc_to_target: "{{['no', 'yes'][label]}}" - "template-9": - doc_to_text: "Is it true that {{question}} based on the following text?\n\n{{passage}}\n\nOPTIONS:\n- no\n- yes" - doc_to_target: "{{['no', 'yes'][label]}}" diff --git a/lm_eval/tasks/benchmarks/flan/prompt_templates/rte.yaml b/lm_eval/tasks/benchmarks/flan/prompt_templates/rte.yaml deleted file mode 100644 index 7893eae44c..0000000000 --- a/lm_eval/tasks/benchmarks/flan/prompt_templates/rte.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Flan Prompt Templates -prompts: - "template-0": - doc_to_text: "{{premise}}\n\nQuestion with options: Based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- yes\n- no" - doc_to_target: "{{['yes', 'no'][label]}}" - "template-1": - doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that the sentence below is true?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no" - doc_to_target: "{{['yes', 'no'][label]}}" - "template-2": - doc_to_text: "{{premise}}\n\nQ with options: Can we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no" - doc_to_target: "{{['yes', 'no'][label]}}" - "template-3": - doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- yes\n- no" - doc_to_target: "{{['yes', 'no'][label]}}" - "template-4": - doc_to_text: "{{premise}}\nOPTIONS:\n- yes\n- no\nQuestion: Can we infer the following?\n{{hypothesis}}" - doc_to_target: "{{['yes', 'no'][label]}}" - "template-5": - doc_to_text: "Read the following paragraph and determine if the hypothesis is true. Select from options at the end:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nThe answer is" - doc_to_target: "{{['yes', 'no'][label]}}" - "template-6": - doc_to_text: "Read the text and determine if the sentence is true:\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nA:" - doc_to_target: "{{['yes', 'no'][label]}}" - "template-7": - doc_to_text: "Question with options: can we draw the following hypothesis from the context? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- yes\n- no\nA:" - doc_to_target: "{{['yes', 'no'][label]}}" - "template-8": - doc_to_text: "Determine if the sentence is true based on the text below. Choose from options.\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- yes\n- no" - doc_to_target: "{{['yes', 'no'][label]}}" diff --git a/lm_eval/tasks/benchmarks/flan/yaml_templates/cot_template_yaml b/lm_eval/tasks/benchmarks/flan/yaml_templates/cot_template_yaml deleted file mode 100644 index a18efa5f7d..0000000000 --- a/lm_eval/tasks/benchmarks/flan/yaml_templates/cot_template_yaml +++ /dev/null @@ -1,21 +0,0 @@ -group: flan-cot -output_type: generate_until -validation_split: validation -doc_to_target: "{{answer}}" -metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true -generation_kwargs: - until: - - "\n\n" - do_sample: false - temperature: 0.0 -filter_list: - - name: "get-answer" - filter: - - function: "regex" - regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" - - function: "take_first" -metadata: - version: 1.0 diff --git a/lm_eval/tasks/benchmarks/multimedqa/multimedqa.yaml b/lm_eval/tasks/benchmarks/multimedqa/multimedqa.yaml index 74b7180b2c..29810bb491 100644 --- a/lm_eval/tasks/benchmarks/multimedqa/multimedqa.yaml +++ b/lm_eval/tasks/benchmarks/multimedqa/multimedqa.yaml @@ -3,9 +3,15 @@ task: - pubmedqa - medmcqa - medqa_4options - - mmlu_anatomy - - mmlu_clinical_knowledge - - mmlu_college_medicine - - mmlu_medical_genetics - - mmlu_professional_medicine - - mmlu_college_biology + - task: mmlu_anatomy + task_alias: "anatomy (mmlu)" + - task: mmlu_clinical_knowledge + task_alias: "clinical_knowledge (mmlu)" + - task: mmlu_college_medicine + task_alias: "college_medicine (mmlu)" + - task: mmlu_medical_genetics + task_alias: "medical_genetics (mmlu)" + - task: mmlu_professional_medicine + task_alias: "professional_medicine (mmlu)" + - task: mmlu_college_biology + task_alias: "college_biology (mmlu)" diff --git a/lm_eval/tasks/bigbench/generate_tasks.py b/lm_eval/tasks/bigbench/generate_tasks.py index 3fd5cd6c2b..08fd0c0a59 100644 --- a/lm_eval/tasks/bigbench/generate_tasks.py +++ b/lm_eval/tasks/bigbench/generate_tasks.py @@ -181,7 +181,7 @@ def main() -> None: for task in all_subtasks: file_name = f"{task}.yaml" try: - with open(f"{path}/{file_name}", "w") as f: + with open(f"{path}/{file_name}", "w", encoding="utf-8") as f: f.write("# Generated by utils.py\n") yaml.dump( { diff --git a/lm_eval/tasks/blimp/generate_configs.py b/lm_eval/tasks/blimp/generate_configs.py index dfc4b4dc95..a768196172 100644 --- a/lm_eval/tasks/blimp/generate_configs.py +++ b/lm_eval/tasks/blimp/generate_configs.py @@ -75,7 +75,7 @@ def main() -> None: for task in all_subtasks: file_name = f"{task}.yaml" try: - with open(f"{file_name}", "w") as f: + with open(f"{file_name}", "w", encoding="utf-8") as f: f.write("# Generated by utils.py\n") yaml.dump( { diff --git a/lm_eval/tasks/ceval/_generate_configs.py b/lm_eval/tasks/ceval/_generate_configs.py index 2b96e00713..2df8ca31e4 100644 --- a/lm_eval/tasks/ceval/_generate_configs.py +++ b/lm_eval/tasks/ceval/_generate_configs.py @@ -79,13 +79,13 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our other YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) if args.cot_prompt_path is not None: import json - with open(args.cot_prompt_path) as f: + with open(args.cot_prompt_path, encoding="utf-8") as f: cot_file = json.load(f) for subject_eng, subject_zh in tqdm(SUBJECTS.items()): @@ -107,7 +107,7 @@ def parse_args(): file_save_path = args.save_prefix_path + f"_{subject_eng}.yaml" eval_logger.info(f"Saving yaml for subset {subject_eng} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, diff --git a/lm_eval/tasks/cmmlu/_generate_configs.py b/lm_eval/tasks/cmmlu/_generate_configs.py index 07553bb1ea..3afb15bf84 100644 --- a/lm_eval/tasks/cmmlu/_generate_configs.py +++ b/lm_eval/tasks/cmmlu/_generate_configs.py @@ -94,13 +94,13 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our other YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) if args.cot_prompt_path is not None: import json - with open(args.cot_prompt_path) as f: + with open(args.cot_prompt_path, encoding="utf-8") as f: cot_file = json.load(f) for subject_eng, subject_zh in tqdm(SUBJECTS.items()): @@ -122,7 +122,7 @@ def parse_args(): file_save_path = args.save_prefix_path + f"_{subject_eng}.yaml" eval_logger.info(f"Saving yaml for subset {subject_eng} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, diff --git a/lm_eval/tasks/code_x_glue/code-text/bleu.py b/lm_eval/tasks/code_x_glue/code-text/bleu.py index a90fc46b17..7f89404649 100644 --- a/lm_eval/tasks/code_x_glue/code-text/bleu.py +++ b/lm_eval/tasks/code_x_glue/code-text/bleu.py @@ -184,7 +184,7 @@ def splitPuncts(line): def computeMaps(predictions, goldfile): predictionMap: Dict[str, list] = {} goldMap: Dict[str, list] = {} - gf = open(goldfile, "r") + gf = open(goldfile, "r", encoding="utf-8") for row in predictions: cols = row.strip().split("\t") diff --git a/lm_eval/tasks/csatqa/_generate_configs.py b/lm_eval/tasks/csatqa/_generate_configs.py index 56fe825a90..bd849c0ae6 100644 --- a/lm_eval/tasks/csatqa/_generate_configs.py +++ b/lm_eval/tasks/csatqa/_generate_configs.py @@ -25,7 +25,7 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our other YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) for name in tqdm(SUBSETS): @@ -39,7 +39,7 @@ def parse_args(): file_save_path = args.save_prefix_path + f"_{name.lower()}.yaml" eval_logger.info(f"Saving yaml for subset {name} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, diff --git a/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml b/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml index 0c9b875352..d92ee342d1 100644 --- a/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml +++ b/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml @@ -31,4 +31,4 @@ filter_list: - function: "majority_vote" - function: "take_first" metadata: - version: 1.0 + version: 2.0 diff --git a/lm_eval/tasks/gsm8k/gsm8k-cot.yaml b/lm_eval/tasks/gsm8k/gsm8k-cot.yaml index c2e64ac2ae..9781d3118f 100644 --- a/lm_eval/tasks/gsm8k/gsm8k-cot.yaml +++ b/lm_eval/tasks/gsm8k/gsm8k-cot.yaml @@ -5,16 +5,16 @@ dataset_path: gsm8k dataset_name: main output_type: generate_until test_split: test -doc_to_text: "Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?\n\nA: There are 15 trees originally. Then there were 21 trees after some more were planted. So there must have been 21 - 15 = 6. The answer is 6.\n\n\ -Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?\n\nA: There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The answer is 5.\n\n\ -Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?\n\nA: Originally, Leah had 32 chocolates. Her sister had 42. So in total they had 32 + 42 = 74. After eating 35, they had 74 - 35 = 39. The answer is 39.\n\n\ -Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?\n\nA: Jason started with 20 lollipops. Then he had 12 after giving some to Denny. So he gave Denny 20 - 12 = 8. The answer is 8.\n\n\ -Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?\n\nA: Shawn started with 5 toys. If he got 2 toys each from his mom and dad, then that is 4 more toys. 5 + 4 = 9. The answer is 9.\n\n\ -Q: There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?\n\nA: There were originally 9 computers. For each of 4 days, 5 more computers were added. So 5 * 4 = 20 computers were added. 9 + 20 is 29. The answer is 29.\n\n\ -Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?\n\nA: Michael started with 58 golf balls. After losing 23 on tuesday, he had 58 - 23 = 35. After losing 2 more, he had 35 - 2 = 33 golf balls. The answer is 33.\n\n\ -Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?\n\nA: Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 dollars. So she has 23 - 15 dollars left. 23 - 15 is 8. The answer is 8.\n\n\ -Q: {{question}}\n\nA:" -doc_to_target: " {{answer.split('### ')[-1].rstrip()}}" +doc_to_text: "Q: There are 15 trees in the grove. Grove workers will plant trees in the grove today. After they are done, there will be 21 trees. How many trees did the grove workers plant today?\nA: There are 15 trees originally. Then there were 21 trees after some more were planted. So there must have been 21 - 15 = 6. The answer is 6.\n\n\ +Q: If there are 3 cars in the parking lot and 2 more cars arrive, how many cars are in the parking lot?\nA: There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The answer is 5.\n\n\ +Q: Leah had 32 chocolates and her sister had 42. If they ate 35, how many pieces do they have left in total?\nA: Originally, Leah had 32 chocolates. Her sister had 42. So in total they had 32 + 42 = 74. After eating 35, they had 74 - 35 = 39. The answer is 39.\n\n\ +Q: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 lollipops. How many lollipops did Jason give to Denny?\nA: Jason started with 20 lollipops. Then he had 12 after giving some to Denny. So he gave Denny 20 - 12 = 8. The answer is 8.\n\n\ +Q: Shawn has five toys. For Christmas, he got two toys each from his mom and dad. How many toys does he have now?\nA: Shawn started with 5 toys. If he got 2 toys each from his mom and dad, then that is 4 more toys. 5 + 4 = 9. The answer is 9.\n\n\ +Q: There were nine computers in the server room. Five more computers were installed each day, from monday to thursday. How many computers are now in the server room?\nA: There were originally 9 computers. For each of 4 days, 5 more computers were added. So 5 * 4 = 20 computers were added. 9 + 20 is 29. The answer is 29.\n\n\ +Q: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, he lost 2 more. How many golf balls did he have at the end of wednesday?\nA: Michael started with 58 golf balls. After losing 23 on tuesday, he had 58 - 23 = 35. After losing 2 more, he had 35 - 2 = 33 golf balls. The answer is 33.\n\n\ +Q: Olivia has $23. She bought five bagels for $3 each. How much money does she have left?\nA: Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 dollars. So she has 23 - 15 dollars left. 23 - 15 is 8. The answer is 8.\n\n\ +Q: {{question}}\nA:" +doc_to_target: "{{answer.split('####')[-1].strip()}}" metric_list: - metric: exact_match aggregation: mean @@ -31,7 +31,6 @@ generation_kwargs: - "Q:" - "\n\n" do_sample: false - temperature: 0.0 repeats: 1 num_fewshot: 0 filter_list: @@ -41,4 +40,5 @@ filter_list: regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)." - function: "take_first" metadata: - version: 1.0 + version: 2.0 + num_fewshot: 8 diff --git a/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml b/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml index 0725feaf96..e630fa8ac9 100644 --- a/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml +++ b/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml @@ -22,3 +22,4 @@ metric_list: num_fewshot: 0 metadata: version: 1.0 + num_fewshot: 4 diff --git a/lm_eval/tasks/mmlu/_generate_configs.py b/lm_eval/tasks/mmlu/_generate_configs.py index e6271bc4c2..1424814e7d 100644 --- a/lm_eval/tasks/mmlu/_generate_configs.py +++ b/lm_eval/tasks/mmlu/_generate_configs.py @@ -85,13 +85,13 @@ def parse_args(): # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs. base_yaml_name = os.path.split(args.base_yaml_path)[-1] - with open(args.base_yaml_path) as f: + with open(args.base_yaml_path, encoding="utf-8") as f: base_yaml = yaml.full_load(f) if args.cot_prompt_path is not None: import json - with open(args.cot_prompt_path) as f: + with open(args.cot_prompt_path, encoding="utf-8") as f: cot_file = json.load(f) ALL_CATEGORIES = [] @@ -120,7 +120,7 @@ def parse_args(): file_save_path = args.save_prefix_path + f"_{subject}.yaml" eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( yaml_dict, yaml_file, @@ -142,7 +142,7 @@ def parse_args(): file_save_path = args.save_prefix_path + ".yaml" eval_logger.info(f"Saving benchmark config to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump( { "group": f"mmlu_{args.task_prefix}" diff --git a/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py b/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py index aecb40a5eb..3a2bac5923 100644 --- a/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py +++ b/lm_eval/tasks/model_written_evals/advanced_ai_risk/_generate_configs.py @@ -9,7 +9,7 @@ def main() -> None: for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()): file_name = f"{task}.yaml" try: - with open(f"{file_name}", "w") as f: + with open(f"{file_name}", "w", encoding="utf-8") as f: f.write("# Generated by _generate_configs.py\n") yaml.dump( { diff --git a/lm_eval/tasks/model_written_evals/persona/_generate_configs.py b/lm_eval/tasks/model_written_evals/persona/_generate_configs.py index 7aff892f03..811e0b1b62 100644 --- a/lm_eval/tasks/model_written_evals/persona/_generate_configs.py +++ b/lm_eval/tasks/model_written_evals/persona/_generate_configs.py @@ -9,7 +9,7 @@ def main() -> None: for task in tqdm(datasets.get_dataset_infos(dataset_path).keys()): file_name = f"{task}.yaml" try: - with open(f"{file_name}", "w") as f: + with open(f"{file_name}", "w", encoding="utf-8") as f: f.write("# Generated by _generate_configs.py\n") yaml.dump( { diff --git a/lm_eval/tasks/nq_open/nq_open.yaml b/lm_eval/tasks/nq_open/nq_open.yaml index ac3f2fd99e..0464ca3abc 100644 --- a/lm_eval/tasks/nq_open/nq_open.yaml +++ b/lm_eval/tasks/nq_open/nq_open.yaml @@ -3,7 +3,7 @@ dataset_path: nq_open output_type: generate_until training_split: train validation_split: validation -description: "Answer these questions:\n" +description: "Answer these questions:\n\n" doc_to_text: "Q: {{question}}?\nA:" doc_to_target: "{{answer}}" # TODO: should be multi-target fewshot_delimiter: "\n" @@ -27,6 +27,6 @@ metric_list: ignore_case: true ignore_punctuation: true regexes_to_ignore: - - "\ban|a|the\b" + - "\\b(?:The |the |An |A |The |a |an )" metadata: - version: 1.0 + version: 3.0 diff --git a/lm_eval/tasks/okapi/arc_multilingual/README.md b/lm_eval/tasks/okapi/arc_multilingual/README.md new file mode 100644 index 0000000000..27c9329d12 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/README.md @@ -0,0 +1,47 @@ +# Multilingual ARC + +### Paper + +Title: `Okapi: Instruction-tuned Large Language Models in Multiple Languages with Reinforcement Learning from Human Feedback` + +Abstract: https://arxiv.org/abs/2307.16039 + +A key technology for the development of large language models (LLMs) involves instruction tuning that helps align the models' responses with human expectations to realize impressive learning abilities. Two major approaches for instruction tuning characterize supervised fine-tuning (SFT) and reinforcement learning from human feedback (RLHF), which are currently applied to produce the best commercial LLMs (e.g., ChatGPT). To improve the accessibility of LLMs for research and development efforts, various instruction-tuned open-source LLMs have also been introduced recently, e.g., Alpaca, Vicuna, to name a few. However, existing open-source LLMs have only been instruction-tuned for English and a few popular languages, thus hindering their impacts and accessibility to many other languages in the world. Among a few very recent work to explore instruction tuning for LLMs in multiple languages, SFT has been used as the only approach to instruction-tune LLMs for multiple languages. This has left a significant gap for fine-tuned LLMs based on RLHF in diverse languages and raised important questions on how RLHF can boost the performance of multilingual instruction tuning. To overcome this issue, we present Okapi, the first system with instruction-tuned LLMs based on RLHF for multiple languages. Okapi introduces instruction and response-ranked data in 26 diverse languages to facilitate the experiments and development of future multilingual LLM research. We also present benchmark datasets to enable the evaluation of generative LLMs in multiple languages. Our experiments demonstrate the advantages of RLHF for multilingual instruction over SFT for different base models and datasets. Our framework and resources are released at this https URL. + +Homepage: `https://github.com/nlp-uoregon/Okapi` + + +### Citation + +``` +@article{dac2023okapi, + title={Okapi: Instruction-tuned Large Language Models in Multiple Languages with Reinforcement Learning from Human Feedback}, + author={Dac Lai, Viet and Van Nguyen, Chien and Ngo, Nghia Trung and Nguyen, Thuat and Dernoncourt, Franck and Rossi, Ryan A and Nguyen, Thien Huu}, + journal={arXiv e-prints}, + pages={arXiv--2307}, + year={2023} +} +``` + +### Groups and Tasks + +#### Groups + +- arc_multilingual + +#### Tasks + +- `arc_{ar,bn,ca,da,de,es,eu,fr,gu,hi,hr,hu,hy,id,it,kn,ml,mr,ne,nl,pt,ro,ru,sk,sr,sv,ta,te,uk,vi,zh}` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/okapi/arc_multilingual/_arc_yaml b/lm_eval/tasks/okapi/arc_multilingual/_arc_yaml new file mode 100644 index 0000000000..9364a271ce --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/_arc_yaml @@ -0,0 +1,23 @@ +group: + - arc_multilingual +dataset_path: null +dataset_name: null +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "query" +doc_to_target: "gold" +doc_to_choice: "choices" +should_decontaminate: true +doc_to_decontamination_query: "query" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_ar.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_ar.yaml new file mode 100644 index 0000000000..9cfecf3e8e --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_ar.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_ar +dataset_path: alexandrainst/m_arc +dataset_name: ar +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_bn.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_bn.yaml new file mode 100644 index 0000000000..345c06398b --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_bn.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_bn +dataset_path: alexandrainst/m_arc +dataset_name: bn +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_ca.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_ca.yaml new file mode 100644 index 0000000000..95b433d6cb --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_ca.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_ca +dataset_path: alexandrainst/m_arc +dataset_name: ca +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_da.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_da.yaml new file mode 100644 index 0000000000..7209f8cbc0 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_da.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_da +dataset_path: alexandrainst/m_arc +dataset_name: da +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_de.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_de.yaml new file mode 100644 index 0000000000..d368292fc9 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_de.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_de +dataset_path: alexandrainst/m_arc +dataset_name: de +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_es.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_es.yaml new file mode 100644 index 0000000000..044210570e --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_es.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_es +dataset_path: alexandrainst/m_arc +dataset_name: es +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_eu.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_eu.yaml new file mode 100644 index 0000000000..13798d45b5 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_eu.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_eu +dataset_path: alexandrainst/m_arc +dataset_name: eu +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_fr.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_fr.yaml new file mode 100644 index 0000000000..712e42030e --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_fr.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_fr +dataset_path: alexandrainst/m_arc +dataset_name: fr +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_gu.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_gu.yaml new file mode 100644 index 0000000000..1d938cba1e --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_gu.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_gu +dataset_path: alexandrainst/m_arc +dataset_name: gu +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_hi.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_hi.yaml new file mode 100644 index 0000000000..8fb0488c79 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_hi.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_hi +dataset_path: alexandrainst/m_arc +dataset_name: hi +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_hr.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_hr.yaml new file mode 100644 index 0000000000..f9bc4c0252 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_hr.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_hr +dataset_path: alexandrainst/m_arc +dataset_name: hr +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_hu.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_hu.yaml new file mode 100644 index 0000000000..c06e9098b5 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_hu.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_hu +dataset_path: alexandrainst/m_arc +dataset_name: hu +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_hy.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_hy.yaml new file mode 100644 index 0000000000..81c7ceab4a --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_hy.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_hy +dataset_path: alexandrainst/m_arc +dataset_name: hy +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_id.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_id.yaml new file mode 100644 index 0000000000..fa02f7ee86 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_id.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_id +dataset_path: alexandrainst/m_arc +dataset_name: id +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_it.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_it.yaml new file mode 100644 index 0000000000..d9318c09fd --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_it.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_it +dataset_path: alexandrainst/m_arc +dataset_name: it +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_kn.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_kn.yaml new file mode 100644 index 0000000000..f5c9fdf064 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_kn.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_kn +dataset_path: alexandrainst/m_arc +dataset_name: kn +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_ml.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_ml.yaml new file mode 100644 index 0000000000..1af64793a7 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_ml.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_ml +dataset_path: alexandrainst/m_arc +dataset_name: ml +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_mr.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_mr.yaml new file mode 100644 index 0000000000..fdc6a693cd --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_mr.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_mr +dataset_path: alexandrainst/m_arc +dataset_name: mr +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_ne.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_ne.yaml new file mode 100644 index 0000000000..52947adf6b --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_ne.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_ne +dataset_path: alexandrainst/m_arc +dataset_name: ne +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_nl.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_nl.yaml new file mode 100644 index 0000000000..771fa60556 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_nl.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_nl +dataset_path: alexandrainst/m_arc +dataset_name: nl +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_pt.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_pt.yaml new file mode 100644 index 0000000000..78c7593220 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_pt.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_pt +dataset_path: alexandrainst/m_arc +dataset_name: pt +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_ro.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_ro.yaml new file mode 100644 index 0000000000..bdf99e8099 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_ro.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_ro +dataset_path: alexandrainst/m_arc +dataset_name: ro +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_ru.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_ru.yaml new file mode 100644 index 0000000000..157f886e2a --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_ru.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_ru +dataset_path: alexandrainst/m_arc +dataset_name: ru +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_sk.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_sk.yaml new file mode 100644 index 0000000000..04ff0182ac --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_sk.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_sk +dataset_path: alexandrainst/m_arc +dataset_name: sk +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_sr.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_sr.yaml new file mode 100644 index 0000000000..aacfc06dd6 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_sr.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_sr +dataset_path: alexandrainst/m_arc +dataset_name: sr +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_sv.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_sv.yaml new file mode 100644 index 0000000000..c557f8e121 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_sv.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_sv +dataset_path: alexandrainst/m_arc +dataset_name: sv +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_ta.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_ta.yaml new file mode 100644 index 0000000000..0af5744eb4 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_ta.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_ta +dataset_path: alexandrainst/m_arc +dataset_name: ta +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_te.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_te.yaml new file mode 100644 index 0000000000..2ee32742aa --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_te.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_te +dataset_path: alexandrainst/m_arc +dataset_name: te +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_uk.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_uk.yaml new file mode 100644 index 0000000000..42b77e4c0e --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_uk.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_uk +dataset_path: alexandrainst/m_arc +dataset_name: uk +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_vi.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_vi.yaml new file mode 100644 index 0000000000..bdcccb3419 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_vi.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_vi +dataset_path: alexandrainst/m_arc +dataset_name: vi +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/arc_zh.yaml b/lm_eval/tasks/okapi/arc_multilingual/arc_zh.yaml new file mode 100644 index 0000000000..3890fd1f9c --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/arc_zh.yaml @@ -0,0 +1,7 @@ +include: _arc_yaml +task: arc_zh +dataset_path: alexandrainst/m_arc +dataset_name: zh +training_split: train +validation_split: validation +test_split: test diff --git a/lm_eval/tasks/okapi/arc_multilingual/utils.py b/lm_eval/tasks/okapi/arc_multilingual/utils.py new file mode 100644 index 0000000000..43cccc5672 --- /dev/null +++ b/lm_eval/tasks/okapi/arc_multilingual/utils.py @@ -0,0 +1,26 @@ +import datasets +import re + + +def preprocess(text): + if text is None: + return " " + text = text.strip() + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + # breakpoint() + out_doc = { + "id": doc["id"], + "query": "Question: " + preprocess(doc["instruction"]) + "\nAnswer:", + "choices": [preprocess(doc['option_a']), preprocess(doc['option_b']), preprocess(doc['option_c']), preprocess(doc['option_d']), preprocess(doc['option_e'])], + "gold": ["A", "B", "C", "D", "E"].index(doc["answer"]), + } + return out_doc + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml b/lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml new file mode 100644 index 0000000000..7a61ba4fe5 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml @@ -0,0 +1,17 @@ +group: + - m_mmlu +dataset_path: alexandrainst/m_mmlu +test_split: test +fewshot_split: train +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "{{instruction.strip()}}\nA. {{option_a}}\nB. {{option_b}}\nC. {{option_c}}\nD. {{option_d}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py b/lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py new file mode 100644 index 0000000000..07e5cd74f2 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/_generate_configs.py @@ -0,0 +1,33 @@ +import yaml +import datasets + +from tqdm import tqdm + + +def main() -> None: + dataset_path = "alexandrainst/m_mmlu" + + # Removed hy and sk subdataset because the original dataset is broken + # I created this PR https://huggingface.co/datasets/alexandrainst/m_mmlu/discussions/3 + # on the dataset for the authors, in case it will be accepeted the filter can be removed + keys_without_hy_sk = list(filter(lambda k: ('hy' not in k and 'sk' not in k), + datasets.get_dataset_infos(dataset_path).keys())) + + for task in tqdm(): + file_name = f"m_mmlu_{task}.yaml" + try: + with open(f"{file_name}", "w") as f: + f.write("# Generated by _generate_configs.py\n") + yaml.dump( + { + "include": "_default_yaml", + "task": f"{dataset_path.split('/')[-1]}_{task}", + "dataset_name": task, + }, + f, + ) + except FileExistsError: + pass + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ar.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ar.yaml new file mode 100644 index 0000000000..70f6473a85 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ar.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ar +include: _default_yaml +task: m_mmlu_ar diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_bn.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_bn.yaml new file mode 100644 index 0000000000..1d16feec91 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_bn.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: bn +include: _default_yaml +task: m_mmlu_bn diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ca.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ca.yaml new file mode 100644 index 0000000000..2fb5f2fcb9 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ca.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ca +include: _default_yaml +task: m_mmlu_ca diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_da.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_da.yaml new file mode 100644 index 0000000000..95eb1dc9b1 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_da.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: da +include: _default_yaml +task: m_mmlu_da diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_de.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_de.yaml new file mode 100644 index 0000000000..83aaba9ede --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_de.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: de +include: _default_yaml +task: m_mmlu_de diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_en.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_en.yaml new file mode 100644 index 0000000000..c1615e30cb --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_en.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: en +include: _default_yaml +task: m_mmlu_en diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_es.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_es.yaml new file mode 100644 index 0000000000..4d36cbe6f2 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_es.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: es +include: _default_yaml +task: m_mmlu_es diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_eu.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_eu.yaml new file mode 100644 index 0000000000..82763eb602 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_eu.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: eu +include: _default_yaml +task: m_mmlu_eu diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_fr.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_fr.yaml new file mode 100644 index 0000000000..eb8cce6ff8 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_fr.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: fr +include: _default_yaml +task: m_mmlu_fr diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_gu.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_gu.yaml new file mode 100644 index 0000000000..18f605fa93 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_gu.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gu +include: _default_yaml +task: m_mmlu_gu diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hi.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hi.yaml new file mode 100644 index 0000000000..bf0064f782 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hi.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: hi +include: _default_yaml +task: m_mmlu_hi diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hr.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hr.yaml new file mode 100644 index 0000000000..0c6e24d8e1 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hr.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: hr +include: _default_yaml +task: m_mmlu_hr diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hu.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hu.yaml new file mode 100644 index 0000000000..d824cb768a --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_hu.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: hu +include: _default_yaml +task: m_mmlu_hu diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_id.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_id.yaml new file mode 100644 index 0000000000..63594e227a --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_id.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: id +include: _default_yaml +task: m_mmlu_id diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_is.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_is.yaml new file mode 100644 index 0000000000..494b0c10ac --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_is.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: is +include: _default_yaml +task: m_mmlu_is diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_it.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_it.yaml new file mode 100644 index 0000000000..30795d329a --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_it.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: it +include: _default_yaml +task: m_mmlu_it diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_kn.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_kn.yaml new file mode 100644 index 0000000000..82d026c7e4 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_kn.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: kn +include: _default_yaml +task: m_mmlu_kn diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ml.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ml.yaml new file mode 100644 index 0000000000..5daf8736a5 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ml.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ml +include: _default_yaml +task: m_mmlu_ml diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_mr.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_mr.yaml new file mode 100644 index 0000000000..f6f6df7f30 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_mr.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: mr +include: _default_yaml +task: m_mmlu_mr diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nb.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nb.yaml new file mode 100644 index 0000000000..76ab5a601d --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nb.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: nb +include: _default_yaml +task: m_mmlu_nb diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ne.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ne.yaml new file mode 100644 index 0000000000..c6f53563ed --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ne.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ne +include: _default_yaml +task: m_mmlu_ne diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nl.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nl.yaml new file mode 100644 index 0000000000..df115a68d0 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_nl.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: nl +include: _default_yaml +task: m_mmlu_nl diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_pt.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_pt.yaml new file mode 100644 index 0000000000..de4bb65953 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_pt.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: pt +include: _default_yaml +task: m_mmlu_pt diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ro.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ro.yaml new file mode 100644 index 0000000000..236d8382d7 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ro.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ro +include: _default_yaml +task: m_mmlu_ro diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ru.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ru.yaml new file mode 100644 index 0000000000..ce379b61e4 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ru.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ru +include: _default_yaml +task: m_mmlu_ru diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sr.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sr.yaml new file mode 100644 index 0000000000..22b0ad7755 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sr.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: sr +include: _default_yaml +task: m_mmlu_sr diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sv.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sv.yaml new file mode 100644 index 0000000000..d433d08259 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_sv.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: sv +include: _default_yaml +task: m_mmlu_sv diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ta.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ta.yaml new file mode 100644 index 0000000000..2314894c2b --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_ta.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: ta +include: _default_yaml +task: m_mmlu_ta diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_te.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_te.yaml new file mode 100644 index 0000000000..0737ed37aa --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_te.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: te +include: _default_yaml +task: m_mmlu_te diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_uk.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_uk.yaml new file mode 100644 index 0000000000..fdc704b7d6 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_uk.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: uk +include: _default_yaml +task: m_mmlu_uk diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_vi.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_vi.yaml new file mode 100644 index 0000000000..e1d6771e5a --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_vi.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: vi +include: _default_yaml +task: m_mmlu_vi diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_zh.yaml b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_zh.yaml new file mode 100644 index 0000000000..bf92a74ff1 --- /dev/null +++ b/lm_eval/tasks/okapi/mmlu_multilingual/m_mmlu_zh.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: zh +include: _default_yaml +task: m_mmlu_zh diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/README.md b/lm_eval/tasks/okapi/truthfulqa_multilingual/README.md new file mode 100644 index 0000000000..324cdce592 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/README.md @@ -0,0 +1,47 @@ +# Multilingual TruthfulQA + +### Paper + +Title: `Okapi: Instruction-tuned Large Language Models in Multiple Languages with Reinforcement Learning from Human Feedback` + +Abstract: https://arxiv.org/abs/2307.16039 + +A key technology for the development of large language models (LLMs) involves instruction tuning that helps align the models' responses with human expectations to realize impressive learning abilities. Two major approaches for instruction tuning characterize supervised fine-tuning (SFT) and reinforcement learning from human feedback (RLHF), which are currently applied to produce the best commercial LLMs (e.g., ChatGPT). To improve the accessibility of LLMs for research and development efforts, various instruction-tuned open-source LLMs have also been introduced recently, e.g., Alpaca, Vicuna, to name a few. However, existing open-source LLMs have only been instruction-tuned for English and a few popular languages, thus hindering their impacts and accessibility to many other languages in the world. Among a few very recent work to explore instruction tuning for LLMs in multiple languages, SFT has been used as the only approach to instruction-tune LLMs for multiple languages. This has left a significant gap for fine-tuned LLMs based on RLHF in diverse languages and raised important questions on how RLHF can boost the performance of multilingual instruction tuning. To overcome this issue, we present Okapi, the first system with instruction-tuned LLMs based on RLHF for multiple languages. Okapi introduces instruction and response-ranked data in 26 diverse languages to facilitate the experiments and development of future multilingual LLM research. We also present benchmark datasets to enable the evaluation of generative LLMs in multiple languages. Our experiments demonstrate the advantages of RLHF for multilingual instruction over SFT for different base models and datasets. Our framework and resources are released at this https URL. + +Homepage: `https://github.com/nlp-uoregon/Okapi` + + +### Citation + +``` +@article{dac2023okapi, + title={Okapi: Instruction-tuned Large Language Models in Multiple Languages with Reinforcement Learning from Human Feedback}, + author={Dac Lai, Viet and Van Nguyen, Chien and Ngo, Nghia Trung and Nguyen, Thuat and Dernoncourt, Franck and Rossi, Ryan A and Nguyen, Thien Huu}, + journal={arXiv e-prints}, + pages={arXiv--2307}, + year={2023} +} +``` + +### Groups and Tasks + +#### Groups + +- truthfulqa_multilingual + +#### Tasks + +- `truthfulqa_{ar,bn,ca,da,de,es,eu,fr,gu,hi,hr,hu,hy,id,it,kn,ml,mr,ne,nl,pt,ro,ru,sk,sr,sv,ta,te,uk,vi,zh}` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml new file mode 100644 index 0000000000..e98f64fde8 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml @@ -0,0 +1,20 @@ +group: + - truthfulqa_multilingual +dataset_path: null +dataset_name: null +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: null +process_docs: !function utils.process_docs +doc_to_text: "query" +doc_to_target: 0 +doc_to_choice: "mc1_choices" +should_decontaminate: True +doc_to_decontamination_query: "question" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc2_yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc2_yaml new file mode 100644 index 0000000000..7c21ca1563 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc2_yaml @@ -0,0 +1,12 @@ +include: _truthfulqa_mc1_yaml +doc_to_target: 0 +doc_to_choice: "mc2_choices" +process_results: !function utils.process_results_mc2 +should_decontaminate: True +doc_to_decontamination_query: "question" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc1.yaml new file mode 100644 index 0000000000..80211f096a --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ar_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ar +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc2.yaml new file mode 100644 index 0000000000..d0249ecf51 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ar_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ar_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ar +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc1.yaml new file mode 100644 index 0000000000..17f39493a5 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_bn_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: bn +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc2.yaml new file mode 100644 index 0000000000..85553b0b1b --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_bn_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_bn_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: bn +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc1.yaml new file mode 100644 index 0000000000..41d0d08173 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ca_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ca +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc2.yaml new file mode 100644 index 0000000000..16db59c883 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ca_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ca_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ca +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc1.yaml new file mode 100644 index 0000000000..4094e74454 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_da_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: da +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc2.yaml new file mode 100644 index 0000000000..4b845f0e2c --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_da_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_da_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: da +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc1.yaml new file mode 100644 index 0000000000..35d7c1569d --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_de_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: de +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc2.yaml new file mode 100644 index 0000000000..2065cf4ac7 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_de_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_de_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: de +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc1.yaml new file mode 100644 index 0000000000..5f8ac1d76f --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_es_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: es +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc2.yaml new file mode 100644 index 0000000000..f522921d51 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_es_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_es_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: es +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc1.yaml new file mode 100644 index 0000000000..ff8156bbdd --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_eu_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: eu +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc2.yaml new file mode 100644 index 0000000000..1d255802c7 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_eu_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_eu_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: eu +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc1.yaml new file mode 100644 index 0000000000..717616c777 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_fr_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: fr +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc2.yaml new file mode 100644 index 0000000000..626851db3f --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_fr_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_fr_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: fr +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc1.yaml new file mode 100644 index 0000000000..413873e172 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_gu_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: gu +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc2.yaml new file mode 100644 index 0000000000..a7eaac4205 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_gu_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_gu_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: gu +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc1.yaml new file mode 100644 index 0000000000..91c43d5d76 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_hi_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hi +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc2.yaml new file mode 100644 index 0000000000..90552090fb --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hi_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_hi_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hi +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc1.yaml new file mode 100644 index 0000000000..aa0c084b82 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_hr_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hr +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc2.yaml new file mode 100644 index 0000000000..cce11a4b63 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hr_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_hr_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hr +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc1.yaml new file mode 100644 index 0000000000..dc9f878ede --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_hu_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hu +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc2.yaml new file mode 100644 index 0000000000..e1458b5064 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hu_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_hu_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hu +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc1.yaml new file mode 100644 index 0000000000..71a8995c23 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_hy_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hy +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc2.yaml new file mode 100644 index 0000000000..d695866272 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_hy_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_hy_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: hy +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc1.yaml new file mode 100644 index 0000000000..09e5261e64 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_id_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: id +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc2.yaml new file mode 100644 index 0000000000..14f4796b35 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_id_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_id_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: id +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc1.yaml new file mode 100644 index 0000000000..8e83a88ead --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_it_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: it +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc2.yaml new file mode 100644 index 0000000000..2111015a74 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_it_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_it_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: it +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc1.yaml new file mode 100644 index 0000000000..866b08f080 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_kn_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: kn +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc2.yaml new file mode 100644 index 0000000000..8160817b0f --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_kn_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_kn_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: kn +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc1.yaml new file mode 100644 index 0000000000..22cbcf7431 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ml_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ml +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc2.yaml new file mode 100644 index 0000000000..1f0555ef5a --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ml_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ml_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ml +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc1.yaml new file mode 100644 index 0000000000..6e9a888a85 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_mr_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: mr +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc2.yaml new file mode 100644 index 0000000000..c0eae50796 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_mr_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_mr_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: mr +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ne_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ne_mc1.yaml new file mode 100644 index 0000000000..1b4ed65da0 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ne_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ne_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ne +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ne_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ne_mc2.yaml new file mode 100644 index 0000000000..3b9ee73d95 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ne_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ne_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ne +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_nl_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_nl_mc1.yaml new file mode 100644 index 0000000000..f45409c525 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_nl_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_nl_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: nl +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_nl_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_nl_mc2.yaml new file mode 100644 index 0000000000..596a15049c --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_nl_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_nl_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: nl +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_pt_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_pt_mc1.yaml new file mode 100644 index 0000000000..2240ef6259 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_pt_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_pt_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: pt +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_pt_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_pt_mc2.yaml new file mode 100644 index 0000000000..24c1328165 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_pt_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_pt_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: pt +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc1.yaml new file mode 100644 index 0000000000..f5e8b16f0d --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ro_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ro +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc2.yaml new file mode 100644 index 0000000000..ab35dd14e1 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ro_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ro_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ro +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ru_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ru_mc1.yaml new file mode 100644 index 0000000000..e9982598ce --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ru_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ru_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ru +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ru_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ru_mc2.yaml new file mode 100644 index 0000000000..ee1bda3dd8 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ru_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ru_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ru +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc1.yaml new file mode 100644 index 0000000000..5a4f6730ae --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_sk_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: sk +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc2.yaml new file mode 100644 index 0000000000..0826ce1269 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sk_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_sk_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: sk +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sr_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sr_mc1.yaml new file mode 100644 index 0000000000..ecb03d923d --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sr_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_sr_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: sr +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sr_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sr_mc2.yaml new file mode 100644 index 0000000000..25f549d962 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sr_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_sr_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: sr +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sv_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sv_mc1.yaml new file mode 100644 index 0000000000..fc457d69d1 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sv_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_sv_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: sv +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sv_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sv_mc2.yaml new file mode 100644 index 0000000000..a1055f1027 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_sv_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_sv_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: sv +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ta_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ta_mc1.yaml new file mode 100644 index 0000000000..8964854a00 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ta_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_ta_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ta +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ta_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ta_mc2.yaml new file mode 100644 index 0000000000..8a03a97a53 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_ta_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_ta_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: ta +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_te_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_te_mc1.yaml new file mode 100644 index 0000000000..a16b06f61b --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_te_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_te_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: te +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_te_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_te_mc2.yaml new file mode 100644 index 0000000000..47258dc01d --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_te_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_te_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: te +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_uk_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_uk_mc1.yaml new file mode 100644 index 0000000000..65578da44b --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_uk_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_uk_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: uk +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_uk_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_uk_mc2.yaml new file mode 100644 index 0000000000..2717d137ff --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_uk_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_uk_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: uk +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_vi_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_vi_mc1.yaml new file mode 100644 index 0000000000..c9a474cf81 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_vi_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_vi_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: vi +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_vi_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_vi_mc2.yaml new file mode 100644 index 0000000000..c229de206b --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_vi_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_vi_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: vi +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_zh_mc1.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_zh_mc1.yaml new file mode 100644 index 0000000000..651412e256 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_zh_mc1.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc1_yaml +task: truthfulqa_zh_mc1 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: zh +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_zh_mc2.yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_zh_mc2.yaml new file mode 100644 index 0000000000..ecd758f09a --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/truthfulqa_zh_mc2.yaml @@ -0,0 +1,7 @@ +include: _truthfulqa_mc2_yaml +task: truthfulqa_zh_mc2 +dataset_path: alexandrainst/m_truthfulqa +dataset_name: zh +training_split: null +validation_split: validation +test_split: null diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/utils.py b/lm_eval/tasks/okapi/truthfulqa_multilingual/utils.py new file mode 100644 index 0000000000..307d5cea44 --- /dev/null +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/utils.py @@ -0,0 +1,54 @@ +import datasets +import re +import numpy as np + +QA_PROMPT = ( + "Q: What is human life expectancy in the United States?\n" + "A: Human life expectancy in the United States is 78 years.\n\n" + "Q: Who was president of the United States in 1955?\n" + "A: Dwight D. Eisenhower was president of the United States in 1955.\n\n" + "Q: Which party did he belong to?\n" + "A: He belonged to the Republican Party.\n\n" + "Q: What is the square root of banana?\n" + "A: I have no comment.\n\n" + "Q: How does a telescope work?\n" + "A: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\n" + "Q: Where were the 1992 Olympics held?\n" + "A: The 1992 Olympics were held in Barcelona, Spain." +) + +def preprocess(text): + if text is None: + return " " + text = text.strip() + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + out_doc = { + "question": preprocess(doc["question"]), + "query": QA_PROMPT + "\n\nQ: " + preprocess(doc["question"]) + "\nA:", + "mc1_choices": doc["mc1_targets_choices"], + "mc2_choices": doc["mc2_targets_choices"], + "gold": " ", + } + return out_doc + + return dataset.map(_process_doc) + + +def process_results_mc2(doc, results): + lls, is_greedy = zip(*results) + + # Split on the first `0` as everything before it is true (`1`). + split_idx = list(doc["mc2_targets"]["labels"]).index(0) + # Compute the normalized probability mass for the correct answer. + ll_true, ll_false = lls[:split_idx], lls[split_idx:] + p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false)) + p_true = p_true / (sum(p_true) + sum(p_false)) + + return {"acc": sum(p_true)} \ No newline at end of file diff --git a/lm_eval/tasks/polemo2/polemo2_in.yaml b/lm_eval/tasks/polemo2/polemo2_in.yaml index 861feb5d30..2fa16db87b 100644 --- a/lm_eval/tasks/polemo2/polemo2_in.yaml +++ b/lm_eval/tasks/polemo2/polemo2_in.yaml @@ -2,7 +2,7 @@ group: - polemo2 task: polemo2_in dataset_path: allegro/klej-polemo2-in -dataset_name: klej-polemo2-in +dataset_name: null output_type: generate_until training_split: train validation_split: validation @@ -41,5 +41,6 @@ metric_list: - metric: accuracy aggregation: mean higher_is_better: true + hf_evaluate: true metadata: version: 1.0 diff --git a/lm_eval/tasks/qasper/bool.yaml b/lm_eval/tasks/qasper/bool.yaml index 468da5c6d1..17d3f1be98 100644 --- a/lm_eval/tasks/qasper/bool.yaml +++ b/lm_eval/tasks/qasper/bool.yaml @@ -1,6 +1,6 @@ group: qasper task: qasper_bool -dataset_path: qasper +dataset_path: allenai/qasper output_type: multiple_choice training_split: train validation_split: validation diff --git a/lm_eval/tasks/qasper/freeform.yaml b/lm_eval/tasks/qasper/freeform.yaml index 13561183f1..ed7a4bc472 100644 --- a/lm_eval/tasks/qasper/freeform.yaml +++ b/lm_eval/tasks/qasper/freeform.yaml @@ -1,6 +1,6 @@ group: qasper task: qasper_freeform -dataset_path: qasper +dataset_path: allenai/qasper output_type: generate_until training_split: train validation_split: validation diff --git a/lm_eval/tasks/qasper/utils.py b/lm_eval/tasks/qasper/utils.py index 7a02237a78..21f5be2435 100644 --- a/lm_eval/tasks/qasper/utils.py +++ b/lm_eval/tasks/qasper/utils.py @@ -50,7 +50,7 @@ def _flatten(doc): obs_list["abstract"].append(abstract) obs_list["question"].append(question) obs_list["answer_type"].append(answer_type) - if type(answer) == list: + if isinstance(answer, list): answer = ", ".join(answer) obs_list["answer"].append(answer) diff --git a/lm_eval/tasks/scrolls/scrolls.yaml b/lm_eval/tasks/scrolls/scrolls.yaml index 6dc315576c..da8d03e89e 100644 --- a/lm_eval/tasks/scrolls/scrolls.yaml +++ b/lm_eval/tasks/scrolls/scrolls.yaml @@ -1,9 +1,16 @@ group: scrolls task: - - scrolls_qasper - - scrolls_quality - - scrolls_narrativeqa - - scrolls_contractnli - - scrolls_govreport - - scrolls_summscreenfd - - scrolls_qmsum + - task: scrolls_qasper + class: !function task.Qasper + - task: scrolls_quality + class: !function task.QuALITY + - task: scrolls_narrativeqa + class: !function task.NarrativeQA + - task: scrolls_contractnli + class: !function task.ContractNLI + - task: scrolls_govreport + class: !function task.GovReport + - task: scrolls_summscreenfd + class: !function task.SummScreenFD + - task: scrolls_qmsum + class: !function task.QMSum diff --git a/lm_eval/tasks/scrolls/task.py b/lm_eval/tasks/scrolls/task.py index 0a7387a6aa..e403fd5e2d 100644 --- a/lm_eval/tasks/scrolls/task.py +++ b/lm_eval/tasks/scrolls/task.py @@ -115,8 +115,10 @@ class _SCROLLSTask(Task): PRUNE_MAX_TOKENS = None PRUNE_NUM_PROC = None - def __post_init__(self): - self.metric = load_metric(_download_metric(), config_name=self.DATASET_NAME) + def __init__(self): + super().__init__() + if self.DATASET_NAME is not None: + self.metric = load_metric(_download_metric(), config_name=self.DATASET_NAME) def has_training_docs(self): return True @@ -224,9 +226,10 @@ def higher_is_better(self): def process_results(self, doc, results): gold = doc["gold"] - acc = 1.0 if np.argmax(results) == gold else 0.0 + lls, _ = zip(*results) + acc = 1.0 if np.argmax(lls) == gold else 0.0 completion_len = np.array([float(len(i)) for i in doc["choices"]]) - acc_norm = 1.0 if np.argmax(results / completion_len) == gold else 0.0 + acc_norm = 1.0 if np.argmax(lls / completion_len) == gold else 0.0 return { "acc": acc, @@ -279,7 +282,6 @@ def doc_to_text(self, doc): return f"{doc['input']}\n\nQuestion: What is a summary of the preceding text?\nAnswer:" -@register_task("scrolls_qasper") class Qasper(_SCROLLSTask): """A Dataset of Information-Seeking Questions and Answers Anchored in Research Papers https://arxiv.org/abs/2105.03011 @@ -337,7 +339,6 @@ def construct_requests(self, doc, ctx, **kwargs): ) -@register_task("scrolls_quality") class QuALITY(_SCROLLSMultipleChoiceTask): """QuALITY: Question Answering with Long Input Texts, Yes! https://arxiv.org/abs/2112.08608 @@ -366,7 +367,6 @@ def _process_doc(self, doc): return [doc] -@register_task("scrolls_narrativeqa") class NarrativeQA(_SCROLLSTask): """The NarrativeQA Reading Comprehension Challenge https://arxiv.org/abs/1712.07040 @@ -400,7 +400,6 @@ def construct_requests(self, doc, ctx, **kwargs): ) -@register_task("scrolls_contractnli") class ContractNLI(_SCROLLSMultipleChoiceTask): """ContractNLI: A Dataset for Document-level Natural Language Inference for Contracts https://arxiv.org/abs/1712.07040 @@ -419,7 +418,6 @@ def doc_to_text(self, doc): return f"{doc['text']}\n\nHypothesis: {doc['question']}\nConclusion:" -@register_task("scrolls_govreport") class GovReport(_SCROLLSSummaryTask): """Efficient Attentions for Long Document Summarization https://arxiv.org/abs/2104.02112 @@ -433,7 +431,6 @@ class GovReport(_SCROLLSSummaryTask): DATASET_NAME = "gov_report" -@register_task("scrolls_summscreenfd") class SummScreenFD(_SCROLLSSummaryTask): """SummScreen: A Dataset for Abstractive Screenplay Summarization https://arxiv.org/abs/2104.07091 @@ -442,7 +439,6 @@ class SummScreenFD(_SCROLLSSummaryTask): DATASET_NAME = "summ_screen_fd" -@register_task("scrolls_qmsum") class QMSum(_SCROLLSSummaryTask): """QMSum: A New Benchmark for Query-based Multi-domain Meeting Summarization diff --git a/lm_eval/tasks/squadv2/squadv2.yaml b/lm_eval/tasks/squadv2/squadv2.yaml new file mode 100644 index 0000000000..13e451645c --- /dev/null +++ b/lm_eval/tasks/squadv2/squadv2.yaml @@ -0,0 +1,2 @@ +task: squadv2 +class: !function task.SQuAD2 diff --git a/lm_eval/tasks/squadv2/task.py b/lm_eval/tasks/squadv2/task.py index 0a8bfa35f5..8af87e7537 100644 --- a/lm_eval/tasks/squadv2/task.py +++ b/lm_eval/tasks/squadv2/task.py @@ -19,9 +19,8 @@ from functools import partial from packaging import version -from lm_eval.api.task import Task +from lm_eval.api.task import ConfigurableTask from lm_eval.api.instance import Instance -from lm_eval.api.registry import register_task _CITATION = """ @misc{rajpurkar2018know, @@ -47,12 +46,14 @@ def _squad_agg(key, items): return _squad_metric(predictions=predictions, references=references).get(key, 0) -@register_task("squadv2") -class SQuAD2(Task): +class SQuAD2(ConfigurableTask): VERSION = 3 DATASET_PATH = "squad_v2" DATASET_NAME = None + def __init__(self): + super().__init__(config={'metadata': {'version': self.VERSION}}) + # HF changed squad on us so we have to make sure we aren't running the old one assert version.parse(datasets.__version__) >= version.parse( "1.11.0" diff --git a/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml b/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml index 5e18acbbfb..6030d1faf2 100644 --- a/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml @@ -7,6 +7,7 @@ training_split: train validation_split: validation output_type: generate_until doc_to_text: !function "t5_utils.doc_to_text" +process_results: !function "t5_utils.process_results" doc_to_target: label generation_kwargs: until: @@ -15,9 +16,5 @@ metric_list: - metric: accuracy aggregation: mean higher_is_better: true -filter_list: - - name: "wsc_postprocessor" - filter: - - function: !function t5_utils.WSCPostprocess metadata: - version: 0.0 + version: 1.0 diff --git a/lm_eval/tasks/super_glue/wsc/t5_utils.py b/lm_eval/tasks/super_glue/wsc/t5_utils.py index eb5331a42a..6570abc732 100644 --- a/lm_eval/tasks/super_glue/wsc/t5_utils.py +++ b/lm_eval/tasks/super_glue/wsc/t5_utils.py @@ -1,6 +1,5 @@ import re -from lm_eval.api.filter import Filter - +from typing import List def doc_to_text(x): text = re.sub(r" X ", " *" + x["span2_text"] + "* ", _wsc_inputs(x)) @@ -24,14 +23,14 @@ def create_input(): [ " ".join(words[:pronoun_index]), "X", - " ".join(words[pronoun_index + 1 :]), + " ".join(words[pronoun_index + 1:]), ] ) # Handle some special cases. if ( - x["text"] - == 'The boy continued to whip the pony , and eventually the pony threw him over. John laughed out quite loud. "Good for him," he said. ' + x["text"] + == 'The boy continued to whip the pony , and eventually the pony threw him over. John laughed out quite loud. "Good for him," he said. ' ): return ( "The boy continued to whip the pony , and eventually the pony threw " @@ -40,8 +39,8 @@ def create_input(): # Using the span2_index, we get 'use' instead of 'it'. if ( - x["text"] - == "When they had eventually calmed down a bit , and had gotten home, Mr. Farley put the magic pebble in an iron safe . Some day they might want to use it , but really for now, what more could they wish for?" + x["text"] + == "When they had eventually calmed down a bit , and had gotten home, Mr. Farley put the magic pebble in an iron safe . Some day they might want to use it , but really for now, what more could they wish for?" ): return ( "When they had eventually calmed down a bit , and had gotten home, " @@ -52,56 +51,53 @@ def create_input(): return create_input() -class WSCPostprocess(Filter): - def __init__(self, **kwargs): - self.determiners = { - "a", - "an", - "few", - "her", - "his", - "each", - "every", - "many", - "much", - "my", - "our", - "some", - "that", - "the", - "their", - "these", - "this", - "those", - "which", - "whose", - "your", - } - - def clean(self, s): - """Ignore capitalization and determiners.""" - s = s.strip().lower() - return " ".join([w for w in s.split(" ") if w not in self.determiners]) - - def apply(self, resps, docs): - filtered_resps = [] - for prediction, reference in zip(*(resps, docs["span1_text"])): - prediction = self.clean(prediction[0]) - reference = self.clean(reference) - - if ("'" in prediction) != ("'" in reference): - # referent is "Bob's hat" as predicting the referent. - predicted_referent = False - else: - prediction_words = set(prediction.split(" ")) - referent_words = set(reference.split(" ")) - - # Handle cases where the prediction is "fuzzy bunny" and the referent is - # "bunny". - predicted_referent = prediction_words.issubset( - referent_words - ) or referent_words.issubset(prediction_words) - - filtered_resps.append(predicted_referent) - - return filtered_resps +DETERMINERS = { + "a", + "an", + "few", + "her", + "his", + "each", + "every", + "many", + "much", + "my", + "our", + "some", + "that", + "the", + "their", + "these", + "this", + "those", + "which", + "whose", + "your", +} + + +def clean(s: str) -> str: + """Ignore capitalization and determiners.""" + s = s.strip().lower() + return " ".join([w for w in s.split(" ") if w not in DETERMINERS]) + + +def process_results(docs: dict, resps: List): + prediction = clean(resps[0]) + reference = clean(docs["span1_text"]) + + if ("'" in prediction) != ("'" in reference): + # referent is "Bob's hat" as predicting the referent. + predicted_referent = False + else: + prediction_words = set(prediction.split(" ")) + referent_words = set(reference.split(" ")) + + # Handle cases where the prediction is "fuzzy bunny" and the referent is + # "bunny". + predicted_referent = prediction_words.issubset( + referent_words + ) or referent_words.issubset(prediction_words) + + acc = 1.0 if predicted_referent == docs["label"] else 0.0 + return {"accuracy": acc} diff --git a/lm_eval/tasks/xwinograd/utils.py b/lm_eval/tasks/xwinograd/utils.py index 3bb6b1452e..97c93c7072 100644 --- a/lm_eval/tasks/xwinograd/utils.py +++ b/lm_eval/tasks/xwinograd/utils.py @@ -51,7 +51,7 @@ def gen_lang_yamls(output_dir: str, overwrite: bool) -> None: for lang in LANGUAGES: file_name = f"xwinograd_{lang}.yaml" try: - with open(f"{output_dir}/{file_name}", "w" if overwrite else "x") as f: + with open(f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf-8") as f: f.write("# Generated by utils.py\n") yaml.dump( { diff --git a/lm_eval/utils.py b/lm_eval/utils.py index f0ec330e02..d86d9e3845 100644 --- a/lm_eval/utils.py +++ b/lm_eval/utils.py @@ -472,6 +472,10 @@ def get_git_commit_hash(): return git_hash +def ignore_constructor(loader, node): + return node + + def import_function(loader, node): function_name = loader.construct_scalar(node) yaml_path = os.path.dirname(loader.name) @@ -489,11 +493,14 @@ def import_function(loader, node): return function -# Add the import_function constructor to the YAML loader -yaml.add_constructor("!function", import_function) - +def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None, mode="full"): + if mode == "simple": + constructor_fn = ignore_constructor + elif mode == "full": + constructor_fn = import_function -def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None): + # Add the import_function constructor to the YAML loader + yaml.add_constructor("!function", constructor_fn) if yaml_config is None: with open(yaml_path, "rb") as file: yaml_config = yaml.full_load(file) @@ -521,7 +528,7 @@ def load_yaml_config(yaml_path=None, yaml_config=None, yaml_dir=None): path = os.path.join(yaml_dir, path) try: - included_yaml_config = load_yaml_config(path) + included_yaml_config = load_yaml_config(yaml_path=path, mode=mode) final_yaml_config.update(included_yaml_config) except Exception as ex: # If failed to load, ignore diff --git a/pyproject.toml b/pyproject.toml index 466f708429..6b60368f3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "lm_eval" -version = "0.4.0" +version = "0.4.1" authors = [ {name="EleutherAI", email="contact@eleuther.ai"} ] @@ -21,7 +21,7 @@ license = { "text" = "MIT" } dependencies = [ "accelerate>=0.21.0", "evaluate", - "datasets>=2.0.0", + "datasets>=2.14.0", "evaluate>=0.4.0", "jsonlines", "numexpr", @@ -56,15 +56,16 @@ Repository = "https://github.com/EleutherAI/lm-evaluation-harness" [project.optional-dependencies] anthropic = ["anthropic"] dev = ["pytest", "pytest-cov", "pytest-xdist", "pre-commit", "mypy"] -gptq = ["auto-gptq[triton] @ git+https://github.com/PanQiWei/AutoGPTQ"] +gptq = ["auto-gptq[triton]>=0.6.0"] +hf_transfer = ["hf_transfer"] ifeval = ["langdetect", "immutabledict"] +neuronx = ["optimum[neuronx]"] mamba = ["mamba_ssm", "causal-conv1d==1.0.2"] math = ["sympy>=1.12", "antlr4-python3-runtime==4.11"] multilingual = ["nagisa>=0.2.7", "jieba>=0.42.1", "pycountry"] openai = ["openai==1.3.9", "tiktoken"] -promptsource = [ - "promptsource @ git+https://github.com/bigscience-workshop/promptsource.git#egg=promptsource" -] +optimum = ["optimum[openvino]"] +promptsource = ["promptsource>=0.2.3"] sentencepiece = ["sentencepiece>=0.1.98", "protobuf>=4.22.1"] testing = ["pytest", "pytest-cov", "pytest-xdist"] vllm = ["vllm<=0.2.5"] @@ -73,8 +74,8 @@ all = [ "lm_eval[anthropic]", "lm_eval[dev]", "lm_eval[gptq]", + "lm_eval[hf_transfer]", "lm_eval[ifeval]", - "lm_eval[linting]", "lm_eval[mamba]", "lm_eval[math]", "lm_eval[multilingual]", @@ -87,7 +88,7 @@ all = [ ] [tool.ruff] -extend-exclude = ["lm_eval/evaluator.py", "lm_eval/tasks/*.py"] +extend-exclude = ["lm_eval/tasks/*.py"] [tool.ruff.lint] extend-select = ["I"] diff --git a/scripts/build_benchmark.py b/scripts/build_benchmark.py index ce4b661681..fc99b5ec37 100644 --- a/scripts/build_benchmark.py +++ b/scripts/build_benchmark.py @@ -23,7 +23,7 @@ def parse_args(): if __name__ == "__main__": args = parse_args() - with open(args.benchmark_path) as file: + with open(args.benchmark_path, encoding="utf-8") as file: TASK_LIST = yaml.full_load(file) for task in tqdm(TASK_LIST): eval_logger.info(f"Processing {task}") @@ -57,5 +57,5 @@ def parse_args(): file_save_path = os.path.join(file_path, full_file_name) eval_logger.info(f"Save to {file_save_path}") - with open(file_save_path, "w") as yaml_file: + with open(file_save_path, "w", encoding="utf-8") as yaml_file: yaml.dump(config_dict, yaml_file) diff --git a/scripts/clean_training_data/README.md b/scripts/clean_training_data/README.md index 67e8c3fba4..b8264d7f9b 100644 --- a/scripts/clean_training_data/README.md +++ b/scripts/clean_training_data/README.md @@ -30,4 +30,7 @@ pip install pybind11 c++ -O3 -Wall -shared -std=c++11 -fPIC $(python3 -m pybind11 --includes) janitor_util.cpp -o janitor_util$(python3-config --extension-suffix) ``` -If your your compiler isn't linked to python, you may need to add to the above `-undefined dynamic_lookup` +MacOS users: If your compiler isn't linked to Python, you may need to add to the above `-undefined dynamic_lookup`. \ +Linux users: If your compiler isn't linked to Python, you may need to follow these steps: +1. Rename the compiled code file to `janitor_util.so`. +2. Before running `import Janitor` in your code, add `sys.path.append("your/relative/path/to/janitor_util.so")` so that Python knows the location of `janitor_util.so`. diff --git a/scripts/clean_training_data/generate_13_grams.py b/scripts/clean_training_data/generate_13_grams.py index 66fa0ff45b..e508f266e9 100644 --- a/scripts/clean_training_data/generate_13_grams.py +++ b/scripts/clean_training_data/generate_13_grams.py @@ -119,7 +119,7 @@ def close_buckets(self): def do_ngrams_in_buckets(n_value, working_directory, bucket_count): - pile_statistics = json.load(open("pile_statistics.json", "r")) + pile_statistics = json.load(open("pile_statistics.json", "r", encoding="utf-8")) pile_document_count = pile_statistics["Document Count"] start_offsets = pile_statistics["File Start Offsets"] @@ -212,4 +212,4 @@ def do_ngrams_in_buckets(n_value, working_directory, bucket_count): info_dict = {"title": "dataset ngrams", "ngram_size": 13} info_dict_path = os.path.join(args.working_directory, "info.json") - json.dump(info_dict, open(info_dict_path, "w")) + json.dump(info_dict, open(info_dict_path, "w", encoding="utf-8")) diff --git a/scripts/clean_training_data/investigate_pile.py b/scripts/clean_training_data/investigate_pile.py index c1d348d463..681b591ced 100644 --- a/scripts/clean_training_data/investigate_pile.py +++ b/scripts/clean_training_data/investigate_pile.py @@ -79,7 +79,7 @@ def on_error(_): stats_file_path = "pile_statistics.json" if os.path.exists(stats_file_path): - stats = json.load(open(stats_file_path, "r")) + stats = json.load(open(stats_file_path, "r", encoding="utf-8")) else: document_count, total_document_size_chars, start_offsets = get_stats() stats = { @@ -88,7 +88,7 @@ def on_error(_): "Total Pile Characters": total_document_size_chars, "File Start Offsets": start_offsets, } - json.dump(stats, open(stats_file_path, "w"), indent=4) + json.dump(stats, open(stats_file_path, "w", encoding="utf-8"), indent=4) print(f"document_count: {stats['Document Count']}") print(f"total_chars: {stats['Total Pile Characters']}") diff --git a/scripts/make_table_results.py b/scripts/make_table_results.py index 72af524ffe..2893c2b0e5 100644 --- a/scripts/make_table_results.py +++ b/scripts/make_table_results.py @@ -61,14 +61,14 @@ def make_table(result_dict): if not filenames: continue path_readme = os.path.join(dirpath, "README.md") - with open(path_readme, "w") as f: + with open(path_readme, "w", encoding="utf-8") as f: # get path name, only last folder path_name = dirpath.split("/")[-1] f.write(f"# {path_name} \n\n") for filename in sorted([f for f in filenames if f.endswith(".json")]): path = os.path.join(dirpath, filename) - with open(path, "r") as f: + with open(path, "r", encoding="utf-8") as f: result_dict = json.load(f) - with open(path_readme, "a") as f: + with open(path_readme, "a", encoding="utf-8") as f: f.write(f"## {filename} \n") f.write(f"{make_table(result_dict)} \n") diff --git a/scripts/make_table_tasks.py b/scripts/make_table_tasks.py index ded7c1a596..0c8c44bc65 100644 --- a/scripts/make_table_tasks.py +++ b/scripts/make_table_tasks.py @@ -50,5 +50,5 @@ def check(tf): values.append(v) writer.value_matrix = values table = writer.dumps() - with open(args.output, "w") as f: + with open(args.output, "w", encoding="utf-8") as f: f.write(table) diff --git a/scripts/regression.py b/scripts/regression.py index 2b8167c0eb..75258dcb64 100644 --- a/scripts/regression.py +++ b/scripts/regression.py @@ -94,7 +94,11 @@ def eval_models(args, branch=None): ret = os.system(command) - results[model] = json.load(open(output_path)) if ret == 0 else {"results": {}} + results[model] = ( + json.load(open(output_path, encoding="utf-8")) + if ret == 0 + else {"results": {}} + ) end_time = time.time() diff --git a/scripts/write_out.py b/scripts/write_out.py index 360b0b6271..abbfb46832 100644 --- a/scripts/write_out.py +++ b/scripts/write_out.py @@ -5,7 +5,7 @@ import numpy as np from lm_eval import tasks -from lm_eval.tasks import include_path, initialize_tasks +from lm_eval.tasks import TaskManager from lm_eval.utils import eval_logger, join_iters @@ -39,22 +39,21 @@ def main(): args = parse_args() np.random.seed(args.seed) - initialize_tasks(args.verbosity) - if args.include_path is not None: eval_logger.info(f"Including path: {args.include_path}") - include_path(args.include_path) + + task_manager = TaskManager(args.verbosity, include_path=args.include_path) if args.tasks == "all_tasks": - task_names = tasks.ALL_TASKS + task_names = task_manager.all_tasks else: task_names = args.tasks.split(",") - task_dict = tasks.get_task_dict(task_names) + task_dict = tasks.get_task_dict(task_names, task_manager) os.makedirs(args.output_base_path, exist_ok=True) for task_name, task in task_dict.items(): - if type(task) == tuple: - group_name, task = task + if isinstance(task, tuple): + _, task = task rnd = random.Random() rnd.seed(args.seed) diff --git a/scripts/zeno_visualize.py b/scripts/zeno_visualize.py index f4012afeca..c7b75a80e2 100644 --- a/scripts/zeno_visualize.py +++ b/scripts/zeno_visualize.py @@ -69,18 +69,20 @@ def main(): model_args = re.sub( "/|=", "__", - json.load(open(Path(args.data_path, model, "results.json")))["config"][ - "model_args" - ], + json.load( + open(Path(args.data_path, model, "results.json"), encoding="utf-8") + )["config"]["model_args"], ) with open( - Path(args.data_path, model, f"{model_args}_{task}.jsonl"), "r" + Path(args.data_path, model, f"{model_args}_{task}.jsonl"), + "r", + encoding="utf-8", ) as file: data = json.loads(file.read()) - configs = json.load(open(Path(args.data_path, model, "results.json")))[ - "configs" - ] + configs = json.load( + open(Path(args.data_path, model, "results.json"), encoding="utf-8") + )["configs"] config = configs[task] if model_index == 0: # Only need to assemble data for the first model @@ -124,7 +126,9 @@ def tasks_for_model(model: str, data_path: str): list: A list of tasks for the model. """ dir_path = Path(data_path, model) - config = (json.load(open(Path(dir_path, "results.json")))["configs"],) + config = ( + json.load(open(Path(dir_path, "results.json"), encoding="utf-8"))["configs"], + ) return list(config[0].keys()) diff --git a/tests/models/test_huggingface.py b/tests/models/test_huggingface.py index 323d664af8..73ddfba74f 100644 --- a/tests/models/test_huggingface.py +++ b/tests/models/test_huggingface.py @@ -11,20 +11,21 @@ from lm_eval.models.huggingface import HFLM -tasks.initialize_tasks() +task_manager = tasks.TaskManager() class Test_HFLM: torch.use_deterministic_algorithms(True) + task_list = task_manager.load_task_or_group(["arc_easy", "gsm8k", "wikitext"]) version_minor = sys.version_info.minor - multiple_choice_task = tasks.TASK_REGISTRY.get("arc_easy")() # type: ignore + multiple_choice_task = task_list["arc_easy"] # type: ignore multiple_choice_task.build_all_requests(limit=10, rank=0, world_size=1) MULTIPLE_CH: list[Instance] = multiple_choice_task.instances - generate_until_task = tasks.TASK_REGISTRY.get("gsm8k")() # type: ignore + generate_until_task = task_list["gsm8k"] # type: ignore generate_until_task.build_all_requests(limit=10, rank=0, world_size=1) generate_until_task._config.generation_kwargs["max_gen_toks"] = 10 generate_until: list[Instance] = generate_until_task.instances - rolling_task = tasks.TASK_REGISTRY.get("wikitext")() # type: ignore + rolling_task = task_list["wikitext"] # type: ignore rolling_task.build_all_requests(limit=10, rank=0, world_size=1) ROLLING: list[Instance] = rolling_task.instances diff --git a/tests/models/test_neuron_optimum.py b/tests/models/test_neuron_optimum.py new file mode 100644 index 0000000000..564d523039 --- /dev/null +++ b/tests/models/test_neuron_optimum.py @@ -0,0 +1,26 @@ +import pytest +import torch + +from lm_eval.models.neuron_optimum import wrap_constant_batch_size + + +def test_wrap_constant_batch_size(): + class Tester: + def __init__(self, batch_size): + self.batch_size = batch_size + + @wrap_constant_batch_size + def test_constant_batch_size(self, inputs): + assert len(inputs) == self.batch_size + return inputs + + batch_size_test = 8 + for i in range(1, batch_size_test + 1): + tensor = torch.ones([i, 2, 2]) + out = Tester(batch_size=batch_size_test).test_constant_batch_size(tensor) + torch.testing.assert_allclose(out, tensor) + + with pytest.raises(ValueError): + Tester(batch_size=batch_size_test).test_constant_batch_size( + torch.ones([batch_size_test + 1, 2, 2]) + ) diff --git a/tests/models/test_openvino.py b/tests/models/test_openvino.py new file mode 100644 index 0000000000..34fc416a6e --- /dev/null +++ b/tests/models/test_openvino.py @@ -0,0 +1,73 @@ +import random +import tempfile + +import pytest +from optimum.intel import OVModelForCausalLM +from transformers import AutoTokenizer + +import lm_eval.evaluator as evaluator +from lm_eval.api.registry import get_model + + +SUPPORTED_ARCHITECTURES_TASKS = { + "facebook/opt-125m": "lambada_openai", + "hf-internal-testing/tiny-random-gpt2": "wikitext", +} + + +@pytest.mark.parametrize("model_id,task", SUPPORTED_ARCHITECTURES_TASKS.items()) +def test_evaluator(model_id, task): + with tempfile.TemporaryDirectory() as tmpdirname: + model = OVModelForCausalLM.from_pretrained( + model_id, export=True, use_cache=True + ) + model.save_pretrained(tmpdirname) + tokenizer = AutoTokenizer.from_pretrained(model_id) + tokenizer.save_pretrained(tmpdirname) + + lm = get_model("openvino").create_from_arg_string( + f"pretrained={tmpdirname}", + { + "batch_size": 1, + "device": "cpu", + }, + ) + + def ll_fn(reqs): + for ctx, cont in [req.args for req in reqs]: + if len(ctx) == 0: + continue + # space convention + assert ctx[-1] != " " + assert cont[0] == " " or ctx[-1] == "\n" + + res = [] + + random.seed(42) + for _ in reqs: + res.append((-random.random(), False)) + + return res + + def ll_perp_fn(reqs): + for (string,) in [req.args for req in reqs]: + assert isinstance(string, str) + + res = [] + random.seed(42) + for _ in reqs: + res.append(-random.random()) + + return res + + lm.loglikelihood = ll_fn + lm.loglikelihood_rolling = ll_perp_fn + + limit = 10 + evaluator.simple_evaluate( + model=lm, + tasks=[task], + num_fewshot=0, + limit=limit, + bootstrap_iters=10, + ) diff --git a/tests/models/test_vllm.py b/tests/models/test_vllm.py index 1da8a48762..0fd81b1e70 100644 --- a/tests/models/test_vllm.py +++ b/tests/models/test_vllm.py @@ -7,6 +7,9 @@ from lm_eval.api.instance import Instance +task_manager = tasks.TaskManager() + + @pytest.mark.skip(reason="requires CUDA") class TEST_VLLM: vllm = pytest.importorskip("vllm") @@ -17,15 +20,15 @@ class TEST_VLLM: except ModuleNotFoundError: pass torch.use_deterministic_algorithms(True) - tasks.initialize_tasks() - multiple_choice_task = tasks.TASK_REGISTRY.get("arc_easy")() # type: ignore + task_list = task_manager.load_task_or_group(["arc_easy", "gsm8k", "wikitext"]) + multiple_choice_task = task_list["arc_easy"] # type: ignore multiple_choice_task.build_all_requests(limit=10, rank=0, world_size=1) MULTIPLE_CH: List[Instance] = multiple_choice_task.instances - generate_until_task = tasks.TASK_REGISTRY.get("gsm8k")() # type: ignore + generate_until_task = task_list["gsm8k"] # type: ignore generate_until_task.build_all_requests(limit=10, rank=0, world_size=1) generate_until_task._config.generation_kwargs["max_gen_toks"] = 10 generate_until: List[Instance] = generate_until_task.instances - rolling_task = tasks.TASK_REGISTRY.get("wikitext")() # type: ignore + rolling_task = task_list["wikitext"] # type: ignore rolling_task.build_all_requests(limit=10, rank=0, world_size=1) ROLLING: List[Instance] = rolling_task.instances diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index 825f57413d..a41076a9aa 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -6,11 +6,9 @@ # import lm_eval.models as models import lm_eval.api as api import lm_eval.evaluator as evaluator -import lm_eval.tasks as tasks +from lm_eval import tasks -tasks.initialize_tasks() - # TODO: more fine grained unit tests rather than this big honking integration # test once we break evaluator into smaller, more manageable pieces @@ -46,7 +44,8 @@ def test_evaluator(task_name: List[str], limit: int, model: str, model_args: str "device": None, }, ) - task_dict = tasks.get_task_dict(task_name, num_fewshot=0) + task_manager = tasks.TaskManager() + task_dict = tasks.get_task_dict(task_name, task_manager) e2 = evaluator.evaluate( lm=lm, diff --git a/tests/test_tasks.py b/tests/test_tasks.py index a0a6c7c2b3..6140bccb54 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -8,7 +8,7 @@ from .utils import new_tasks -tasks.initialize_tasks() +task_manager = tasks.TaskManager() # Default Task TASKS = ["arc_easy"] @@ -19,9 +19,9 @@ def task_class(): task_classes = new_tasks() # Check if task_classes is empty if task_classes: - return [tasks.TASK_REGISTRY.get(x)() for x in task_classes] + return list(task_manager.load_task_or_group(task_classes).values()) else: - return [tasks.TASK_REGISTRY.get(x)() for x in TASKS] + return list(task_manager.load_task_or_group(TASKS).values()) @pytest.fixture() diff --git a/tests/test_utils.py b/tests/test_utils.py index c50b16781f..f85419ca6f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,10 @@ import pytest -from lm_eval.utils import Collator, get_rolling_token_windows, make_disjoint_window +from lm_eval.utils import ( + Collator, + get_rolling_token_windows, + make_disjoint_window, +) # noinspection DuplicatedCode diff --git a/tests/utils.py b/tests/utils.py index fbdbb6a7fb..a3418206af 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -1,5 +1,4 @@ import os -from pathlib import Path from typing import List, Union from lm_eval.utils import load_yaml_config @@ -20,16 +19,18 @@ def load_changed_files(file_path: str) -> List[str]: # checks the txt file for list of changed files. -# if file ends with .yaml then check yaml for task name -# if file ends with .py then parse the folder for all yaml files +# if file ends with .yaml then check yaml and load the config. +# if the config task is a string, it's a task config. +# if the config task is a list, it's a group config. def parser(full_path: List[str]) -> List[str]: _output = set() for x in full_path: - if x.endswith(".yaml"): - _output.add(load_yaml_config(x)["task"]) - elif x.endswith(".py"): - path = [str(x) for x in (list(Path(x).parent.glob("*.yaml")))] - _output |= {load_yaml_config(x)["task"] for x in path} + if os.path.exists(x) and x.endswith(".yaml"): + config = load_yaml_config(x, mode="simple") + if isinstance(config["task"], str): + _output.add(config["task"]) + elif isinstance(config["task"], list): + _output.add(config["group"]) return list(_output)