Version 0.3.0

.github/workflows/python-package-pip.yaml

@@ -13,34 +13,50 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    # Checkout repository and submodules
-    - name: Checkout repository and submodules
+    # --- INSTALLATIONS ---
+    - name: Checkout repository
       uses: actions/checkout@v2
-    # Set up Python
+
     - name: Set up Python 3.8
       uses: actions/setup-python@v2
       with:
         python-version: 3.8
-    # Load cache of pip dependencies
-    - uses: actions/cache@master
+
+    - name: Load cache of pip dependencies
+      uses: actions/cache@master
       id: cache_requirements
       with:
         path: ${{ env.pythonLocation }}/lib/python3.8/site-packages/*
         key: ${{ runner.os }}-pip-${{ hashFiles('setup.cfg') }}
         restore-keys: |
           ${{ runner.os }}-pip-
           ${{ runner.os }}-
-    # Install pip dependencies if needed
-    - name: Install pip dependencies
-      if: steps.cache_requirements.outputs.cache-hit != 'true'
+
+    - name: Install pip dev dependencies + package if needed
+      if: steps.cache_requirements.outputs.cache-hit == 'false'
       run: |
         python -m pip install --upgrade pip
         python -m pip install -e .[dev]
-    # Run flake8
+
+    - name: Install package if needed
+      if: steps.cache_requirements.outputs.cache-hit == 'true'
+      run: |
+        python -m pip install -e . --no-dependencies
+
+    - name: Install soundfile
+      run: |
+        # For soundfile dep
+        sudo apt-get install libsndfile1
+
+    # --- TESTS ---
     - name: Lint with flake8
       run: |
         python -m flake8 --config .flake8 --exit-zero --show-source --statistics src
-    # Run black
+  
     - name: Check format with Black
       run: |
         python -m black --check --diff src
+      
+    - name: Print install info
+      run: |
+        aac-datasets-info

CHANGELOG.md

@@ -2,7 +2,28 @@
 
 All notable changes to this project will be documented in this file.
 
-## [0.2.0] (2022-08-30)
+## [0.3.0] 2022-09-28
+### Added
+- Add `column_names`, `info` and `shape` properties in datasets.
+- Add `is_loaded` and `set_transform` methods in datasets.
+- Add column argument for method `getitem` in datasets.
+- Entrypoints for command line scripts `aac-datasets-check`, `aac-datasets-download` and `aac-datasets-info`.
+
+### Changed
+- Enforce datasets order to sort by filename to avoid different orders returned by `os.listdir`.
+- Function `check_directory` now returns the length of each dataset found in directory.
+- Rename `get_field` methods in datasets by `at` and add support for Iterable of keys and None key.
+- Change `at` arguments order and names.
+- Split `BasicCollate` into 2 classes: `BasicCollate` without padding and `AdvancedCollate` with padding options.
+- Weak private methods are now strongly private in datasets.
+- Rename `item_transform` to `transform` in datasets.
+- Rename `load_tags` to `with_tags` in `AudioCaps`.
+
+### Fixed
+- AudioCaps loading when `with_tags` is False.
+- Clotho files download.
+
+## [0.2.0] 2022-08-30
 ### Added
 - CHANGELOG file.
 - First version of the API documentation.

CITATION.cff

@@ -9,14 +9,17 @@ authors:
     family-names: Labbé
     affiliation: IRIT
     orcid: 'https://orcid.org/0000-0002-7219-5463'
-repository-code: 'https://github.com/Labbeti/aac_datasets/'
-abstract: Automated Audio Captioning datasets for Pytorch.
+repository-code: 'https://github.com/Labbeti/aac-datasets/'
+abstract: Audio Captioning datasets for Pytorch.
 keywords:
   - audio
+  - deep-learning
   - pytorch
+  - dataset
   - caption
   - datasets
   - captioning
+  - audio-captioning
 license: MIT
-version: 0.2.0
-date-released: '2022-08-30'
+version: 0.3.0
+date-released: '2022-09-28'

MANIFEST.in

@@ -0,0 +1,4 @@
+
+recursive-include . *.py
+global-exclude *.pyc
+global-exclude __pycache__

README.md

@@ -2,14 +2,14 @@
 
 <div align="center">
 
-# Automated Audio Captioning datasets for Pytorch
+# Audio Captioning datasets for Pytorch
 
 <a href="https://www.python.org/"><img alt="Python" src="https://img.shields.io/badge/-Python 3.8+-blue?style=for-the-badge&logo=python&logoColor=white"></a>
 <a href="https://pytorch.org/get-started/locally/"><img alt="PyTorch" src="https://img.shields.io/badge/-PyTorch 1.10.1-ee4c2c?style=for-the-badge&logo=pytorch&logoColor=white"></a>
 <a href="https://black.readthedocs.io/en/stable/"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-black.svg?style=for-the-badge&labelColor=gray"></a>
-<a href="https://github.com/Labbeti/aac_datasets/actions"><img alt="Build" src="https://img.shields.io/github/workflow/status/Labbeti/aac_datasets/Python%20package%20using%20Pip/main?style=for-the-badge&logo=github"></a>
+<a href="https://github.com/Labbeti/aac-datasets/actions"><img alt="Build" src="https://img.shields.io/github/workflow/status/Labbeti/aac-datasets/Python%20package%20using%20Pip/main?style=for-the-badge&logo=github"></a>
 
-Automated Audio Captioning Unofficial datasets source code for **AudioCaps** [1], **Clotho** [2], and **MACS** [3], designed for Pytorch.
+Audio Captioning unofficial datasets source code for **AudioCaps** [1], **Clotho** [2], and **MACS** [3], designed for Pytorch.
 
 </div>
 
@@ -32,7 +32,7 @@ audio, captions = item["audio"], item["captions"]
 # captions: list of str captions
 ```
 
-### Build Pytorch dataloader with MACS
+### Build Pytorch dataloader with Clotho
 
 ```python
 from torch.utils.data.dataloader import DataLoader
@@ -42,9 +42,9 @@ from aac_datasets.utils import BasicCollate
 dataset = Clotho(root=".", download=True)
 dataloader = DataLoader(dataset, batch_size=4, collate_fn=BasicCollate())
 
-for audio_batch, captions_batch in dataloader:
-    # audio_batch: Tensor of shape (batch_size, n_channels, audio_max_size)
-    # captions_batch: list of list of str
+for batch in dataloader:
+    # batch["audio"]: list of Tensor of shape (n_channels, audio_size)
+    # batch["captions"]: list of list of str
     ...
 ```
 
@@ -124,8 +124,8 @@ If you use this software, please consider cite it as below :
     license = {MIT},
     month = {6},
     title = {{aac-datasets}},
-    url = {https://github.com/Labbeti/aac_datasets/},
-    version = {0.1.1},
+    url = {https://github.com/Labbeti/aac-datasets/},
+    version = {0.3.0},
     year = {2022}
 }
 ```

docs/aac_datasets.info.rst

@@ -0,0 +1,7 @@
+aac\_datasets.info module
+=========================
+
+.. automodule:: aac_datasets.info
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/aac_datasets.rst

@@ -13,6 +13,7 @@ Subpackages
    :maxdepth: 4
 
    aac_datasets.datasets
+   aac_datasets.utils
 
 Submodules
 ----------
@@ -22,5 +23,4 @@ Submodules
 
    aac_datasets.check
    aac_datasets.download
-   aac_datasets.utils
-   aac_datasets.version
+   aac_datasets.info

docs/aac_datasets.utils.collate.rst

@@ -0,0 +1,7 @@
+aac\_datasets.utils.collate module
+==================================
+
+.. automodule:: aac_datasets.utils.collate
+   :members:
+   :undoc-members:
+   :show-inheritance:

docs/aac_datasets.utils.rst

@@ -1,7 +1,15 @@
-aac\_datasets.utils module
-==========================
+aac\_datasets.utils package
+===========================
 
 .. automodule:: aac_datasets.utils
    :members:
    :undoc-members:
    :show-inheritance:
+
+Submodules
+----------
+
+.. toctree::
+   :maxdepth: 4
+
+   aac_datasets.utils.collate

docs/index.rst

@@ -3,14 +3,15 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-Welcome to aac-datasets's documentation!
+aac-datasets's documentation
 ========================================
 
 .. toctree::
    :maxdepth: 4
    :caption: Contents:
 
    aac_datasets
+   usage
 
 
 Indices and tables

docs/modules.rst

@@ -0,0 +1,7 @@
+aac_datasets
+============
+
+.. toctree::
+   :maxdepth: 4
+
+   aac_datasets

docs/usage.rst

@@ -0,0 +1,70 @@
+Usage
+========================
+
+Download a dataset
+########################
+
+You can download each dataset subset by using the download=True option in dataset constructor.
+
+.. code-block:: python
+    :caption: Download Clotho development dataset (python).
+
+    from aac_datasets import Clotho
+
+    _ = Clotho("/my/path/to/data", subset="dev", download=True)
+
+You can also do the same by the command line :
+
+.. code-block:: bash
+    :caption: Download Clotho development dataset (command).
+
+    aacd-download --root "/my/path/to/data" clotho --subsets dev
+
+
+Load data
+########################
+
+Datasets are standard pytorch `Dataset`, and the can be loaded with squared brackets `[]`:
+
+.. code-block:: python
+    :caption: Load an item.
+
+    from aac_datasets import Clotho
+
+    clotho_dev_ds = Clotho("/my/path/to/data", subset="dev")
+    print(clotho_dev_ds[0])
+    # Returns the first Clotho item (audio, captions and metadata...) as dict.
+    # {"audio": tensor([[...]]), "captions": ['A wild...', ...], ...}
+
+
+You can also use the method `at` to get a specific data in the dataset:
+
+.. code-block:: python
+    :caption: Load only the captions.
+
+    print(clotho_dev_ds.at(0, "captions"))
+    # Returns the 5 captions of the first audio (WITHOUT loading audio):
+    # ['A wild assortment of birds are chirping and calling out in nature.',
+    #  'Several different types of bird are tweeting and making calls.',
+    #  'Birds tweeting and chirping happily, engine in the distance.',
+    #  'An assortment of  wild birds are chirping and calling out in nature.',
+    #  'Birds are chirping and making loud bird noises.']
+
+
+Build PyTorch DataLoader
+########################
+
+Pytorch DataLoader can be easely created for AAC datasets if you override the `collate_fn` argument.
+
+.. code-block:: python
+    :caption: Build pytorch dataloader.
+
+    from aac_datasets import Clotho
+    from aac_datasets.utils import BasicCollate
+
+    clotho_dev_ds = Clotho("/my/path/to/data", subset="dev")
+    collate = BasicCollate()
+
+    loader = DataLoader(clotho_dev_ds, batch_size=32, collate_fn=collate)
+    for batch in loader:
+        ...

examples/dataloader.ipynb

@@ -53,11 +53,29 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import aac_datasets\n",
     "import yaml\n",
     "\n",
     "from aac_datasets import Clotho"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "aac-datasets version: 0.2.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(f\"aac-datasets version: {aac_datasets.__version__}\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 3,
@@ -155,7 +173,7 @@
     "dataloader = DataLoader(clotho_dev, batch_size=4, collate_fn=BasicCollate())\n",
     "\n",
     "batch_0 = next(iter(dataloader))\n",
-    "batch_0_audio, batch_0_captions = batch_0\n",
+    "batch_0_audio, batch_0_captions = batch_0[\"audio\"], batch_0[\"captions\"]\n",
     "\n",
     "print(f\"Batch 0 audio shape: {batch_0_audio.shape}\")\n",
     "print(f\"Batch 0 captions:\\n{yaml.dump(batch_0_captions, sort_keys=False)}\")"
@@ -164,7 +182,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3.10.4 ('env_test')",
+   "display_name": "Python 3.9.7 ('env_aac')",
    "language": "python",
    "name": "python3"
   },
@@ -178,12 +196,12 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.4"
+   "version": "3.9.7"
   },
   "orig_nbformat": 4,
   "vscode": {
    "interpreter": {
-    "hash": "adbf2a06558efebe75cd4d4584e84bb0c5aedfcd5d209081fe4ee5869df3f6d4"
+    "hash": "7063ab30e97d60a2fe29668a999abf1b5c461060cdc7a81c0f2e53653790b9f3"
    }
   }
  },