data_processing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import json\n",
    "import pretty_midi as pm\n",
    "import sys\n",
    "import mido\n",
    "\n",
    "from tiv import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MIDI processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "is_executing": false
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def midifile_to_dict(mid):\n",
    "    tracks = []\n",
    "    for track in mid.tracks:\n",
    "        tracks.append([vars(msg).copy() for msg in track])\n",
    "    return {\n",
    "        'ticks_per_beat': mid.ticks_per_beat,\n",
    "        'tracks': tracks,\n",
    "    }\n",
    "\n",
    "def get_bpm(file_name):\n",
    "    mid = mido.MidiFile(file_name)\n",
    "    dict = midifile_to_dict(mid)\n",
    "    tempo = dict['tracks'][0][0]['tempo']\n",
    "    bpm = 0\n",
    "\n",
    "    if (bpm == 0):\n",
    "        bpm = mido.tempo2bpm(tempo)\n",
    "    else:\n",
    "        tempo = mido.bpm2tempo(bpm)\n",
    "\n",
    "    return bpm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "is_executing": false
    }
   },
   "outputs": [],
   "source": [
    "with open('chord_set/json/bar8_list.json', 'r') as bl:\n",
    "    bar8_list = json.load(bl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'what-hurts-the-most_intro'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bar8_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "a = pm.PrettyMIDI('midi/what-hurts-the-most_intro.mid')\n",
    "bpm = get_bpm('midi/what-hurts-the-most_intro.mid')\n",
    "bpm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "a.get_piano_roll(fs=bpm/60).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# get 8-bar chroma data (in format .npy), size: 32*12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "bar8_data = []\n",
    "\n",
    "for fn in bar8_list:\n",
    "    a = pm.PrettyMIDI('midi/'+fn+'.mid')\n",
    "    bpm = get_bpm('midi/'+fn+'.mid')\n",
    "    #print(bpm)\n",
    "#     if a.get_chroma(fs=bpm/60+0.001).shape != (12,32):\n",
    "#         print(0)\n",
    "    #print(a.get_chroma(fs=bpm/60+0.001).shape)\n",
    "    \n",
    "    bar8_data.append((a.get_chroma(fs=bpm/60+0.001)/100).transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "bar8_np = np.stack(bar8_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "np.save('bar8_cp_np.npy', bar8_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "b = np.load('data/bar8_cp_np.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32, 12)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "bar8_with_quality_data = []\n",
    "\n",
    "for fn in bar8_list:\n",
    "    a = pm.PrettyMIDI('midi/'+fn+'.mid')\n",
    "    bpm = get_bpm('midi/'+fn+'.mid')\n",
    "    \n",
    "    #print(bpm)\n",
    "#     if a.get_chroma(fs=bpm/60+0.001).shape != (12,32):\n",
    "#         print(0)\n",
    "    #print(a.get_chroma(fs=bpm/60+0.001).shape)\n",
    "    \n",
    "    bar8_with_quality_data.append((a.get_chroma(fs=bpm/60+0.001)/100).transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "bar8_with_quality_np = np.stack(bar8_with_quality_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "is_executing": false,
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "np.save('bar8_cp_with_quality_np.npy', bar8_with_quality_np)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "a.get_piano_roll(fs=bpm/60).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# get 8-bar chroma data (in format .npy), size: 32*12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "bar8_data = []\n",
    "\n",
    "for fn in bar8_list:\n",
    "    a = pm.PrettyMIDI('midi/'+fn+'.mid')\n",
    "    bpm = get_bpm('midi/'+fn+'.mid')\n",
    "    #print(bpm)\n",
    "#     if a.get_chroma(fs=bpm/60+0.001).shape != (12,32):\n",
    "#         print(0)\n",
    "    #print(a.get_chroma(fs=bpm/60+0.001).shape)\n",
    "    \n",
    "    bar8_data.append((a.get_chroma(fs=bpm/60+0.001)/100).transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "bar8_np = np.stack(bar8_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "np.save('bar8_cp_np.npy', bar8_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "b = np.load('bar8_cp_np.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "b[0] == bar8_np[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "jupyter": {
     "outputs_hidden": false
    },
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(128, 32)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a.get_piano_roll(fs=bpm/60).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# get 8-bar chroma data (in format .npy), size: 32*12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "bar8_data = []\n",
    "\n",
    "for fn in bar8_list:\n",
    "    a = pm.PrettyMIDI('midi/'+fn+'.mid')\n",
    "    bpm = get_bpm('midi/'+fn+'.mid')\n",
    "    #print(bpm)\n",
    "#     if a.get_chroma(fs=bpm/60+0.001).shape != (12,32):\n",
    "#         print(0)\n",
    "    #print(a.get_chroma(fs=bpm/60+0.001).shape)\n",
    "    \n",
    "    bar8_data.append((a.get_chroma(fs=bpm/60+0.001)/100).transpose())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "bar8_np = np.stack(bar8_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('bar8_cp_np.npy', bar8_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "b = np.load('bar8_cp_np.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True],\n",
       "       [ True,  True,  True,  True,  True,  True,  True,  True,  True,\n",
       "         True,  True,  True]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b[0] == bar8_np[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TIV processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_tiv_polar(chroma):\n",
    "    '''\n",
    "    input: 12 dims chroma\n",
    "    return: 12 dims TIV in polar coordinate, first 6 dims--mags, last 6 dims-- phases\n",
    "    '''\n",
    "    tiv = TIV.from_pcp(chroma)\n",
    "    tiv_polar = np.concatenate((tiv.mags(), tiv.phases()))\n",
    "    \n",
    "    return tiv_polar\n",
    "\n",
    "def get_tiv_polar_list(chroma_list):\n",
    "    tiv_list = chroma_list.copy()\n",
    "    for i in range(chroma_list.shape[0]):\n",
    "        tiv_list[i][:] = get_tiv_polar(chroma_list[i][:])\n",
    "    return tiv_list\n",
    "\n",
    "def chroma2tiv_np(chroma_chunk):\n",
    "    tiv_chunk = chroma_chunk.copy()\n",
    "    for i in range(chroma_chunk.shape[0]):\n",
    "        tiv_chunk[i] = get_tiv_polar_list(chroma_chunk[i])\n",
    "    return tiv_chunk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "bar8_cp_tiv_np = chroma2tiv_np(bar8_cp_np)\n",
    "bar8_cp_wq_tiv_np = chroma2tiv_np(bar8_cp_wq_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('bar8_cp_tiv_np.npy', bar8_cp_tiv_np)\n",
    "np.save('bar8_cp_wq_tiv_np.npy', bar8_cp_wq_tiv_np)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "pycharm": {
   "stem_cell": {
    "cell_type": "raw",
    "metadata": {
     "collapsed": false
    },
    "source": []
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}