cat_dog_SQL_readWav.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-14T02:24:43.642278Z",
     "start_time": "2017-11-14T02:24:41.065542Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import numpy as np\n",
    "import glob, os, re\n",
    "import librosa\n",
    "\n",
    "from pylab import *\n",
    "from scipy.fftpack import fft\n",
    "\n",
    "# And the display module for visualization\n",
    "import librosa.display\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from sqlalchemy import create_engine\n",
    "\n",
    "path = '/Volumes/ext200/Dropbox/metis/project3_mcnulty/pickle/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def catdog_files(direct, form, animal):\n",
    "    \"\"\" Generate a list of paths to the all the sound files in the directory, \n",
    "       \n",
    "   IN:  direct = path to directory where sound files are stored (string)\n",
    "         form = extention of the files (string) i.e. 'wav' \n",
    "                  \n",
    "    OUT: save to a .csv containing the path, filename and ID for each .wav file\"\"\"\n",
    "      \n",
    "    filenames = librosa.util.find_files(direct, ext=[form])\n",
    "    \n",
    "    # grab all the .wav files in the directory\n",
    "    os.chdir(direct)    \n",
    "    filez = []    \n",
    "    for file in glob.glob(\"*.wav\"):\n",
    "        filez.append(file)\n",
    "    \n",
    "    # get the number for each file for ID purposes store in file_id\n",
    "    file_id = []\n",
    "    for item in filez:\n",
    "        id_ = re.findall('[0-9]+', item)\n",
    "        id_ = str(id_)\n",
    "        id_ = id_.strip(\"'[]'\")\n",
    "        file_id.append(id_)\n",
    "    \n",
    "    # save to df\n",
    "    path_d = {'path':filenames,'file': filez, 'id': file_id}\n",
    "    path_df = pd.DataFrame(data=path_d)\n",
    "    \n",
    "    # save df as a csv file  \n",
    "    path_df.to_csv(animal + '.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 292,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Volumes/ext200/Dropbox/metis/project3_mcnulty\n",
      "/Volumes/ext200/Dropbox/metis/project3_mcnulty\n"
     ]
    }
   ],
   "source": [
    "dog_path= catdog_files('./dogs/', 'wav', 'dog')\n",
    "% cd ../\n",
    "cat_path= catdog_files('./cats/', 'wav','cat')\n",
    "% cd ../"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### (upload to aws EC2 and copy into SQL table was done in the terminal)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use sql table to get list of paths and save as data frames "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "engine_name = 'postgresql://ubuntu:pass@18.216.42.136:5432/ubuntu'\n",
    "cnx = create_engine(engine_name)\n",
    "#port ~ 5432"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# list of paths to .wav files for cats\n",
    "cat_path_list = pd.read_sql_query('''SELECT path  FROM cats ''',cnx)\n",
    "\n",
    "# list of IDs cats\n",
    "cat_id_list = pd.read_sql_query('''SELECT file  FROM cats ''',cnx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# list of paths to .wav files for dogs\n",
    "\n",
    "dog_path_list = pd.read_sql_query('''SELECT path  FROM dogs ''',cnx)\n",
    "\n",
    "# list of IDs dogs\n",
    "dog_id_list = pd.read_sql_query('''SELECT file  FROM dogs ''',cnx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pass dataframes into function that calculates the fft for each file and saves as binary file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 295,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_catdog_wav(direct, filenames, file_id, animal):\n",
    "    \"\"\" load .wav files, convert to floating point time series numpy array, \n",
    "    Calculate the FFT/PSD for each and save in binary file\n",
    "    -----------------------\n",
    "   \n",
    "   IN:  direct = path to directory where binary files will be saved\n",
    "        filenames = a dataframe of paths to the all the sound files in the directory\n",
    "        file_id = dataframe of the ID (number) associated with each sample \n",
    "        animal = animal that the file was recorded from \n",
    "         \n",
    "    OUT: binary files for each sample\n",
    "        fft = fft values (power)\n",
    "        freqs = freqs for fft\n",
    "        raw_ts = raw numpy array of wav file\"\"\"\n",
    "          \n",
    "    for i in range(len(filenames)):\n",
    "        \n",
    "        filename = filenames.iloc[i].values\n",
    "        filename = str(filename)\n",
    "        filename = filename.strip(\"'[]'\")\n",
    "        \n",
    "        id_ = file_id.iloc[i].values\n",
    "        id_ = str(id_)\n",
    "        id_ = id_.strip(\"'[]'\")\n",
    "       \n",
    "        #get raw time sereis and sample rate\n",
    "        #sampling rate which will be the default of 22050 \n",
    "        # y = audio as a np array\n",
    "        y, sr = librosa.load(filename)\n",
    "        \n",
    "        #psd \n",
    "        ps = np.abs(np.fft.fft(y))**2\n",
    "\n",
    "        ## Get frequencies corresponding to signal PSD\n",
    "        time_step = 1 / sr\n",
    "        \n",
    "        freqs1 = np.fft.fftfreq(y.size, time_step)\n",
    "\n",
    "        #only need the positive half\n",
    "        index_pos = freqs1 > 0\n",
    "        freqs = freqs1[index_pos]\n",
    "\n",
    "        periodo = ps[index_pos]\n",
    "        \n",
    "        # take log10 for decibel units and scaling \n",
    "        periodogram = 10 * np.log10(periodo)\n",
    "        \n",
    "        # save as binary files      \n",
    "        np.save( direct + animal + id_ + '_fft' , periodogram) \n",
    "        np.save( direct + animal + id_ + '_freqs' , freqs ) \n",
    "        np.save( direct + animal + id_ + '_raw_ts' , y ) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "load_catdog_wav('./cats/', cat_path_list, cat_id_list, 'cat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "load_catdog_wav('./dogs/', dog_path_list, dog_id_list, 'dog')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 297,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Volumes/ext200/Dropbox/metis/project3_mcnulty\n"
     ]
    }
   ],
   "source": [
    "%cd project3_mcnulty/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-11-14T02:25:01.104613Z",
     "start_time": "2017-11-14T02:25:01.093172Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  1.08300589e+00,   2.16601179e+00,   3.24901768e+00, ...,\n",
       "         1.10217510e+04,   1.10228340e+04,   1.10239170e+04])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " master_freq = np.load('master_frequencies.npy')\n",
    "master_freq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pass ffts into a function that matches the freqs to the master freqs , gets the index, and pulls the corresponding fft value. Save as a new binary file and a data frame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 298,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def resample_freq(direct):\n",
    "    \"\"\" load master frequencies as template. get each file's fft and \n",
    "    find the nearest frequency to the one in template, \n",
    "    then get the index for that freq and save, repeat till you get to 818\n",
    "   \n",
    "   IN:  direct = path to directory where fft and freq files are stored (string)\n",
    "         \n",
    "    OUT: fft_df = dataframe containing the fft values for each sample(row) by freq(col) \n",
    "         freq_df =df with the corresponding freqs \"\"\"\n",
    "    \n",
    "    def find_nearest_freq(array,value):\n",
    "        indexx = (np.abs(array-value)).argmin()\n",
    "        return indexx\n",
    "    \n",
    "    master_freq = np.load('master_frequencies.npy') \n",
    "    \n",
    "    # get the lists of the 2 binary files we will need\n",
    "    \n",
    "    os.chdir(direct)\n",
    "    file_freq= []\n",
    "    for file in glob.glob(\"*_freqs.npy\"):\n",
    "        file_freq.append(file)\n",
    "    \n",
    "    file_fft= []\n",
    "    for file in glob.glob(\"*_fft.npy\"):\n",
    "        file_fft.append(file)\n",
    "    \n",
    "    \n",
    "    # create new dfs\n",
    "    freq_df = pd.DataFrame(index=range(len(master_freq)),columns=range(len(file_fft)))\n",
    "    fft_df = pd.DataFrame(index=range(len(master_freq)),columns=range(len(file_fft)))\n",
    "  \n",
    "    col = 0\n",
    "    \n",
    "    # open the fft and freqs for a sample\n",
    "    for file1, file2 in zip(file_fft,file_freq):\n",
    "        old_fft = np.load(file1)\n",
    "        old_freq = np.load(file2)\n",
    "        idf = []\n",
    "    \n",
    "        #for each freq in master, find the index for  the nearest freq\n",
    "        for freq in master_freq:\n",
    "        \n",
    "            ind = find_nearest_freq(old_freq,freq)\n",
    "            idf.append(ind)\n",
    "            \n",
    "        # using our index, pull out freqs, and fft (power) values \n",
    "        new_fft = old_fft[idf]\n",
    "        new_freq = old_freq[idf]\n",
    "        \n",
    "        # make a new df for this animal\n",
    "        fft_df[col] = pd.Series(new_fft)\n",
    "        freq_df[col] = pd.Series(new_freq)\n",
    "        \n",
    "        #save as binary file\n",
    "        np.save('nfft', file1  , new_fft ) \n",
    "        np.save('nfreq' , file1 , new_freq ) \n",
    "\n",
    "        col += 1\n",
    "    \n",
    "    return fft_df, freq_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10179,)"
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# open file w/ master freqs\n",
    "master_freq = np.load('master_frequencies.npy') \n",
    "master_freq.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 300,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "C_fft_df, C_freq_df = resample_freq('./cats/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Volumes/ext200/Dropbox/metis/project3_mcnulty\n"
     ]
    }
   ],
   "source": [
    "%cd ../"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 306,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "D_fft_df, D_freq_df = resample_freq('./dogs/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# transpose the data frames so that features are the coloumns, and add a 'target' col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dog = D_fft_df.T\n",
    "dog['y_val'] = 1\n",
    "cat = C_fft_df.T\n",
    "cat['y_val'] = 0\n",
    "\n",
    "# concat into one\n",
    "both = pd.concat([cat,dog])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(path + 'both.pkl', 'wb') as picklefile:\n",
    "        pickle.dump(both, picklefile)    \n",
    "with open(path + 'dog.pkl', 'wb') as picklefile:\n",
    "        pickle.dump(dog, picklefile)    \n",
    "with open(path + 'cat.pkl', 'wb') as picklefile:\n",
    "        pickle.dump(cat, picklefile)            "
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Raw Cell Format",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}