Reading AXF METAR and SYNOP files.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from os.path import join as pjoin\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "from glob import glob\n",
    "from Utilities.dynarray import DynamicRecArray as recarray\n",
    "from Utilities.metutils import convert\n",
    "# IDY03101 - METAR data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def parseTime(datestr, timestr):\n",
    "    return datetime.strptime(\"{0} {1}\".format(datestr, timestr), \"%Y%m%d %H%M\")\n",
    "\n",
    "METAR_DTYPE=[(\"stnWMO\", '|S5'), (\"stnCode\", \"|S4\"), (\"dtDate\", \"|S8\"), (\"dtTime\", \"|S4\"), \n",
    "       (\"stnLat\", \"f8\"), (\"stnLon\", \"f8\"), (\"winddir\", \"f8\"), (\"windspeed\", \"f8\"), \n",
    "       (\"tempDB\", \"f8\"), (\"dewpt\", \"f8\"), (\"QNH\", \"f8\"), (\"RF9am\", \"f8\"), (\"RF10min\", \"f8\"),\n",
    "       (\"vis\", \"f8\"), (\"Avis\", \"f8\"), (\"gust\", \"f8\")]\n",
    "METAR_NAMES = [field[0] for field in METAR_DTYPE]\n",
    "METAR_COLS = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]\n",
    "METAR_CONV = {'stnCode': lambda s: s.strip('\"'),\n",
    "              'windspeed': lambda s: convert(s, \"kts\", \"mps\"),\n",
    "              'gust': lambda s: convert(s, \"kts\", \"mps\")}\n",
    "\n",
    "SYNOP_DTYPE = [('stnWMO', '|S5'), ('stnName', '|S31'), ('stnCode', '|S4'), ('dtDate', '|S8'), \n",
    "               ('dtTime', '|S4'), ('stnLat', 'f8'), ('stnLon', 'f8'), ('winddir', 'f8'), \n",
    "               ('windspeed', 'f8'), ('vis', 'i'), ('presentWxCode', '|S8'), ('pastWxCode', 'i'),\n",
    "               ('mslp', 'f8'), ('tempDB', 'f8'), ('dewpt', 'f8')]\n",
    "SYNOP_NAMES = [field[0] for field in SYNOP_DTYPE]\n",
    "SYNOP_COLS = [0, 1, 2, 3, 4, 5, 6, 10, 11, 12, 13, 14, 16, 21, 22]\n",
    "SYNOP_CONV = {'stnName': lambda s: s.strip('\"').rstrip(' '),\n",
    "              'stnCode': lambda s: s.strip('\"').rstrip(' '),\n",
    "              'presentWxCode': lambda s: s.strip('\"').rstrip(' ')}\n",
    "\n",
    "inputPath = \"N:/climate_change/CHARS/B_Wind/data/raw/obs/axf/\"\n",
    "outputPath = \"N:/climate_change/CHARS/B_Wind/data/derived/obs/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "metarbasename = \"IDY03101.*.axf\"\n",
    "\n",
    "metardata = {}\n",
    "\n",
    "filelist = glob(pjoin(inputPath, metarbasename))\n",
    "for f in filelist:\n",
    "    arr = np.genfromtxt(f, dtype=METAR_DTYPE, delimiter=',', skip_header=2,\n",
    "                        skip_footer=1, usecols=METAR_COLS, names=METAR_NAMES,\n",
    "                        autostrip=True, converters=METAR_CONV)\n",
    "    for i in range(len(arr)):\n",
    "        key = \"{0}-{1}\".format(arr[\"stnWMO\"][i], arr['stnCode'][i])\n",
    "        if metardata.has_key(key):\n",
    "            metardata[key] = np.append(metardata[key], arr[i])\n",
    "        else:\n",
    "            metardata[key] = recarray(METAR_DTYPE)\n",
    "            metardata[key] = arr[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "metarfmt = ['%s', '%s', '%s', '%s', '%6.2f', '%6.2f', '%5.1f', \n",
    "           '%5.1f', '%5.1f', '%5.1f', '%6.1f', '%5.1f', '%5.1f', \n",
    "           '%7.1f', '%7.1f', '%6.1f']\n",
    "for key in metardata.keys():\n",
    "    fname = pjoin(outputPath, \"metar/{0}.csv\".format(key))\n",
    "    df = pd.DataFrame(metardata[key])\n",
    "    dfdup = df.drop_duplicates(['dtDate', 'dtTime'])\n",
    "    da = dfdup.to_records(index=False)\n",
    "    np.savetxt(fname, da, fmt=metarfmt, delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "synopbasename = \"IDY03000.*.axf\"\n",
    "synopdata = {}\n",
    "\n",
    "filelist = glob(pjoin(inputPath, synopbasename))\n",
    "for f in filelist:\n",
    "    try:\n",
    "        arr = np.genfromtxt(f, dtype=SYNOP_DTYPE, delimiter=',', skip_header=2,\n",
    "                            skip_footer=1, usecols=SYNOP_COLS, names=SYNOP_NAMES,\n",
    "                            autostrip=True, converters=SYNOP_CONV)\n",
    "    except IndexError:\n",
    "        print f\n",
    "    if arr.size == 1:\n",
    "        key = \"{0}-{1}\".format(arr[\"stnWMO\"], arr['stnCode'])\n",
    "        if synopdata.has_key(key):\n",
    "            synopdata[key] = np.append(synopdata[key], arr)\n",
    "        else:\n",
    "            synopdata[key] = recarray(SYNOP_DTYPE)\n",
    "            synopdata[key] = arr\n",
    "    else:\n",
    "        for i in range(len(arr)):\n",
    "            key = \"{0}-{1}\".format(arr[\"stnWMO\"][i], arr['stnCode'][i])\n",
    "            if synopdata.has_key(key):\n",
    "                synopdata[key] = np.append(synopdata[key], arr[i])\n",
    "            else:\n",
    "                synopdata[key] = recarray(SYNOP_DTYPE)\n",
    "                synopdata[key] = arr[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "synopfmt = ['%s', '%s', '%s', '%s', '%s', '%6.2f', '%6.2f', '%5.1f', \n",
    "            '%5.1f', '%d', '%s', '%d', '%6.1f', '%4.1f', '%4.1f']\n",
    "for key in synopdata.keys():\n",
    "    fname = pjoin(outputPath, \"synop/{0}.csv\".format(key))\n",
    "    try:\n",
    "        df = pd.DataFrame(synopdata[key])\n",
    "        dfdup = df.drop_duplicates(['dtDate', 'dtTime'])\n",
    "        da = dfdup.to_records(index=False)\n",
    "        np.savetxt(fname, da, fmt=synopfmt, delimiter=',')\n",
    "    except IndexError:\n",
    "        print key"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}