diff --git a/Source/0_DataGeneration/placeholder b/Source/0_DataGeneration/placeholder deleted file mode 100644 index e69de29..0000000 diff --git a/Source/0_ModelTraining/CrossValidation.ipynb b/Source/0_ModelTraining/CrossValidation.ipynb new file mode 100644 index 0000000..5bef7be --- /dev/null +++ b/Source/0_ModelTraining/CrossValidation.ipynb @@ -0,0 +1,602 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "54891470-b955-49af-b7bd-07be107712e6", + "metadata": {}, + "source": [ + "# Training and Cross Validation" + ] + }, + { + "cell_type": "markdown", + "id": "3428d2c8-984e-42c3-81c5-220bd647d1d5", + "metadata": {}, + "source": [ + "## Load all required packages" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "579799e9-3336-453c-b933-2d35cbffbf18", + "metadata": {}, + "outputs": [], + "source": [ + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "336ea0f2-c8f8-4406-bb68-7153f38df3bb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2.7.9\n" + ] + } + ], + "source": [ + "import fastai\n", + "print(fastai.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "elementary-vault", + "metadata": {}, + "outputs": [], + "source": [ + "from fastai.vision.all import *" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "eba296f3-c3c1-40a4-b782-fe5e143b81ff", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from pathlib import Path\n", + "import rasterio\n", + "import rasterio.plot\n", + "import json\n", + "from py_linq import Enumerable\n", + "import pandas as pd" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "495478f3-e482-411c-a674-a74bd6ce8cc3", + "metadata": {}, + "outputs": [], + "source": [ + "from rasterio import logging" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3995a92f-be4d-4acb-bf1d-651a572affc5", + "metadata": {}, + "outputs": [], + "source": [ + "log = logging.getLogger()\n", + "log.setLevel(logging.ERROR)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "obvious-extra", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "module_path = os.path.abspath(os.path.join('..'))\n", + "if module_path not in sys.path:\n", + " sys.path.append(module_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "1c1b20fe-6d44-44dc-abca-3f540c0acfc6", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "622643a9-5863-4cb4-a9ae-f8bb97821b51", + "metadata": {}, + "outputs": [], + "source": [ + "import Helpers.MODIS8DaysHelper as mh\n", + "import Helpers.GEEHelpers as GEEHelpers\n", + "import Helpers.StaticFeaturesHelper as StaticFeaturesHelper" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "3328d6d7-f636-4293-a08e-9a9fd095137c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "importlib.reload(mh)\n", + "importlib.reload(GEEHelpers)\n", + "importlib.reload(StaticFeaturesHelper)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "6478233c-514d-4979-b311-f012570a82d4", + "metadata": {}, + "outputs": [], + "source": [ + "from ModelClasses.Model import CNNLSTM as CNNLSTM" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e8482fb-145a-4279-99e9-77200734b12e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "2bf89542-32bd-4c7b-93b9-7e7d2e077dfa", + "metadata": {}, + "source": [ + "## Define Data Path and load data references" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "id": "098dbc28-c469-43e3-a0c4-d23761faffe8", + "metadata": {}, + "outputs": [], + "source": [ + "dataPath = Path('../../Data/ModelData/Data')" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "064b639c-fb0f-467f-97c6-4f65140b27b2", + "metadata": {}, + "outputs": [], + "source": [ + "lstmDF = pd.read_json(dataPath/'lstmFiles.json') # dataframe for LSTM with corresponding data" + ] + }, + { + "cell_type": "markdown", + "id": "7bb6688b-7ed1-4f67-9403-b3f35107fc45", + "metadata": {}, + "source": [ + "## Define number of time steps for LSTM" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "87963ade-b336-4521-98f7-004c6af18b94", + "metadata": {}, + "outputs": [], + "source": [ + "timeSteps = 10" + ] + }, + { + "cell_type": "markdown", + "id": "aeb3db09-fff1-4b1a-93ea-3aeb3b14fe4a", + "metadata": {}, + "source": [ + "## Define functions to access file path and open files" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "1540353d-c714-4e16-a17a-9407a983b02a", + "metadata": {}, + "outputs": [], + "source": [ + "def getModisFileFromLabel(filePath):\n", + " fileDir = filePath.parent.parent/\"MOD09A1.061\"\n", + " return [fileDir/item for item in lstmDF[lstmDF.File == filePath.name].FeatureFiles.values[0]]" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "bbfdc095-58b4-439a-9d01-bd9d01d3dbd9", + "metadata": {}, + "outputs": [], + "source": [ + "def getStaticFeaturesFromLabel(filePath):\n", + " elevation = np.expand_dims(StaticFeaturesHelper.getScaledElevation(filePath.parent.parent/'Elevation'/('_'.join(filePath.stem.split('_')[0:2]) + '.tif')), 0)\n", + " slopeFile = np.expand_dims(StaticFeaturesHelper.getScaledHAND(filePath.parent.parent/'Slope'/('_'.join(filePath.stem.split('_')[0:2]) + '.tif')), 0)\n", + " hand = np.expand_dims(StaticFeaturesHelper.getSlope(filePath.parent.parent/'HAND'/('_'.join(filePath.stem.split('_')[0:2]) + '.tif')), 0)\n", + " return np.concatenate((elevation, slopeFile, hand))" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "db502cd6-576b-4b35-bf5f-b006efeaa478", + "metadata": {}, + "outputs": [], + "source": [ + "def readImage(file, bandsToUse):\n", + " return mh.getScaledModisFileBands(file, bandsToUse)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "9634c24d-05bf-48b0-a8d1-570593e5c24c", + "metadata": {}, + "outputs": [], + "source": [ + "# Open MODIS files and indices\n", + "def open_features(fn, chnls=None):\n", + " # Stack MODIS time steps\n", + " bandsToUse = ['sur_refl_b03', 'sur_refl_b02', 'sur_refl_b01', 'sur_refl_b04', 'sur_refl_b05', 'sur_refl_b06', 'sur_refl_b07']\n", + " files = getModisFileFromLabel(fn)[0:timeSteps]\n", + " \n", + " staticFeatures = getStaticFeaturesFromLabel(fn)\n", + " \n", + " img = np.empty((0,32,32))\n", + " for file in files:\n", + " try:\n", + " newimg = readImage(file, bandsToUse)\n", + " except:\n", + " newimg = readImage(file, bandsToUse)\n", + " \n", + " newimg = np.concatenate((newimg, staticFeatures))\n", + " img = np.concatenate((newimg, img))\n", + " \n", + " img = img.astype(np.float32)\n", + " img = torch.from_numpy(img)\n", + " return img\n", + "\n", + "# open ground truth\n", + "def open_mask(fn, chnls=None, cls=torch.Tensor):\n", + " img = np.expand_dims(rasterio.open(fn).read(1),0)\n", + " img = img.astype(np.float32)\n", + " npimg = torch.from_numpy(img)\n", + " clsImg = cls(npimg)\n", + " return clsImg" + ] + }, + { + "cell_type": "markdown", + "id": "fdfc6910-3a27-481b-b799-5b6291c257f6", + "metadata": {}, + "source": [ + "## define function wot work with multi-band data" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "df48333e-f04b-41e3-80c1-24427b895a56", + "metadata": {}, + "outputs": [], + "source": [ + "class MultiChannelTensorImage(TensorImage):\n", + " _show_args = ArrayImageBase._show_args\n", + " def show(self, channels=[1], ctx=None, vmin=None, vmax=None, **kwargs):\n", + " if len(channels) == 3: \n", + " return show_composite(self, channels=channels, ctx=ctx, vmin=vmin, vmax=vmax,\n", + " **{**self._show_args, **kwargs})\n", + " \n", + " \n", + " def norm(vals, vmin=None, vmax=None):\n", + " vmin = ifnone(vmin, np.quantile(vals, 0.01))\n", + " vmax = ifnone(vmax, np.quantile(vals, 0.99))\n", + " return (vals - vmin)/(vmax-vmin)\n", + "\n", + " def show_composite(img, channels, ax=None, figsize=(3,3), title=None, scale=True,\n", + " ctx=None, vmin=None, vmax=None, **kwargs)->plt.Axes:\n", + " \n", + " ax = ifnone(ax, ctx)\n", + " if ax is None: _, ax = plt.subplots() \n", + " r, g, b = channels\n", + " tempim = img.data.cpu().numpy()\n", + " im = np.zeros((tempim.shape[1], tempim.shape[2], 3))\n", + " im[...,0] = tempim[r]\n", + " im[...,1] = tempim[g]\n", + " im[...,2] = tempim[b]\n", + " if scale: im = norm(im, vmin, vmax)\n", + " ax.imshow(im, **kwargs)\n", + " ax.axis('off')\n", + " if title is not None: ax.set_title(title)\n", + " return ax\n", + "\n", + " @classmethod\n", + " def create(cls, fn, chans=None, **kwargs) ->None:\n", + " return cls(open_features(fn=fn, chnls=chans))\n", + " \n", + " def __repr__(self): return f'{self.__class__.__name__} size={\"x\".join([str(d) for d in self.shape])}'\n", + " \n", + "MultiChannelTensorImage.create = Transform(MultiChannelTensorImage.create)\n", + "\n", + "def MultiChannelImageBlock(cls=MultiChannelTensorImage, chans=None):\n", + " return TransformBlock(partial(cls.create, chans=chans))" + ] + }, + { + "cell_type": "markdown", + "id": "745692df-5418-4d81-80ec-4d24e6012421", + "metadata": {}, + "source": [ + "## create image blocks" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "3e8901c6-1bde-487b-bb21-829601cf2ffb", + "metadata": {}, + "outputs": [], + "source": [ + "# create image blocks\n", + "ImageBlock = MultiChannelImageBlock(chans=None)\n", + "MaskBlock = TransformBlock(type_tfms=[partial(open_mask, cls=TensorImage)])" + ] + }, + { + "cell_type": "markdown", + "id": "cd8be7bc-5228-4ddb-8ce5-88c3241f3121", + "metadata": {}, + "source": [ + "## Split dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "00a2c0f3-9661-49a8-bb9d-9ca901f0e304", + "metadata": {}, + "outputs": [], + "source": [ + "def FileSplitter(leaveOutYear):\n", + " def _func(x): return int(x.stem.split('_')[2]) >= int(GEEHelpers.GetGEETimeStampFromDate(leaveOutYear,1,1))\\\n", + " and int(x.stem.split('_')[2]) <= int(GEEHelpers.GetGEETimeStampFromDate(leaveOutYear,12,31))\n", + " def _inner(o, **kwargs): return FuncSplitter(_func)(o)\n", + " return _inner" + ] + }, + { + "cell_type": "markdown", + "id": "0e58986a-c5fa-4e21-a27f-dc1d93aae098", + "metadata": {}, + "source": [ + "## define function to get dataset" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "imported-transcription", + "metadata": {}, + "outputs": [], + "source": [ + "def getFilesForStudy(path, items=lstmDF.File.values):\n", + " return [path/('_'.join(item.split('_')[0:2]))/'Sen1FractionInundatedArea'/item for item in items]" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "15fccd7e-4dde-4b14-8ed0-d3cb08983437", + "metadata": {}, + "outputs": [], + "source": [ + "# uncomment this to check if items are found\n", + "# items = getFilesForStudy(dataPath, lstmDF.File.values)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fa022ce8-066f-48d0-916a-0a6993aca4ce", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "05085e75-de2a-4cbf-8be7-612c3be32d9e", + "metadata": {}, + "source": [ + "## Define model params, output dir and tfms" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "a6ae714d-6657-4ef6-a160-c59c3c661bcf", + "metadata": {}, + "outputs": [], + "source": [ + "(Path('models')/'CNNLSTM').mkdir(parents=True, exist_ok=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "id": "62aaf0b2-baaf-4dad-8b3a-537ad25050b0", + "metadata": {}, + "outputs": [], + "source": [ + "batch_tfms = [Rotate(), Flip(), Dihedral()]" + ] + }, + { + "cell_type": "markdown", + "id": "5e87505d-e2ec-4a89-aa57-04529ee6401e", + "metadata": {}, + "source": [ + "## Train Model" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "9d405346-bb55-420a-92ba-06cd69adc35c", + "metadata": {}, + "outputs": [], + "source": [ + "def train(leaveOutYear):\n", + " \n", + " # Define data loaders\n", + " db = DataBlock(blocks=(ImageBlock, MaskBlock),\n", + " get_items = getFilesForStudy,\n", + " splitter=FileSplitter(leaveOutYear),\n", + " batch_tfms = batch_tfms,\n", + " )\n", + "\n", + " dl = db.dataloaders(dataPath, num_workers=20, bs=128)#os.cpu_count()-20, num_workers=20, num_workers=int((os.cpu_count()-20) / 3)\n", + "\n", + " # Set model metrics\n", + " acc_metric = [mse, rmse, R2Score()]\n", + " loss_fn = MSELossFlat()\n", + "\n", + " # create model\n", + " model = CNNLSTM(nbTimeSteps = timeSteps)\n", + "\n", + " # create learner\n", + " learn = Learner(dl, model, loss_func = loss_fn, metrics=acc_metric, opt_func=ranger, cbs=CSVLogger(append=True, fname='history_' + str(leaveOutYear) + '.csv'))\n", + "\n", + " # in case we want to load a previous iteration of learning (also modify the for loop below)\n", + " # learn_iter = 0\n", + " # learn.load(\"'CNNLSTM/' + str(leaveOutYear) + str(0), with_opt=False)\n", + "\n", + " # in case we want to find the learning rate valley\n", + " # lr = learn.lr_find().valley\n", + " # print(lr)\n", + "\n", + " lrs = [.001, .0001, .00001]\n", + " # epochs = [20, 5, 5]\n", + " epochs = [3, 1, 1]\n", + "\n", + " # train\n", + " for k in range(3):\n", + " lrslice = slice(lrs[k])\n", + " learn.fit_flat_cos(epochs[k], lr=lrslice)\n", + " learn.save('CNNLSTM/' + str(leaveOutYear) + \"_\" + str(k))\n", + " print('done saving ' + str(k))\n", + "\n", + " print('Done with Leave-out year ' + str(leaveOutYear))" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "db084ffc-a918-4eb4-b0e0-b2fef714904e", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "# Loop through cross validated years\n", + "for leaveOutYear in range(2017, 2022):\n", + " print(' Starting Leave-out year ' + str(leaveOutYear))\n", + " train(leaveOutYear)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbce5cf4-23fb-49f7-82eb-34c9f4940e96", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1313297f-667f-442e-a506-2086289177b1", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "886eb264-6767-4843-884f-e88f6b341019", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a95b3558-b2fa-4407-bd3c-12941e5bf122", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "57b038c6-9bff-48d0-8693-a80245338cec", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/Source/1_Inference/InferenceAndPostProcessing.ipynb b/Source/1_Inference/InferenceAndPostProcessing.ipynb new file mode 100644 index 0000000..43a7056 --- /dev/null +++ b/Source/1_Inference/InferenceAndPostProcessing.ipynb @@ -0,0 +1,770 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f0c97d6f-be29-4d4e-9d4c-95df913abeaf", + "metadata": {}, + "source": [ + "# Inference" + ] + }, + { + "cell_type": "markdown", + "id": "e44bc5b6-ee0e-46ec-9f08-fbef736ee6b7", + "metadata": {}, + "source": [ + "## Load all required packages" + ] + }, + { + "cell_type": "code", + "execution_count": 63, + "id": "375407ae-eb55-4fcd-b65e-1cf2fb890694", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2.7.9\n" + ] + } + ], + "source": [ + "import fastai\n", + "print(fastai.__version__)" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "id": "elementary-vault", + "metadata": {}, + "outputs": [], + "source": [ + "from fastai.vision.all import *" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "id": "24e89830-8d12-449f-aab1-b0dab2d3b890", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from pathlib import Path\n", + "import rasterio\n", + "import rasterio.plot\n", + "import rioxarray as riox\n", + "from rioxarray.merge import merge_arrays\n", + "import json\n", + "from py_linq import Enumerable\n", + "import wget\n", + "import shutil\n", + "import multiprocessing as mp\n", + "from functools import partial" + ] + }, + { + "cell_type": "code", + "execution_count": 66, + "id": "obvious-extra", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "module_path = os.path.abspath(os.path.join('..'))\n", + "if module_path not in sys.path:\n", + " sys.path.append(module_path)" + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "id": "regional-orientation", + "metadata": {}, + "outputs": [], + "source": [ + "import importlib\n", + "import Helpers.MODIS8DaysHelper as mh\n", + "import Helpers.GEEHelpers as GEEHelpers\n", + "import Helpers.StaticFeaturesHelper as StaticFeaturesHelper" + ] + }, + { + "cell_type": "code", + "execution_count": 68, + "id": "ff87cc4b-434e-4796-8e42-b94dff3956bc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 68, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "importlib.reload(mh)\n", + "importlib.reload(GEEHelpers)\n", + "importlib.reload(StaticFeaturesHelper)" + ] + }, + { + "cell_type": "code", + "execution_count": 69, + "id": "d7a263fa-20bb-4f9c-bbd0-1b67de2cb1d3", + "metadata": {}, + "outputs": [], + "source": [ + "from ModelClasses.Model import CNNLSTM as CNNLSTM" + ] + }, + { + "cell_type": "markdown", + "id": "c589b4bd-cd2b-41f2-84ea-d7ab6c1251f5", + "metadata": {}, + "source": [ + "## Define Data Path and load data references" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "id": "477851ef-0c86-49f3-a570-beef36e5ce05", + "metadata": {}, + "outputs": [], + "source": [ + "dataPath = Path('../../Data/InferenceData/Data')" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "id": "064b639c-fb0f-467f-97c6-4f65140b27b2", + "metadata": {}, + "outputs": [], + "source": [ + "lstmDF = pd.read_json(dataPath/'lstmFilesInference.json')" + ] + }, + { + "cell_type": "markdown", + "id": "d097baed-1ad4-497e-985b-b35be0892cf5", + "metadata": {}, + "source": [ + "## Define number of time steps for LSTM" + ] + }, + { + "cell_type": "code", + "execution_count": 72, + "id": "0379c033-5621-429b-ab37-45a04637a6a6", + "metadata": {}, + "outputs": [], + "source": [ + "timeSteps = 10" + ] + }, + { + "cell_type": "markdown", + "id": "a89da150-bd7e-4f8f-80ef-a3d9d45ab7fd", + "metadata": {}, + "source": [ + "## Define functions to access file path and open files" + ] + }, + { + "cell_type": "code", + "execution_count": 73, + "id": "1cef4603-403d-4191-bc13-cf85f1710cc2", + "metadata": {}, + "outputs": [], + "source": [ + "def getStaticFeaturesFromLabel(filePath):\n", + " elevation = np.expand_dims(StaticFeaturesHelper.getScaledElevation(filePath.parent.parent/'Elevation'/('_'.join(filePath.stem.split('_')[0:2]) + '.tif')), 0)\n", + " slopeFile = np.expand_dims(StaticFeaturesHelper.getScaledHAND(filePath.parent.parent/'Slope'/('_'.join(filePath.stem.split('_')[0:2]) + '.tif')), 0)\n", + " hand = np.expand_dims(StaticFeaturesHelper.getSlope(filePath.parent.parent/'HAND'/('_'.join(filePath.stem.split('_')[0:2]) + '.tif')), 0)\n", + " return np.concatenate((elevation, slopeFile, hand))" + ] + }, + { + "cell_type": "code", + "execution_count": 74, + "id": "039baecc-1f47-4a1d-a3be-76462eae76e8", + "metadata": {}, + "outputs": [], + "source": [ + "def getModisFileFromLabel(filePath):\n", + " fileDir = filePath.parent\n", + " return [fileDir/item for item in lstmDF[lstmDF.File == filePath.name].FeatureFiles.values[0]]" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "id": "61ed3a37-15c1-4748-a0fc-cc4d5c9ab684", + "metadata": {}, + "outputs": [], + "source": [ + "def readImage(file, bandsToUse):\n", + " return mh.getScaledModisFileBands(file, bandsToUse)" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "id": "63e4fc17-0513-4fd9-bf55-21788387210d", + "metadata": {}, + "outputs": [], + "source": [ + "def open_features(fn, chnls=None):\n", + " bandsToUse = ['sur_refl_b03', 'sur_refl_b02', 'sur_refl_b01', 'sur_refl_b04', 'sur_refl_b05', 'sur_refl_b06', 'sur_refl_b07']\n", + " files = getModisFileFromLabel(fn)[0:timeSteps]\n", + " \n", + " staticFeatures = getStaticFeaturesFromLabel(fn)\n", + " \n", + " img = np.empty((0,32,32))\n", + " for file in files:\n", + " try:\n", + " newimg = readImage(file, bandsToUse)\n", + " except:\n", + " newimg = readImage(file, bandsToUse)\n", + " \n", + " newimg = np.concatenate((newimg, staticFeatures))\n", + " img = np.concatenate((newimg, img))\n", + " \n", + " img = img.astype(np.float32)\n", + " img = torch.from_numpy(img)\n", + " return img\n", + "\n", + "def open_mask(fn, chnls=None, cls=torch.Tensor):\n", + " img = np.expand_dims(rasterio.open(fn).read(1),0)\n", + " img = img.astype(np.float32)\n", + " npimg = torch.from_numpy(img)\n", + " clsImg = cls(npimg)\n", + " return clsImg" + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "id": "e068d761-bbef-466a-b881-2e0905162e31", + "metadata": {}, + "outputs": [], + "source": [ + "class MultiChannelTensorImage(TensorImage):\n", + " _show_args = ArrayImageBase._show_args\n", + " def show(self, channels=[1], ctx=None, vmin=None, vmax=None, **kwargs):\n", + " if len(channels) == 3: \n", + " return show_composite(self, channels=channels, ctx=ctx, vmin=vmin, vmax=vmax,\n", + " **{**self._show_args, **kwargs})\n", + " \n", + " \n", + " def norm(vals, vmin=None, vmax=None):\n", + " vmin = ifnone(vmin, np.quantile(vals, 0.01))\n", + " vmax = ifnone(vmax, np.quantile(vals, 0.99))\n", + " return (vals - vmin)/(vmax-vmin)\n", + "\n", + " def show_composite(img, channels, ax=None, figsize=(3,3), title=None, scale=True,\n", + " ctx=None, vmin=None, vmax=None, **kwargs)->plt.Axes:\n", + " \n", + " ax = ifnone(ax, ctx)\n", + " if ax is None: _, ax = plt.subplots() \n", + " r, g, b = channels\n", + " tempim = img.data.cpu().numpy()\n", + " im = np.zeros((tempim.shape[1], tempim.shape[2], 3))\n", + " im[...,0] = tempim[r]\n", + " im[...,1] = tempim[g]\n", + " im[...,2] = tempim[b]\n", + " if scale: im = norm(im, vmin, vmax)\n", + " ax.imshow(im, **kwargs)\n", + " ax.axis('off')\n", + " if title is not None: ax.set_title(title)\n", + " return ax\n", + "\n", + " @classmethod\n", + " def create(cls, fn, chans=None, **kwargs) ->None:\n", + " return cls(open_features(fn=fn, chnls=chans))\n", + " \n", + " def __repr__(self): return f'{self.__class__.__name__} size={\"x\".join([str(d) for d in self.shape])}'\n", + " \n", + "MultiChannelTensorImage.create = Transform(MultiChannelTensorImage.create)\n", + "\n", + "def MultiChannelImageBlock(cls=MultiChannelTensorImage, chans=None):\n", + " return TransformBlock(partial(cls.create, chans=chans))" + ] + }, + { + "cell_type": "markdown", + "id": "e6ca67af-b312-4a60-8c24-4b9e6aecc213", + "metadata": {}, + "source": [ + "## create image blocks" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "id": "7cf18395-f939-4cce-9f5d-dc96c0618652", + "metadata": {}, + "outputs": [], + "source": [ + "ImageBlock = MultiChannelImageBlock(chans=None)\n", + "MaskBlock = TransformBlock(type_tfms=[partial(open_mask, cls=TensorImage)])" + ] + }, + { + "cell_type": "markdown", + "id": "ee544584-a9e0-4d1e-9c14-68e46701a20e", + "metadata": {}, + "source": [ + "## Get files for inference" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "id": "imported-transcription", + "metadata": {}, + "outputs": [], + "source": [ + "def getFilesForStudy(path, items=lstmDF.File.values):\n", + " return [path/('_'.join(item.split('_')[0:2]))/'MOD09A1.061'/item for item in items]" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "id": "15fccd7e-4dde-4b14-8ed0-d3cb08983437", + "metadata": {}, + "outputs": [], + "source": [ + "items = getFilesForStudy(dataPath)" + ] + }, + { + "cell_type": "markdown", + "id": "d574edf1-b2a1-45e5-8bfa-e65663409870", + "metadata": {}, + "source": [ + "## Create data blocks" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "id": "moving-literacy", + "metadata": {}, + "outputs": [], + "source": [ + "db = DataBlock(blocks=(ImageBlock, MaskBlock),\n", + " get_items = getFilesForStudy\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "850d4bde-096f-48f6-bea4-10cc37e85917", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "662cb52c-0292-474f-b993-b5c18fa469e7", + "metadata": {}, + "source": [ + "## Run Inference" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "id": "913a84fa-1765-4f68-b474-09398e5ca4ac", + "metadata": {}, + "outputs": [], + "source": [ + "model = CNNLSTM(nbTimeSteps = timeSteps)" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "id": "piano-superintendent", + "metadata": {}, + "outputs": [], + "source": [ + "dl = db.dataloaders(dataPath, num_workers=10, bs=180)" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "id": "9e7eefa8-b4ba-4a07-bd8a-1f06bc77601e", + "metadata": {}, + "outputs": [], + "source": [ + "acc_metric = [mse, rmse, R2Score()]\n", + "loss_fn = MSELossFlat()\n", + "learn = Learner(dl, model, loss_func = loss_fn, metrics=acc_metric, opt_func=ranger)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a8a868ac-1644-4725-8874-6b3216291c71", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "d9b0c30c-8474-48ac-9ca6-445e339187bd", + "metadata": {}, + "source": [ + "### Get model weights" + ] + }, + { + "cell_type": "code", + "execution_count": 92, + "id": "14567488-4f72-4ec5-afad-a0e8f9be32bb", + "metadata": {}, + "outputs": [], + "source": [ + "modelFolder = Path('./models')\n", + "modelFolder.mkdir(exist_ok=True, parents=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "a48e7cf0-94fc-4d23-a356-4428c75b529c", + "metadata": {}, + "outputs": [], + "source": [ + "downloadLink = \"https://github.com/GieziJo/cvpr23-earthvision-CNN-LSTM-Inundation/releases/download/v1.0.0/ModelWeights.zip\"\n", + "zipFilePath = modelFolder/'ModelWeights.zip'\n", + "wget.download(downloadLink, out = (zipFilePath).as_posix())\n", + "shutil.unpack_archive(zipFilePath, modelFolder)" + ] + }, + { + "cell_type": "code", + "execution_count": 103, + "id": "b83171f4-7d45-4ec2-af1d-1bf6b9411d16", + "metadata": {}, + "outputs": [], + "source": [ + "modelFolder = Path('./ModelWeights')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "376aaf8f-95df-4162-8403-79aea6fcee5b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "e722a454-663d-4c5e-960b-bc70cf1198e1", + "metadata": {}, + "source": [ + "### Prepare output folders" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "id": "d5173568-8d42-41f9-9c98-53b2b835f969", + "metadata": {}, + "outputs": [], + "source": [ + "outputFolder = Path('./InferredData')\n", + "outputFolder.mkdir(exist_ok=True, parents=True)\n", + "\n", + "outputFolder_pickleData = outputFolder/'PickleData'\n", + "outputFolder_pickleData.mkdir(exist_ok=True, parents=True)\n", + "\n", + "outputFolder_individualTifs = outputFolder/'IndividualTifs'\n", + "outputFolder_individualTifs.mkdir(exist_ok=True, parents=True)\n", + "\n", + "outputFolder_fullTifs = outputFolder/'FullTifs'\n", + "outputFolder_fullTifs.mkdir(exist_ok=True, parents=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8994b6bf-9e3e-4867-8ad4-59c7f718cd8b", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "824d6e45-c42e-4dfa-87ca-3c5cf64a1627", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "0203e80b-6c9f-446d-a80f-de1e13477aac", + "metadata": {}, + "source": [ + "### Run inference and save results as raster" + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "id": "53ee6dce-723b-47d4-8949-dc3b7ce6b774", + "metadata": {}, + "outputs": [], + "source": [ + "# define raster creation function\n", + "def processResultAsRaster(k, items, targetPath, preds):\n", + " item = items[k]\n", + " # new file target path\n", + " fileTargetPath = targetPath/('_'.join(item.stem.split('_')[0:2]))\n", + " fileTargetPath.mkdir(exist_ok=True, parents=True)\n", + " \n", + " # new file\n", + " targetFile = fileTargetPath/item.name\n", + " if targetFile.exists():\n", + " return\n", + " \n", + " with rasterio.open(item) as r:\n", + " profile = r.profile.copy()\n", + " profile.update(count = 1)\n", + " with rasterio.open(targetFile, 'w', **profile) as dst:\n", + " dst.write(preds[k,::],1)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "786417d0-4446-4289-9cb8-17cb71e564f4", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a5c5c34-49c6-4a79-bfde-f39a6e3f2a11", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 114, + "id": "c9217be6-a615-43e2-8c7e-233f5e30f3ac", + "metadata": {}, + "outputs": [], + "source": [ + "test_dl = learn.dls.test_dl(items)\n", + "\n", + "# run inference for each leave-out year\n", + "for year in range(2017,2022):\n", + " featurePicklePath = outputFolder_pickleData/(str(year) + '_Features')\n", + " predictionPicklePath = outputFolder_pickleData/(str(year) + '_Infered')\n", + " \n", + " outputFolder_individualTifs_year = outputFolder_individualTifs/(str(year))\n", + " \n", + " # only run inference if pickle files don't exist\n", + " if not (featurePicklePath.exists() and predictionPicklePath.exists()):\n", + " \n", + " # load model for leave-out year\n", + " modelNamePath = modelFolder/str(year)\n", + " learn.load(modelNamePath)\n", + "\n", + " # infer for all items\n", + " preds, _ = learn.get_preds(dl=test_dl)\n", + " preds = preds.squeeze()\n", + "\n", + " # save pickel files in case something goes wrong\n", + " with open(featurePicklePath,\"wb\") as f:\n", + " pickle.dump(items, f)\n", + " with open(predictionPicklePath,\"wb\") as f:\n", + " pickle.dump(preds, f)\n", + " \n", + " else:\n", + " with open(predictionPicklePath, \"rb\") as f:\n", + " preds = np.array(pickle.load(f))\n", + " \n", + " # process results as tifs in parallel\n", + " func_part = partial(processResultAsRaster, items=items, targetPath=outputFolder_individualTifs_year, preds=preds)\n", + " _ = mp.Pool(10).map(func_part, range(len(items)))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9630fbfe-711b-4e6c-9508-1f24e7a20f52", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "5b7697f5-5789-4275-b477-d52ca991b530", + "metadata": {}, + "source": [ + "## Results Post-processing" + ] + }, + { + "cell_type": "markdown", + "id": "a0536dd5-ec51-4a26-842f-d2a38e0d47a3", + "metadata": {}, + "source": [ + "### create bangladesh full raster for each cross validated year and each time step" + ] + }, + { + "cell_type": "code", + "execution_count": 110, + "id": "a21b1759-04e2-410d-b656-924ff6dcf209", + "metadata": {}, + "outputs": [], + "source": [ + "def createRasterForTime(time, files, targetPath):\n", + " \n", + " fileName = targetPath/(str(time) + '.tif')\n", + " if fileName.exists():\n", + " return\n", + " \n", + " filesForTime = Enumerable(files).where(lambda item: int(item.stem.split('_')[2]) == time)\n", + " rasters = list(map(lambda item: riox.open_rasterio(item), filesForTime))\n", + " \n", + " gt = rasters[0].rio.transform()\n", + " res = (gt[0], -gt[4])\n", + " crs = str(rasters[0].rio.crs)\n", + " \n", + " merged_raster = merge_arrays(dataarrays = rasters, res = res, crs=crs, nodata = -9999)\n", + " \n", + " merged_raster.rio.to_raster(fileName)\n", + " \n", + " rasters = list(map(lambda item: item.close(), rasters))" + ] + }, + { + "cell_type": "code", + "execution_count": 112, + "id": "e9fa5e90-5d18-4409-9bd9-9ea65e52be4a", + "metadata": {}, + "outputs": [], + "source": [ + "times = np.sort(np.unique(list(map(lambda item: int(item.stem.split('_')[2]), items))))\n", + "\n", + "for year in range(2017,2022):\n", + " outputFolder_fullTifs_year = outputFolder_fullTifs/(str(year))\n", + " outputFolder_fullTifs_year.mkdir(exist_ok=True, parents=True)\n", + "\n", + " outputFolder_individualTifs_year = outputFolder_individualTifs/(str(year))\n", + " files = Enumerable(outputFolder_individualTifs_year.rglob(\"*\")).where(lambda p: p.suffix == '.tif').to_list()\n", + " \n", + " mp.Pool(10).map(partial(createRasterForTime, files = files, targetPath=outputFolder_fullTifs_year), times)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "77165a48-b909-4ebe-8eb9-8e6bbebb0faf", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d946a5d8-f432-4391-97cf-bd870fea2e55", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "4dc1a524-64d1-4b1c-bbd7-009a483cb89a", + "metadata": {}, + "source": [ + "### create bangladesh ensemble full raster for each time step from cross-validated models" + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "id": "ee8fee01-9de1-4750-a279-ad2ab549d638", + "metadata": {}, + "outputs": [], + "source": [ + "outputFolder_fullTifs_ensemble = outputFolder_fullTifs/'Ensemble'\n", + "outputFolder_fullTifs_ensemble.mkdir(exist_ok=True, parents=True)\n", + "\n", + "for time in times:\n", + " fileName = outputFolder_fullTifs_ensemble/(str(time) + '.tif')\n", + " \n", + " if fileName.exists():\n", + " continue\n", + " \n", + " vals = []\n", + " \n", + " for year in range(2017,2022):\n", + " file = outputFolder_fullTifs/(str(year))/(str(time) + '.tif')\n", + " with riox.open_rasterio(file) as r:\n", + " vals.append(r.values)\n", + " \n", + " if year == 2021:\n", + " out = np.median(np.array(vals), axis=0)\n", + " out[out < 0] = -9999\n", + " \n", + " r.values = out\n", + " r.rio.to_raster(fileName)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a15b5c22-dc36-4fa1-8b3d-5b774ee07d6d", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/Source/1_ModelTraining/placeholder b/Source/1_ModelTraining/placeholder deleted file mode 100644 index e69de29..0000000 diff --git a/Source/2_Inference/MODISInfo.csv b/Source/2_Inference/MODISInfo.csv deleted file mode 100644 index efa1eab..0000000 --- a/Source/2_Inference/MODISInfo.csv +++ /dev/null @@ -1,1012 +0,0 @@ -Time,Date,StartDate,EndDate -978652800000,2001-01-05,978307200000,978998400000 -979344000000,2001-01-13,978998400000,979689600000 -980035200000,2001-01-21,979689600000,980380800000 -980726400000,2001-01-29,980380800000,981072000000 -981417600000,2001-02-06,981072000000,981763200000 -982108800000,2001-02-14,981763200000,982454400000 -982800000000,2001-02-22,982454400000,983145600000 -983491200000,2001-03-02,983145600000,983836800000 -984182400000,2001-03-10,983836800000,984528000000 -984873600000,2001-03-18,984528000000,985219200000 -985564800000,2001-03-26,985219200000,985910400000 -986256000000,2001-04-03,985910400000,986601600000 -986947200000,2001-04-11,986601600000,987292800000 -987638400000,2001-04-19,987292800000,987984000000 -988329600000,2001-04-27,987984000000,988675200000 -989020800000,2001-05-05,988675200000,989366400000 -989712000000,2001-05-13,989366400000,990057600000 -990403200000,2001-05-21,990057600000,990748800000 -991094400000,2001-05-29,990748800000,991440000000 -991785600000,2001-06-06,991440000000,992131200000 -992476800000,2001-06-14,992131200000,992822400000 -993859200000,2001-06-30,993513600000,994204800000 -994550400000,2001-07-08,994204800000,994896000000 -995241600000,2001-07-16,994896000000,995587200000 -995932800000,2001-07-24,995587200000,996278400000 -996624000000,2001-08-01,996278400000,996969600000 -997315200000,2001-08-09,996969600000,997660800000 -998006400000,2001-08-17,997660800000,998352000000 -998697600000,2001-08-25,998352000000,999043200000 -999388800000,2001-09-02,999043200000,999734400000 -1000080000000,2001-09-10,999734400000,1000425600000 -1000771200000,2001-09-18,1000425600000,1001116800000 -1001462400000,2001-09-26,1001116800000,1001808000000 -1002153600000,2001-10-04,1001808000000,1002499200000 -1002844800000,2001-10-12,1002499200000,1003190400000 -1003536000000,2001-10-20,1003190400000,1003881600000 -1004227200000,2001-10-28,1003881600000,1004572800000 -1004918400000,2001-11-05,1004572800000,1005264000000 -1005609600000,2001-11-13,1005264000000,1005955200000 -1006300800000,2001-11-21,1005955200000,1006646400000 -1006992000000,2001-11-29,1006646400000,1007337600000 -1007683200000,2001-12-07,1007337600000,1008028800000 -1008374400000,2001-12-15,1008028800000,1008720000000 -1009065600000,2001-12-23,1008720000000,1009411200000 -1009756800000,2001-12-31,1009411200000,1010102400000 -1010188800000,2002-01-05,1009843200000,1010534400000 -1010880000000,2002-01-13,1010534400000,1011225600000 -1011571200000,2002-01-21,1011225600000,1011916800000 -1012262400000,2002-01-29,1011916800000,1012608000000 -1012953600000,2002-02-06,1012608000000,1013299200000 -1013644800000,2002-02-14,1013299200000,1013990400000 -1014336000000,2002-02-22,1013990400000,1014681600000 -1015027200000,2002-03-02,1014681600000,1015372800000 -1015718400000,2002-03-10,1015372800000,1016064000000 -1016409600000,2002-03-18,1016064000000,1016755200000 -1017100800000,2002-03-26,1016755200000,1017446400000 -1017792000000,2002-04-03,1017446400000,1018137600000 -1018483200000,2002-04-11,1018137600000,1018828800000 -1019174400000,2002-04-19,1018828800000,1019520000000 -1019865600000,2002-04-27,1019520000000,1020211200000 -1020556800000,2002-05-05,1020211200000,1020902400000 -1021248000000,2002-05-13,1020902400000,1021593600000 -1021939200000,2002-05-21,1021593600000,1022284800000 -1022630400000,2002-05-29,1022284800000,1022976000000 -1023321600000,2002-06-06,1022976000000,1023667200000 -1024012800000,2002-06-14,1023667200000,1024358400000 -1024704000000,2002-06-22,1024358400000,1025049600000 -1025395200000,2002-06-30,1025049600000,1025740800000 -1026086400000,2002-07-08,1025740800000,1026432000000 -1026777600000,2002-07-16,1026432000000,1027123200000 -1027468800000,2002-07-24,1027123200000,1027814400000 -1028160000000,2002-08-01,1027814400000,1028505600000 -1028851200000,2002-08-09,1028505600000,1029196800000 -1029542400000,2002-08-17,1029196800000,1029888000000 -1030233600000,2002-08-25,1029888000000,1030579200000 -1030924800000,2002-09-02,1030579200000,1031270400000 -1031616000000,2002-09-10,1031270400000,1031961600000 -1032307200000,2002-09-18,1031961600000,1032652800000 -1032998400000,2002-09-26,1032652800000,1033344000000 -1033689600000,2002-10-04,1033344000000,1034035200000 -1034380800000,2002-10-12,1034035200000,1034726400000 -1035072000000,2002-10-20,1034726400000,1035417600000 -1035763200000,2002-10-28,1035417600000,1036108800000 -1036454400000,2002-11-05,1036108800000,1036800000000 -1037145600000,2002-11-13,1036800000000,1037491200000 -1037836800000,2002-11-21,1037491200000,1038182400000 -1038528000000,2002-11-29,1038182400000,1038873600000 -1039219200000,2002-12-07,1038873600000,1039564800000 -1039910400000,2002-12-15,1039564800000,1040256000000 -1040601600000,2002-12-23,1040256000000,1040947200000 -1041292800000,2002-12-31,1040947200000,1041638400000 -1041724800000,2003-01-05,1041379200000,1042070400000 -1042416000000,2003-01-13,1042070400000,1042761600000 -1043107200000,2003-01-21,1042761600000,1043452800000 -1043798400000,2003-01-29,1043452800000,1044144000000 -1044489600000,2003-02-06,1044144000000,1044835200000 -1045180800000,2003-02-14,1044835200000,1045526400000 -1045872000000,2003-02-22,1045526400000,1046217600000 -1046563200000,2003-03-02,1046217600000,1046908800000 -1047254400000,2003-03-10,1046908800000,1047600000000 -1047945600000,2003-03-18,1047600000000,1048291200000 -1048636800000,2003-03-26,1048291200000,1048982400000 -1049328000000,2003-04-03,1048982400000,1049673600000 -1050019200000,2003-04-11,1049673600000,1050364800000 -1050710400000,2003-04-19,1050364800000,1051056000000 -1051401600000,2003-04-27,1051056000000,1051747200000 -1052092800000,2003-05-05,1051747200000,1052438400000 -1052784000000,2003-05-13,1052438400000,1053129600000 -1053475200000,2003-05-21,1053129600000,1053820800000 -1054166400000,2003-05-29,1053820800000,1054512000000 -1054857600000,2003-06-06,1054512000000,1055203200000 -1055548800000,2003-06-14,1055203200000,1055894400000 -1056240000000,2003-06-22,1055894400000,1056585600000 -1056931200000,2003-06-30,1056585600000,1057276800000 -1057622400000,2003-07-08,1057276800000,1057968000000 -1058313600000,2003-07-16,1057968000000,1058659200000 -1059004800000,2003-07-24,1058659200000,1059350400000 -1059696000000,2003-08-01,1059350400000,1060041600000 -1060387200000,2003-08-09,1060041600000,1060732800000 -1061078400000,2003-08-17,1060732800000,1061424000000 -1061769600000,2003-08-25,1061424000000,1062115200000 -1062460800000,2003-09-02,1062115200000,1062806400000 -1063152000000,2003-09-10,1062806400000,1063497600000 -1063843200000,2003-09-18,1063497600000,1064188800000 -1064534400000,2003-09-26,1064188800000,1064880000000 -1065225600000,2003-10-04,1064880000000,1065571200000 -1065916800000,2003-10-12,1065571200000,1066262400000 -1066608000000,2003-10-20,1066262400000,1066953600000 -1067299200000,2003-10-28,1066953600000,1067644800000 -1067990400000,2003-11-05,1067644800000,1068336000000 -1068681600000,2003-11-13,1068336000000,1069027200000 -1069372800000,2003-11-21,1069027200000,1069718400000 -1070064000000,2003-11-29,1069718400000,1070409600000 -1070755200000,2003-12-07,1070409600000,1071100800000 -1071446400000,2003-12-15,1071100800000,1071792000000 -1072137600000,2003-12-23,1071792000000,1072483200000 -1072828800000,2003-12-31,1072483200000,1073174400000 -1073260800000,2004-01-05,1072915200000,1073606400000 -1073952000000,2004-01-13,1073606400000,1074297600000 -1074643200000,2004-01-21,1074297600000,1074988800000 -1075334400000,2004-01-29,1074988800000,1075680000000 -1076025600000,2004-02-06,1075680000000,1076371200000 -1076716800000,2004-02-14,1076371200000,1077062400000 -1077408000000,2004-02-22,1077062400000,1077753600000 -1078099200000,2004-03-01,1077753600000,1078444800000 -1078790400000,2004-03-09,1078444800000,1079136000000 -1079481600000,2004-03-17,1079136000000,1079827200000 -1080172800000,2004-03-25,1079827200000,1080518400000 -1080864000000,2004-04-02,1080518400000,1081209600000 -1081555200000,2004-04-10,1081209600000,1081900800000 -1082246400000,2004-04-18,1081900800000,1082592000000 -1082937600000,2004-04-26,1082592000000,1083283200000 -1083628800000,2004-05-04,1083283200000,1083974400000 -1084320000000,2004-05-12,1083974400000,1084665600000 -1085011200000,2004-05-20,1084665600000,1085356800000 -1085702400000,2004-05-28,1085356800000,1086048000000 -1086393600000,2004-06-05,1086048000000,1086739200000 -1087084800000,2004-06-13,1086739200000,1087430400000 -1087776000000,2004-06-21,1087430400000,1088121600000 -1088467200000,2004-06-29,1088121600000,1088812800000 -1089158400000,2004-07-07,1088812800000,1089504000000 -1089849600000,2004-07-15,1089504000000,1090195200000 -1090540800000,2004-07-23,1090195200000,1090886400000 -1091232000000,2004-07-31,1090886400000,1091577600000 -1091923200000,2004-08-08,1091577600000,1092268800000 -1092614400000,2004-08-16,1092268800000,1092960000000 -1093305600000,2004-08-24,1092960000000,1093651200000 -1093996800000,2004-09-01,1093651200000,1094342400000 -1094688000000,2004-09-09,1094342400000,1095033600000 -1095379200000,2004-09-17,1095033600000,1095724800000 -1096070400000,2004-09-25,1095724800000,1096416000000 -1096761600000,2004-10-03,1096416000000,1097107200000 -1097452800000,2004-10-11,1097107200000,1097798400000 -1098144000000,2004-10-19,1097798400000,1098489600000 -1098835200000,2004-10-27,1098489600000,1099180800000 -1099526400000,2004-11-04,1099180800000,1099872000000 -1100217600000,2004-11-12,1099872000000,1100563200000 -1100908800000,2004-11-20,1100563200000,1101254400000 -1101600000000,2004-11-28,1101254400000,1101945600000 -1102291200000,2004-12-06,1101945600000,1102636800000 -1102982400000,2004-12-14,1102636800000,1103328000000 -1103673600000,2004-12-22,1103328000000,1104019200000 -1104364800000,2004-12-30,1104019200000,1104710400000 -1104883200000,2005-01-05,1104537600000,1105228800000 -1105574400000,2005-01-13,1105228800000,1105920000000 -1106265600000,2005-01-21,1105920000000,1106611200000 -1106956800000,2005-01-29,1106611200000,1107302400000 -1107648000000,2005-02-06,1107302400000,1107993600000 -1108339200000,2005-02-14,1107993600000,1108684800000 -1109030400000,2005-02-22,1108684800000,1109376000000 -1109721600000,2005-03-02,1109376000000,1110067200000 -1110412800000,2005-03-10,1110067200000,1110758400000 -1111104000000,2005-03-18,1110758400000,1111449600000 -1111795200000,2005-03-26,1111449600000,1112140800000 -1112486400000,2005-04-03,1112140800000,1112832000000 -1113177600000,2005-04-11,1112832000000,1113523200000 -1113868800000,2005-04-19,1113523200000,1114214400000 -1114560000000,2005-04-27,1114214400000,1114905600000 -1115251200000,2005-05-05,1114905600000,1115596800000 -1115942400000,2005-05-13,1115596800000,1116288000000 -1116633600000,2005-05-21,1116288000000,1116979200000 -1117324800000,2005-05-29,1116979200000,1117670400000 -1118016000000,2005-06-06,1117670400000,1118361600000 -1118707200000,2005-06-14,1118361600000,1119052800000 -1119398400000,2005-06-22,1119052800000,1119744000000 -1120089600000,2005-06-30,1119744000000,1120435200000 -1120780800000,2005-07-08,1120435200000,1121126400000 -1121472000000,2005-07-16,1121126400000,1121817600000 -1122163200000,2005-07-24,1121817600000,1122508800000 -1122854400000,2005-08-01,1122508800000,1123200000000 -1123545600000,2005-08-09,1123200000000,1123891200000 -1124236800000,2005-08-17,1123891200000,1124582400000 -1124928000000,2005-08-25,1124582400000,1125273600000 -1125619200000,2005-09-02,1125273600000,1125964800000 -1126310400000,2005-09-10,1125964800000,1126656000000 -1127001600000,2005-09-18,1126656000000,1127347200000 -1127692800000,2005-09-26,1127347200000,1128038400000 -1128384000000,2005-10-04,1128038400000,1128729600000 -1129075200000,2005-10-12,1128729600000,1129420800000 -1129766400000,2005-10-20,1129420800000,1130112000000 -1130457600000,2005-10-28,1130112000000,1130803200000 -1131148800000,2005-11-05,1130803200000,1131494400000 -1131840000000,2005-11-13,1131494400000,1132185600000 -1132531200000,2005-11-21,1132185600000,1132876800000 -1133222400000,2005-11-29,1132876800000,1133568000000 -1133913600000,2005-12-07,1133568000000,1134259200000 -1134604800000,2005-12-15,1134259200000,1134950400000 -1135296000000,2005-12-23,1134950400000,1135641600000 -1135987200000,2005-12-31,1135641600000,1136332800000 -1136419200000,2006-01-05,1136073600000,1136764800000 -1137110400000,2006-01-13,1136764800000,1137456000000 -1137801600000,2006-01-21,1137456000000,1138147200000 -1138492800000,2006-01-29,1138147200000,1138838400000 -1139184000000,2006-02-06,1138838400000,1139529600000 -1139875200000,2006-02-14,1139529600000,1140220800000 -1140566400000,2006-02-22,1140220800000,1140912000000 -1141257600000,2006-03-02,1140912000000,1141603200000 -1141948800000,2006-03-10,1141603200000,1142294400000 -1142640000000,2006-03-18,1142294400000,1142985600000 -1143331200000,2006-03-26,1142985600000,1143676800000 -1144022400000,2006-04-03,1143676800000,1144368000000 -1144713600000,2006-04-11,1144368000000,1145059200000 -1145404800000,2006-04-19,1145059200000,1145750400000 -1146096000000,2006-04-27,1145750400000,1146441600000 -1146787200000,2006-05-05,1146441600000,1147132800000 -1147478400000,2006-05-13,1147132800000,1147824000000 -1148169600000,2006-05-21,1147824000000,1148515200000 -1148860800000,2006-05-29,1148515200000,1149206400000 -1149552000000,2006-06-06,1149206400000,1149897600000 -1150243200000,2006-06-14,1149897600000,1150588800000 -1150934400000,2006-06-22,1150588800000,1151280000000 -1151625600000,2006-06-30,1151280000000,1151971200000 -1152316800000,2006-07-08,1151971200000,1152662400000 -1153008000000,2006-07-16,1152662400000,1153353600000 -1153699200000,2006-07-24,1153353600000,1154044800000 -1154390400000,2006-08-01,1154044800000,1154736000000 -1155081600000,2006-08-09,1154736000000,1155427200000 -1155772800000,2006-08-17,1155427200000,1156118400000 -1156464000000,2006-08-25,1156118400000,1156809600000 -1157155200000,2006-09-02,1156809600000,1157500800000 -1157846400000,2006-09-10,1157500800000,1158192000000 -1158537600000,2006-09-18,1158192000000,1158883200000 -1159228800000,2006-09-26,1158883200000,1159574400000 -1159920000000,2006-10-04,1159574400000,1160265600000 -1160611200000,2006-10-12,1160265600000,1160956800000 -1161302400000,2006-10-20,1160956800000,1161648000000 -1161993600000,2006-10-28,1161648000000,1162339200000 -1162684800000,2006-11-05,1162339200000,1163030400000 -1163376000000,2006-11-13,1163030400000,1163721600000 -1164067200000,2006-11-21,1163721600000,1164412800000 -1164758400000,2006-11-29,1164412800000,1165104000000 -1165449600000,2006-12-07,1165104000000,1165795200000 -1166140800000,2006-12-15,1165795200000,1166486400000 -1166832000000,2006-12-23,1166486400000,1167177600000 -1167523200000,2006-12-31,1167177600000,1167868800000 -1167955200000,2007-01-05,1167609600000,1168300800000 -1168646400000,2007-01-13,1168300800000,1168992000000 -1169337600000,2007-01-21,1168992000000,1169683200000 -1170028800000,2007-01-29,1169683200000,1170374400000 -1170720000000,2007-02-06,1170374400000,1171065600000 -1171411200000,2007-02-14,1171065600000,1171756800000 -1172102400000,2007-02-22,1171756800000,1172448000000 -1172793600000,2007-03-02,1172448000000,1173139200000 -1173484800000,2007-03-10,1173139200000,1173830400000 -1174176000000,2007-03-18,1173830400000,1174521600000 -1174867200000,2007-03-26,1174521600000,1175212800000 -1175558400000,2007-04-03,1175212800000,1175904000000 -1176249600000,2007-04-11,1175904000000,1176595200000 -1176940800000,2007-04-19,1176595200000,1177286400000 -1177632000000,2007-04-27,1177286400000,1177977600000 -1178323200000,2007-05-05,1177977600000,1178668800000 -1179014400000,2007-05-13,1178668800000,1179360000000 -1179705600000,2007-05-21,1179360000000,1180051200000 -1180396800000,2007-05-29,1180051200000,1180742400000 -1181088000000,2007-06-06,1180742400000,1181433600000 -1181779200000,2007-06-14,1181433600000,1182124800000 -1182470400000,2007-06-22,1182124800000,1182816000000 -1183161600000,2007-06-30,1182816000000,1183507200000 -1183852800000,2007-07-08,1183507200000,1184198400000 -1184544000000,2007-07-16,1184198400000,1184889600000 -1185235200000,2007-07-24,1184889600000,1185580800000 -1185926400000,2007-08-01,1185580800000,1186272000000 -1186617600000,2007-08-09,1186272000000,1186963200000 -1187308800000,2007-08-17,1186963200000,1187654400000 -1188000000000,2007-08-25,1187654400000,1188345600000 -1188691200000,2007-09-02,1188345600000,1189036800000 -1189382400000,2007-09-10,1189036800000,1189728000000 -1190073600000,2007-09-18,1189728000000,1190419200000 -1190764800000,2007-09-26,1190419200000,1191110400000 -1191456000000,2007-10-04,1191110400000,1191801600000 -1192147200000,2007-10-12,1191801600000,1192492800000 -1192838400000,2007-10-20,1192492800000,1193184000000 -1193529600000,2007-10-28,1193184000000,1193875200000 -1194220800000,2007-11-05,1193875200000,1194566400000 -1194912000000,2007-11-13,1194566400000,1195257600000 -1195603200000,2007-11-21,1195257600000,1195948800000 -1196294400000,2007-11-29,1195948800000,1196640000000 -1196985600000,2007-12-07,1196640000000,1197331200000 -1197676800000,2007-12-15,1197331200000,1198022400000 -1198368000000,2007-12-23,1198022400000,1198713600000 -1199059200000,2007-12-31,1198713600000,1199404800000 -1199491200000,2008-01-05,1199145600000,1199836800000 -1200182400000,2008-01-13,1199836800000,1200528000000 -1200873600000,2008-01-21,1200528000000,1201219200000 -1201564800000,2008-01-29,1201219200000,1201910400000 -1202256000000,2008-02-06,1201910400000,1202601600000 -1202947200000,2008-02-14,1202601600000,1203292800000 -1203638400000,2008-02-22,1203292800000,1203984000000 -1204329600000,2008-03-01,1203984000000,1204675200000 -1205020800000,2008-03-09,1204675200000,1205366400000 -1205712000000,2008-03-17,1205366400000,1206057600000 -1206403200000,2008-03-25,1206057600000,1206748800000 -1207094400000,2008-04-02,1206748800000,1207440000000 -1207785600000,2008-04-10,1207440000000,1208131200000 -1208476800000,2008-04-18,1208131200000,1208822400000 -1209168000000,2008-04-26,1208822400000,1209513600000 -1209859200000,2008-05-04,1209513600000,1210204800000 -1210550400000,2008-05-12,1210204800000,1210896000000 -1211241600000,2008-05-20,1210896000000,1211587200000 -1211932800000,2008-05-28,1211587200000,1212278400000 -1212624000000,2008-06-05,1212278400000,1212969600000 -1213315200000,2008-06-13,1212969600000,1213660800000 -1214006400000,2008-06-21,1213660800000,1214352000000 -1214697600000,2008-06-29,1214352000000,1215043200000 -1215388800000,2008-07-07,1215043200000,1215734400000 -1216080000000,2008-07-15,1215734400000,1216425600000 -1216771200000,2008-07-23,1216425600000,1217116800000 -1217462400000,2008-07-31,1217116800000,1217808000000 -1218153600000,2008-08-08,1217808000000,1218499200000 -1218844800000,2008-08-16,1218499200000,1219190400000 -1219536000000,2008-08-24,1219190400000,1219881600000 -1220227200000,2008-09-01,1219881600000,1220572800000 -1220918400000,2008-09-09,1220572800000,1221264000000 -1221609600000,2008-09-17,1221264000000,1221955200000 -1222300800000,2008-09-25,1221955200000,1222646400000 -1222992000000,2008-10-03,1222646400000,1223337600000 -1223683200000,2008-10-11,1223337600000,1224028800000 -1224374400000,2008-10-19,1224028800000,1224720000000 -1225065600000,2008-10-27,1224720000000,1225411200000 -1225756800000,2008-11-04,1225411200000,1226102400000 -1226448000000,2008-11-12,1226102400000,1226793600000 -1227139200000,2008-11-20,1226793600000,1227484800000 -1227830400000,2008-11-28,1227484800000,1228176000000 -1228521600000,2008-12-06,1228176000000,1228867200000 -1229212800000,2008-12-14,1228867200000,1229558400000 -1229904000000,2008-12-22,1229558400000,1230249600000 -1230595200000,2008-12-30,1230249600000,1230940800000 -1231113600000,2009-01-05,1230768000000,1231459200000 -1231804800000,2009-01-13,1231459200000,1232150400000 -1232496000000,2009-01-21,1232150400000,1232841600000 -1233187200000,2009-01-29,1232841600000,1233532800000 -1233878400000,2009-02-06,1233532800000,1234224000000 -1234569600000,2009-02-14,1234224000000,1234915200000 -1235260800000,2009-02-22,1234915200000,1235606400000 -1235952000000,2009-03-02,1235606400000,1236297600000 -1236643200000,2009-03-10,1236297600000,1236988800000 -1237334400000,2009-03-18,1236988800000,1237680000000 -1238025600000,2009-03-26,1237680000000,1238371200000 -1238716800000,2009-04-03,1238371200000,1239062400000 -1239408000000,2009-04-11,1239062400000,1239753600000 -1240099200000,2009-04-19,1239753600000,1240444800000 -1240790400000,2009-04-27,1240444800000,1241136000000 -1241481600000,2009-05-05,1241136000000,1241827200000 -1242172800000,2009-05-13,1241827200000,1242518400000 -1242864000000,2009-05-21,1242518400000,1243209600000 -1243555200000,2009-05-29,1243209600000,1243900800000 -1244246400000,2009-06-06,1243900800000,1244592000000 -1244937600000,2009-06-14,1244592000000,1245283200000 -1245628800000,2009-06-22,1245283200000,1245974400000 -1246320000000,2009-06-30,1245974400000,1246665600000 -1247011200000,2009-07-08,1246665600000,1247356800000 -1247702400000,2009-07-16,1247356800000,1248048000000 -1248393600000,2009-07-24,1248048000000,1248739200000 -1249084800000,2009-08-01,1248739200000,1249430400000 -1249776000000,2009-08-09,1249430400000,1250121600000 -1250467200000,2009-08-17,1250121600000,1250812800000 -1251158400000,2009-08-25,1250812800000,1251504000000 -1251849600000,2009-09-02,1251504000000,1252195200000 -1252540800000,2009-09-10,1252195200000,1252886400000 -1253232000000,2009-09-18,1252886400000,1253577600000 -1253923200000,2009-09-26,1253577600000,1254268800000 -1254614400000,2009-10-04,1254268800000,1254960000000 -1255305600000,2009-10-12,1254960000000,1255651200000 -1255996800000,2009-10-20,1255651200000,1256342400000 -1256688000000,2009-10-28,1256342400000,1257033600000 -1257379200000,2009-11-05,1257033600000,1257724800000 -1258070400000,2009-11-13,1257724800000,1258416000000 -1258761600000,2009-11-21,1258416000000,1259107200000 -1259452800000,2009-11-29,1259107200000,1259798400000 -1260144000000,2009-12-07,1259798400000,1260489600000 -1260835200000,2009-12-15,1260489600000,1261180800000 -1261526400000,2009-12-23,1261180800000,1261872000000 -1262217600000,2009-12-31,1261872000000,1262563200000 -1262649600000,2010-01-05,1262304000000,1262995200000 -1263340800000,2010-01-13,1262995200000,1263686400000 -1264032000000,2010-01-21,1263686400000,1264377600000 -1264723200000,2010-01-29,1264377600000,1265068800000 -1265414400000,2010-02-06,1265068800000,1265760000000 -1266105600000,2010-02-14,1265760000000,1266451200000 -1266796800000,2010-02-22,1266451200000,1267142400000 -1267488000000,2010-03-02,1267142400000,1267833600000 -1268179200000,2010-03-10,1267833600000,1268524800000 -1268870400000,2010-03-18,1268524800000,1269216000000 -1269561600000,2010-03-26,1269216000000,1269907200000 -1270252800000,2010-04-03,1269907200000,1270598400000 -1270944000000,2010-04-11,1270598400000,1271289600000 -1271635200000,2010-04-19,1271289600000,1271980800000 -1272326400000,2010-04-27,1271980800000,1272672000000 -1273017600000,2010-05-05,1272672000000,1273363200000 -1273708800000,2010-05-13,1273363200000,1274054400000 -1274400000000,2010-05-21,1274054400000,1274745600000 -1275091200000,2010-05-29,1274745600000,1275436800000 -1275782400000,2010-06-06,1275436800000,1276128000000 -1276473600000,2010-06-14,1276128000000,1276819200000 -1277164800000,2010-06-22,1276819200000,1277510400000 -1277856000000,2010-06-30,1277510400000,1278201600000 -1278547200000,2010-07-08,1278201600000,1278892800000 -1279238400000,2010-07-16,1278892800000,1279584000000 -1279929600000,2010-07-24,1279584000000,1280275200000 -1280620800000,2010-08-01,1280275200000,1280966400000 -1281312000000,2010-08-09,1280966400000,1281657600000 -1282003200000,2010-08-17,1281657600000,1282348800000 -1282694400000,2010-08-25,1282348800000,1283040000000 -1283385600000,2010-09-02,1283040000000,1283731200000 -1284076800000,2010-09-10,1283731200000,1284422400000 -1284768000000,2010-09-18,1284422400000,1285113600000 -1285459200000,2010-09-26,1285113600000,1285804800000 -1286150400000,2010-10-04,1285804800000,1286496000000 -1286841600000,2010-10-12,1286496000000,1287187200000 -1287532800000,2010-10-20,1287187200000,1287878400000 -1288224000000,2010-10-28,1287878400000,1288569600000 -1288915200000,2010-11-05,1288569600000,1289260800000 -1289606400000,2010-11-13,1289260800000,1289952000000 -1290297600000,2010-11-21,1289952000000,1290643200000 -1290988800000,2010-11-29,1290643200000,1291334400000 -1291680000000,2010-12-07,1291334400000,1292025600000 -1292371200000,2010-12-15,1292025600000,1292716800000 -1293062400000,2010-12-23,1292716800000,1293408000000 -1293753600000,2010-12-31,1293408000000,1294099200000 -1294185600000,2011-01-05,1293840000000,1294531200000 -1294876800000,2011-01-13,1294531200000,1295222400000 -1295568000000,2011-01-21,1295222400000,1295913600000 -1296259200000,2011-01-29,1295913600000,1296604800000 -1296950400000,2011-02-06,1296604800000,1297296000000 -1297641600000,2011-02-14,1297296000000,1297987200000 -1298332800000,2011-02-22,1297987200000,1298678400000 -1299024000000,2011-03-02,1298678400000,1299369600000 -1299715200000,2011-03-10,1299369600000,1300060800000 -1300406400000,2011-03-18,1300060800000,1300752000000 -1301097600000,2011-03-26,1300752000000,1301443200000 -1301788800000,2011-04-03,1301443200000,1302134400000 -1302480000000,2011-04-11,1302134400000,1302825600000 -1303171200000,2011-04-19,1302825600000,1303516800000 -1303862400000,2011-04-27,1303516800000,1304208000000 -1304553600000,2011-05-05,1304208000000,1304899200000 -1305244800000,2011-05-13,1304899200000,1305590400000 -1305936000000,2011-05-21,1305590400000,1306281600000 -1306627200000,2011-05-29,1306281600000,1306972800000 -1307318400000,2011-06-06,1306972800000,1307664000000 -1308009600000,2011-06-14,1307664000000,1308355200000 -1308700800000,2011-06-22,1308355200000,1309046400000 -1309392000000,2011-06-30,1309046400000,1309737600000 -1310083200000,2011-07-08,1309737600000,1310428800000 -1310774400000,2011-07-16,1310428800000,1311120000000 -1311465600000,2011-07-24,1311120000000,1311811200000 -1312156800000,2011-08-01,1311811200000,1312502400000 -1312848000000,2011-08-09,1312502400000,1313193600000 -1313539200000,2011-08-17,1313193600000,1313884800000 -1314230400000,2011-08-25,1313884800000,1314576000000 -1314921600000,2011-09-02,1314576000000,1315267200000 -1315612800000,2011-09-10,1315267200000,1315958400000 -1316304000000,2011-09-18,1315958400000,1316649600000 -1316995200000,2011-09-26,1316649600000,1317340800000 -1317686400000,2011-10-04,1317340800000,1318032000000 -1318377600000,2011-10-12,1318032000000,1318723200000 -1319068800000,2011-10-20,1318723200000,1319414400000 -1319760000000,2011-10-28,1319414400000,1320105600000 -1320451200000,2011-11-05,1320105600000,1320796800000 -1321142400000,2011-11-13,1320796800000,1321488000000 -1321833600000,2011-11-21,1321488000000,1322179200000 -1322524800000,2011-11-29,1322179200000,1322870400000 -1323216000000,2011-12-07,1322870400000,1323561600000 -1323907200000,2011-12-15,1323561600000,1324252800000 -1324598400000,2011-12-23,1324252800000,1324944000000 -1325289600000,2011-12-31,1324944000000,1325635200000 -1325721600000,2012-01-05,1325376000000,1326067200000 -1326412800000,2012-01-13,1326067200000,1326758400000 -1327104000000,2012-01-21,1326758400000,1327449600000 -1327795200000,2012-01-29,1327449600000,1328140800000 -1328486400000,2012-02-06,1328140800000,1328832000000 -1329177600000,2012-02-14,1328832000000,1329523200000 -1329868800000,2012-02-22,1329523200000,1330214400000 -1330560000000,2012-03-01,1330214400000,1330905600000 -1331251200000,2012-03-09,1330905600000,1331596800000 -1331942400000,2012-03-17,1331596800000,1332288000000 -1332633600000,2012-03-25,1332288000000,1332979200000 -1333324800000,2012-04-02,1332979200000,1333670400000 -1334016000000,2012-04-10,1333670400000,1334361600000 -1334707200000,2012-04-18,1334361600000,1335052800000 -1335398400000,2012-04-26,1335052800000,1335744000000 -1336089600000,2012-05-04,1335744000000,1336435200000 -1336780800000,2012-05-12,1336435200000,1337126400000 -1337472000000,2012-05-20,1337126400000,1337817600000 -1338163200000,2012-05-28,1337817600000,1338508800000 -1338854400000,2012-06-05,1338508800000,1339200000000 -1339545600000,2012-06-13,1339200000000,1339891200000 -1340236800000,2012-06-21,1339891200000,1340582400000 -1340928000000,2012-06-29,1340582400000,1341273600000 -1341619200000,2012-07-07,1341273600000,1341964800000 -1342310400000,2012-07-15,1341964800000,1342656000000 -1343001600000,2012-07-23,1342656000000,1343347200000 -1343692800000,2012-07-31,1343347200000,1344038400000 -1344384000000,2012-08-08,1344038400000,1344729600000 -1345075200000,2012-08-16,1344729600000,1345420800000 -1345766400000,2012-08-24,1345420800000,1346112000000 -1346457600000,2012-09-01,1346112000000,1346803200000 -1347148800000,2012-09-09,1346803200000,1347494400000 -1347840000000,2012-09-17,1347494400000,1348185600000 -1348531200000,2012-09-25,1348185600000,1348876800000 -1349222400000,2012-10-03,1348876800000,1349568000000 -1349913600000,2012-10-11,1349568000000,1350259200000 -1350604800000,2012-10-19,1350259200000,1350950400000 -1351296000000,2012-10-27,1350950400000,1351641600000 -1351987200000,2012-11-04,1351641600000,1352332800000 -1352678400000,2012-11-12,1352332800000,1353024000000 -1353369600000,2012-11-20,1353024000000,1353715200000 -1354060800000,2012-11-28,1353715200000,1354406400000 -1354752000000,2012-12-06,1354406400000,1355097600000 -1355443200000,2012-12-14,1355097600000,1355788800000 -1356134400000,2012-12-22,1355788800000,1356480000000 -1356825600000,2012-12-30,1356480000000,1357171200000 -1357344000000,2013-01-05,1356998400000,1357689600000 -1358035200000,2013-01-13,1357689600000,1358380800000 -1358726400000,2013-01-21,1358380800000,1359072000000 -1359417600000,2013-01-29,1359072000000,1359763200000 -1360108800000,2013-02-06,1359763200000,1360454400000 -1360800000000,2013-02-14,1360454400000,1361145600000 -1361491200000,2013-02-22,1361145600000,1361836800000 -1362182400000,2013-03-02,1361836800000,1362528000000 -1362873600000,2013-03-10,1362528000000,1363219200000 -1363564800000,2013-03-18,1363219200000,1363910400000 -1364256000000,2013-03-26,1363910400000,1364601600000 -1364947200000,2013-04-03,1364601600000,1365292800000 -1365638400000,2013-04-11,1365292800000,1365984000000 -1366329600000,2013-04-19,1365984000000,1366675200000 -1367020800000,2013-04-27,1366675200000,1367366400000 -1367712000000,2013-05-05,1367366400000,1368057600000 -1368403200000,2013-05-13,1368057600000,1368748800000 -1369094400000,2013-05-21,1368748800000,1369440000000 -1369785600000,2013-05-29,1369440000000,1370131200000 -1370476800000,2013-06-06,1370131200000,1370822400000 -1371168000000,2013-06-14,1370822400000,1371513600000 -1371859200000,2013-06-22,1371513600000,1372204800000 -1372550400000,2013-06-30,1372204800000,1372896000000 -1373241600000,2013-07-08,1372896000000,1373587200000 -1373932800000,2013-07-16,1373587200000,1374278400000 -1374624000000,2013-07-24,1374278400000,1374969600000 -1375315200000,2013-08-01,1374969600000,1375660800000 -1376006400000,2013-08-09,1375660800000,1376352000000 -1376697600000,2013-08-17,1376352000000,1377043200000 -1377388800000,2013-08-25,1377043200000,1377734400000 -1378080000000,2013-09-02,1377734400000,1378425600000 -1378771200000,2013-09-10,1378425600000,1379116800000 -1379462400000,2013-09-18,1379116800000,1379808000000 -1380153600000,2013-09-26,1379808000000,1380499200000 -1380844800000,2013-10-04,1380499200000,1381190400000 -1381536000000,2013-10-12,1381190400000,1381881600000 -1382227200000,2013-10-20,1381881600000,1382572800000 -1382918400000,2013-10-28,1382572800000,1383264000000 -1383609600000,2013-11-05,1383264000000,1383955200000 -1384300800000,2013-11-13,1383955200000,1384646400000 -1384992000000,2013-11-21,1384646400000,1385337600000 -1385683200000,2013-11-29,1385337600000,1386028800000 -1386374400000,2013-12-07,1386028800000,1386720000000 -1387065600000,2013-12-15,1386720000000,1387411200000 -1387756800000,2013-12-23,1387411200000,1388102400000 -1388448000000,2013-12-31,1388102400000,1388793600000 -1388880000000,2014-01-05,1388534400000,1389225600000 -1389571200000,2014-01-13,1389225600000,1389916800000 -1390262400000,2014-01-21,1389916800000,1390608000000 -1390953600000,2014-01-29,1390608000000,1391299200000 -1391644800000,2014-02-06,1391299200000,1391990400000 -1392336000000,2014-02-14,1391990400000,1392681600000 -1393027200000,2014-02-22,1392681600000,1393372800000 -1393718400000,2014-03-02,1393372800000,1394064000000 -1394409600000,2014-03-10,1394064000000,1394755200000 -1395100800000,2014-03-18,1394755200000,1395446400000 -1395792000000,2014-03-26,1395446400000,1396137600000 -1396483200000,2014-04-03,1396137600000,1396828800000 -1397174400000,2014-04-11,1396828800000,1397520000000 -1397865600000,2014-04-19,1397520000000,1398211200000 -1398556800000,2014-04-27,1398211200000,1398902400000 -1399248000000,2014-05-05,1398902400000,1399593600000 -1399939200000,2014-05-13,1399593600000,1400284800000 -1400630400000,2014-05-21,1400284800000,1400976000000 -1401321600000,2014-05-29,1400976000000,1401667200000 -1402012800000,2014-06-06,1401667200000,1402358400000 -1402704000000,2014-06-14,1402358400000,1403049600000 -1403395200000,2014-06-22,1403049600000,1403740800000 -1404086400000,2014-06-30,1403740800000,1404432000000 -1404777600000,2014-07-08,1404432000000,1405123200000 -1405468800000,2014-07-16,1405123200000,1405814400000 -1406160000000,2014-07-24,1405814400000,1406505600000 -1406851200000,2014-08-01,1406505600000,1407196800000 -1407542400000,2014-08-09,1407196800000,1407888000000 -1408233600000,2014-08-17,1407888000000,1408579200000 -1408924800000,2014-08-25,1408579200000,1409270400000 -1409616000000,2014-09-02,1409270400000,1409961600000 -1410307200000,2014-09-10,1409961600000,1410652800000 -1410998400000,2014-09-18,1410652800000,1411344000000 -1411689600000,2014-09-26,1411344000000,1412035200000 -1412380800000,2014-10-04,1412035200000,1412726400000 -1413072000000,2014-10-12,1412726400000,1413417600000 -1413763200000,2014-10-20,1413417600000,1414108800000 -1414454400000,2014-10-28,1414108800000,1414800000000 -1415145600000,2014-11-05,1414800000000,1415491200000 -1415836800000,2014-11-13,1415491200000,1416182400000 -1416528000000,2014-11-21,1416182400000,1416873600000 -1417219200000,2014-11-29,1416873600000,1417564800000 -1417910400000,2014-12-07,1417564800000,1418256000000 -1418601600000,2014-12-15,1418256000000,1418947200000 -1419292800000,2014-12-23,1418947200000,1419638400000 -1419984000000,2014-12-31,1419638400000,1420329600000 -1420416000000,2015-01-05,1420070400000,1420761600000 -1421107200000,2015-01-13,1420761600000,1421452800000 -1421798400000,2015-01-21,1421452800000,1422144000000 -1422489600000,2015-01-29,1422144000000,1422835200000 -1423180800000,2015-02-06,1422835200000,1423526400000 -1423872000000,2015-02-14,1423526400000,1424217600000 -1424563200000,2015-02-22,1424217600000,1424908800000 -1425254400000,2015-03-02,1424908800000,1425600000000 -1425945600000,2015-03-10,1425600000000,1426291200000 -1426636800000,2015-03-18,1426291200000,1426982400000 -1427328000000,2015-03-26,1426982400000,1427673600000 -1428019200000,2015-04-03,1427673600000,1428364800000 -1428710400000,2015-04-11,1428364800000,1429056000000 -1429401600000,2015-04-19,1429056000000,1429747200000 -1430092800000,2015-04-27,1429747200000,1430438400000 -1430784000000,2015-05-05,1430438400000,1431129600000 -1431475200000,2015-05-13,1431129600000,1431820800000 -1432166400000,2015-05-21,1431820800000,1432512000000 -1432857600000,2015-05-29,1432512000000,1433203200000 -1433548800000,2015-06-06,1433203200000,1433894400000 -1434240000000,2015-06-14,1433894400000,1434585600000 -1434931200000,2015-06-22,1434585600000,1435276800000 -1435622400000,2015-06-30,1435276800000,1435968000000 -1436313600000,2015-07-08,1435968000000,1436659200000 -1437004800000,2015-07-16,1436659200000,1437350400000 -1437696000000,2015-07-24,1437350400000,1438041600000 -1438387200000,2015-08-01,1438041600000,1438732800000 -1439078400000,2015-08-09,1438732800000,1439424000000 -1439769600000,2015-08-17,1439424000000,1440115200000 -1440460800000,2015-08-25,1440115200000,1440806400000 -1441152000000,2015-09-02,1440806400000,1441497600000 -1441843200000,2015-09-10,1441497600000,1442188800000 -1442534400000,2015-09-18,1442188800000,1442880000000 -1443225600000,2015-09-26,1442880000000,1443571200000 -1443916800000,2015-10-04,1443571200000,1444262400000 -1444608000000,2015-10-12,1444262400000,1444953600000 -1445299200000,2015-10-20,1444953600000,1445644800000 -1445990400000,2015-10-28,1445644800000,1446336000000 -1446681600000,2015-11-05,1446336000000,1447027200000 -1447372800000,2015-11-13,1447027200000,1447718400000 -1448064000000,2015-11-21,1447718400000,1448409600000 -1448755200000,2015-11-29,1448409600000,1449100800000 -1449446400000,2015-12-07,1449100800000,1449792000000 -1450137600000,2015-12-15,1449792000000,1450483200000 -1450828800000,2015-12-23,1450483200000,1451174400000 -1451520000000,2015-12-31,1451174400000,1451865600000 -1451952000000,2016-01-05,1451606400000,1452297600000 -1452643200000,2016-01-13,1452297600000,1452988800000 -1453334400000,2016-01-21,1452988800000,1453680000000 -1454025600000,2016-01-29,1453680000000,1454371200000 -1454716800000,2016-02-06,1454371200000,1455062400000 -1455408000000,2016-02-14,1455062400000,1455753600000 -1456099200000,2016-02-22,1455753600000,1456444800000 -1456790400000,2016-03-01,1456444800000,1457136000000 -1457481600000,2016-03-09,1457136000000,1457827200000 -1458172800000,2016-03-17,1457827200000,1458518400000 -1458864000000,2016-03-25,1458518400000,1459209600000 -1459555200000,2016-04-02,1459209600000,1459900800000 -1460246400000,2016-04-10,1459900800000,1460592000000 -1460937600000,2016-04-18,1460592000000,1461283200000 -1461628800000,2016-04-26,1461283200000,1461974400000 -1462320000000,2016-05-04,1461974400000,1462665600000 -1463011200000,2016-05-12,1462665600000,1463356800000 -1463702400000,2016-05-20,1463356800000,1464048000000 -1464393600000,2016-05-28,1464048000000,1464739200000 -1465084800000,2016-06-05,1464739200000,1465430400000 -1465776000000,2016-06-13,1465430400000,1466121600000 -1466467200000,2016-06-21,1466121600000,1466812800000 -1467158400000,2016-06-29,1466812800000,1467504000000 -1467849600000,2016-07-07,1467504000000,1468195200000 -1468540800000,2016-07-15,1468195200000,1468886400000 -1469232000000,2016-07-23,1468886400000,1469577600000 -1469923200000,2016-07-31,1469577600000,1470268800000 -1470614400000,2016-08-08,1470268800000,1470960000000 -1471305600000,2016-08-16,1470960000000,1471651200000 -1471996800000,2016-08-24,1471651200000,1472342400000 -1472688000000,2016-09-01,1472342400000,1473033600000 -1473379200000,2016-09-09,1473033600000,1473724800000 -1474070400000,2016-09-17,1473724800000,1474416000000 -1474761600000,2016-09-25,1474416000000,1475107200000 -1475452800000,2016-10-03,1475107200000,1475798400000 -1476144000000,2016-10-11,1475798400000,1476489600000 -1476835200000,2016-10-19,1476489600000,1477180800000 -1477526400000,2016-10-27,1477180800000,1477872000000 -1478217600000,2016-11-04,1477872000000,1478563200000 -1478908800000,2016-11-12,1478563200000,1479254400000 -1479600000000,2016-11-20,1479254400000,1479945600000 -1480291200000,2016-11-28,1479945600000,1480636800000 -1480982400000,2016-12-06,1480636800000,1481328000000 -1481673600000,2016-12-14,1481328000000,1482019200000 -1482364800000,2016-12-22,1482019200000,1482710400000 -1483056000000,2016-12-30,1482710400000,1483401600000 -1483574400000,2017-01-05,1483228800000,1483920000000 -1484265600000,2017-01-13,1483920000000,1484611200000 -1484956800000,2017-01-21,1484611200000,1485302400000 -1485648000000,2017-01-29,1485302400000,1485993600000 -1486339200000,2017-02-06,1485993600000,1486684800000 -1487030400000,2017-02-14,1486684800000,1487376000000 -1487721600000,2017-02-22,1487376000000,1488067200000 -1488412800000,2017-03-02,1488067200000,1488758400000 -1489104000000,2017-03-10,1488758400000,1489449600000 -1489795200000,2017-03-18,1489449600000,1490140800000 -1490486400000,2017-03-26,1490140800000,1490832000000 -1491177600000,2017-04-03,1490832000000,1491523200000 -1491868800000,2017-04-11,1491523200000,1492214400000 -1492560000000,2017-04-19,1492214400000,1492905600000 -1493251200000,2017-04-27,1492905600000,1493596800000 -1493942400000,2017-05-05,1493596800000,1494288000000 -1494633600000,2017-05-13,1494288000000,1494979200000 -1495324800000,2017-05-21,1494979200000,1495670400000 -1496016000000,2017-05-29,1495670400000,1496361600000 -1496707200000,2017-06-06,1496361600000,1497052800000 -1497398400000,2017-06-14,1497052800000,1497744000000 -1498089600000,2017-06-22,1497744000000,1498435200000 -1498780800000,2017-06-30,1498435200000,1499126400000 -1499472000000,2017-07-08,1499126400000,1499817600000 -1500163200000,2017-07-16,1499817600000,1500508800000 -1500854400000,2017-07-24,1500508800000,1501200000000 -1501545600000,2017-08-01,1501200000000,1501891200000 -1502236800000,2017-08-09,1501891200000,1502582400000 -1502928000000,2017-08-17,1502582400000,1503273600000 -1503619200000,2017-08-25,1503273600000,1503964800000 -1504310400000,2017-09-02,1503964800000,1504656000000 -1505001600000,2017-09-10,1504656000000,1505347200000 -1505692800000,2017-09-18,1505347200000,1506038400000 -1506384000000,2017-09-26,1506038400000,1506729600000 -1507075200000,2017-10-04,1506729600000,1507420800000 -1507766400000,2017-10-12,1507420800000,1508112000000 -1508457600000,2017-10-20,1508112000000,1508803200000 -1509148800000,2017-10-28,1508803200000,1509494400000 -1509840000000,2017-11-05,1509494400000,1510185600000 -1510531200000,2017-11-13,1510185600000,1510876800000 -1511222400000,2017-11-21,1510876800000,1511568000000 -1511913600000,2017-11-29,1511568000000,1512259200000 -1512604800000,2017-12-07,1512259200000,1512950400000 -1513296000000,2017-12-15,1512950400000,1513641600000 -1513987200000,2017-12-23,1513641600000,1514332800000 -1514678400000,2017-12-31,1514332800000,1515024000000 -1515110400000,2018-01-05,1514764800000,1515456000000 -1515801600000,2018-01-13,1515456000000,1516147200000 -1516492800000,2018-01-21,1516147200000,1516838400000 -1517184000000,2018-01-29,1516838400000,1517529600000 -1517875200000,2018-02-06,1517529600000,1518220800000 -1518566400000,2018-02-14,1518220800000,1518912000000 -1519257600000,2018-02-22,1518912000000,1519603200000 -1519948800000,2018-03-02,1519603200000,1520294400000 -1520640000000,2018-03-10,1520294400000,1520985600000 -1521331200000,2018-03-18,1520985600000,1521676800000 -1522022400000,2018-03-26,1521676800000,1522368000000 -1522713600000,2018-04-03,1522368000000,1523059200000 -1523404800000,2018-04-11,1523059200000,1523750400000 -1524096000000,2018-04-19,1523750400000,1524441600000 -1524787200000,2018-04-27,1524441600000,1525132800000 -1525478400000,2018-05-05,1525132800000,1525824000000 -1526169600000,2018-05-13,1525824000000,1526515200000 -1526860800000,2018-05-21,1526515200000,1527206400000 -1527552000000,2018-05-29,1527206400000,1527897600000 -1528243200000,2018-06-06,1527897600000,1528588800000 -1528934400000,2018-06-14,1528588800000,1529280000000 -1529625600000,2018-06-22,1529280000000,1529971200000 -1530316800000,2018-06-30,1529971200000,1530662400000 -1531008000000,2018-07-08,1530662400000,1531353600000 -1531699200000,2018-07-16,1531353600000,1532044800000 -1532390400000,2018-07-24,1532044800000,1532736000000 -1533081600000,2018-08-01,1532736000000,1533427200000 -1533772800000,2018-08-09,1533427200000,1534118400000 -1534464000000,2018-08-17,1534118400000,1534809600000 -1535155200000,2018-08-25,1534809600000,1535500800000 -1535846400000,2018-09-02,1535500800000,1536192000000 -1536537600000,2018-09-10,1536192000000,1536883200000 -1537228800000,2018-09-18,1536883200000,1537574400000 -1537920000000,2018-09-26,1537574400000,1538265600000 -1538611200000,2018-10-04,1538265600000,1538956800000 -1539302400000,2018-10-12,1538956800000,1539648000000 -1539993600000,2018-10-20,1539648000000,1540339200000 -1540684800000,2018-10-28,1540339200000,1541030400000 -1541376000000,2018-11-05,1541030400000,1541721600000 -1542067200000,2018-11-13,1541721600000,1542412800000 -1542758400000,2018-11-21,1542412800000,1543104000000 -1543449600000,2018-11-29,1543104000000,1543795200000 -1544140800000,2018-12-07,1543795200000,1544486400000 -1544832000000,2018-12-15,1544486400000,1545177600000 -1545523200000,2018-12-23,1545177600000,1545868800000 -1546214400000,2018-12-31,1545868800000,1546560000000 -1546646400000,2019-01-05,1546300800000,1546992000000 -1547337600000,2019-01-13,1546992000000,1547683200000 -1548028800000,2019-01-21,1547683200000,1548374400000 -1548720000000,2019-01-29,1548374400000,1549065600000 -1549411200000,2019-02-06,1549065600000,1549756800000 -1550102400000,2019-02-14,1549756800000,1550448000000 -1550793600000,2019-02-22,1550448000000,1551139200000 -1551484800000,2019-03-02,1551139200000,1551830400000 -1552176000000,2019-03-10,1551830400000,1552521600000 -1552867200000,2019-03-18,1552521600000,1553212800000 -1553558400000,2019-03-26,1553212800000,1553904000000 -1554249600000,2019-04-03,1553904000000,1554595200000 -1554940800000,2019-04-11,1554595200000,1555286400000 -1555632000000,2019-04-19,1555286400000,1555977600000 -1556323200000,2019-04-27,1555977600000,1556668800000 -1557014400000,2019-05-05,1556668800000,1557360000000 -1557705600000,2019-05-13,1557360000000,1558051200000 -1558396800000,2019-05-21,1558051200000,1558742400000 -1559088000000,2019-05-29,1558742400000,1559433600000 -1559779200000,2019-06-06,1559433600000,1560124800000 -1560470400000,2019-06-14,1560124800000,1560816000000 -1561161600000,2019-06-22,1560816000000,1561507200000 -1561852800000,2019-06-30,1561507200000,1562198400000 -1562544000000,2019-07-08,1562198400000,1562889600000 -1563235200000,2019-07-16,1562889600000,1563580800000 -1563926400000,2019-07-24,1563580800000,1564272000000 -1564617600000,2019-08-01,1564272000000,1564963200000 -1565308800000,2019-08-09,1564963200000,1565654400000 -1566000000000,2019-08-17,1565654400000,1566345600000 -1566691200000,2019-08-25,1566345600000,1567036800000 -1567382400000,2019-09-02,1567036800000,1567728000000 -1568073600000,2019-09-10,1567728000000,1568419200000 -1568764800000,2019-09-18,1568419200000,1569110400000 -1569456000000,2019-09-26,1569110400000,1569801600000 -1570147200000,2019-10-04,1569801600000,1570492800000 -1570838400000,2019-10-12,1570492800000,1571184000000 -1571529600000,2019-10-20,1571184000000,1571875200000 -1572220800000,2019-10-28,1571875200000,1572566400000 -1572912000000,2019-11-05,1572566400000,1573257600000 -1573603200000,2019-11-13,1573257600000,1573948800000 -1574294400000,2019-11-21,1573948800000,1574640000000 -1574985600000,2019-11-29,1574640000000,1575331200000 -1575676800000,2019-12-07,1575331200000,1576022400000 -1576368000000,2019-12-15,1576022400000,1576713600000 -1577059200000,2019-12-23,1576713600000,1577404800000 -1577750400000,2019-12-31,1577404800000,1578096000000 -1578182400000,2020-01-05,1577836800000,1578528000000 -1578873600000,2020-01-13,1578528000000,1579219200000 -1579564800000,2020-01-21,1579219200000,1579910400000 -1580256000000,2020-01-29,1579910400000,1580601600000 -1580947200000,2020-02-06,1580601600000,1581292800000 -1581638400000,2020-02-14,1581292800000,1581984000000 -1582329600000,2020-02-22,1581984000000,1582675200000 -1583020800000,2020-03-01,1582675200000,1583366400000 -1583712000000,2020-03-09,1583366400000,1584057600000 -1584403200000,2020-03-17,1584057600000,1584748800000 -1585094400000,2020-03-25,1584748800000,1585440000000 -1585785600000,2020-04-02,1585440000000,1586131200000 -1586476800000,2020-04-10,1586131200000,1586822400000 -1587168000000,2020-04-18,1586822400000,1587513600000 -1587859200000,2020-04-26,1587513600000,1588204800000 -1588550400000,2020-05-04,1588204800000,1588896000000 -1589241600000,2020-05-12,1588896000000,1589587200000 -1589932800000,2020-05-20,1589587200000,1590278400000 -1590624000000,2020-05-28,1590278400000,1590969600000 -1591315200000,2020-06-05,1590969600000,1591660800000 -1592006400000,2020-06-13,1591660800000,1592352000000 -1592697600000,2020-06-21,1592352000000,1593043200000 -1593388800000,2020-06-29,1593043200000,1593734400000 -1594080000000,2020-07-07,1593734400000,1594425600000 -1594771200000,2020-07-15,1594425600000,1595116800000 -1595462400000,2020-07-23,1595116800000,1595808000000 -1596153600000,2020-07-31,1595808000000,1596499200000 -1596844800000,2020-08-08,1596499200000,1597190400000 -1597536000000,2020-08-16,1597190400000,1597881600000 -1598227200000,2020-08-24,1597881600000,1598572800000 -1598918400000,2020-09-01,1598572800000,1599264000000 -1599609600000,2020-09-09,1599264000000,1599955200000 -1600300800000,2020-09-17,1599955200000,1600646400000 -1600992000000,2020-09-25,1600646400000,1601337600000 -1601683200000,2020-10-03,1601337600000,1602028800000 -1602374400000,2020-10-11,1602028800000,1602720000000 -1603065600000,2020-10-19,1602720000000,1603411200000 -1603756800000,2020-10-27,1603411200000,1604102400000 -1604448000000,2020-11-04,1604102400000,1604793600000 -1605139200000,2020-11-12,1604793600000,1605484800000 -1605830400000,2020-11-20,1605484800000,1606176000000 -1606521600000,2020-11-28,1606176000000,1606867200000 -1607212800000,2020-12-06,1606867200000,1607558400000 -1607904000000,2020-12-14,1607558400000,1608249600000 -1608595200000,2020-12-22,1608249600000,1608940800000 -1609286400000,2020-12-30,1608940800000,1609632000000 -1609804800000,2021-01-05,1609459200000,1610150400000 -1610496000000,2021-01-13,1610150400000,1610841600000 -1611187200000,2021-01-21,1610841600000,1611532800000 -1611878400000,2021-01-29,1611532800000,1612224000000 -1612569600000,2021-02-06,1612224000000,1612915200000 -1613260800000,2021-02-14,1612915200000,1613606400000 -1613952000000,2021-02-22,1613606400000,1614297600000 -1614643200000,2021-03-02,1614297600000,1614988800000 -1615334400000,2021-03-10,1614988800000,1615680000000 -1616025600000,2021-03-18,1615680000000,1616371200000 -1616716800000,2021-03-26,1616371200000,1617062400000 -1617408000000,2021-04-03,1617062400000,1617753600000 -1618099200000,2021-04-11,1617753600000,1618444800000 -1618790400000,2021-04-19,1618444800000,1619136000000 -1619481600000,2021-04-27,1619136000000,1619827200000 -1620172800000,2021-05-05,1619827200000,1620518400000 -1620864000000,2021-05-13,1620518400000,1621209600000 -1621555200000,2021-05-21,1621209600000,1621900800000 -1622246400000,2021-05-29,1621900800000,1622592000000 -1622937600000,2021-06-06,1622592000000,1623283200000 -1623628800000,2021-06-14,1623283200000,1623974400000 -1624320000000,2021-06-22,1623974400000,1624665600000 -1625011200000,2021-06-30,1624665600000,1625356800000 -1625702400000,2021-07-08,1625356800000,1626048000000 -1626393600000,2021-07-16,1626048000000,1626739200000 -1627084800000,2021-07-24,1626739200000,1627430400000 -1627776000000,2021-08-01,1627430400000,1628121600000 -1628467200000,2021-08-09,1628121600000,1628812800000 -1629158400000,2021-08-17,1628812800000,1629504000000 -1629849600000,2021-08-25,1629504000000,1630195200000 -1630540800000,2021-09-02,1630195200000,1630886400000 -1631232000000,2021-09-10,1630886400000,1631577600000 -1631923200000,2021-09-18,1631577600000,1632268800000 -1632614400000,2021-09-26,1632268800000,1632960000000 -1633305600000,2021-10-04,1632960000000,1633651200000 -1633996800000,2021-10-12,1633651200000,1634342400000 -1634688000000,2021-10-20,1634342400000,1635033600000 -1635379200000,2021-10-28,1635033600000,1635724800000 -1636070400000,2021-11-05,1635724800000,1636416000000 -1636761600000,2021-11-13,1636416000000,1637107200000 -1637452800000,2021-11-21,1637107200000,1637798400000 -1638144000000,2021-11-29,1637798400000,1638489600000 -1638835200000,2021-12-07,1638489600000,1639180800000 -1639526400000,2021-12-15,1639180800000,1639872000000 -1640217600000,2021-12-23,1639872000000,1640563200000 -1640908800000,2021-12-31,1640563200000,1641254400000 -1641340800000,2022-01-05,1640995200000,1641686400000 -1642032000000,2022-01-13,1641686400000,1642377600000 -1642723200000,2022-01-21,1642377600000,1643068800000 -1643414400000,2022-01-29,1643068800000,1643760000000 -1644105600000,2022-02-06,1643760000000,1644451200000 -1644796800000,2022-02-14,1644451200000,1645142400000 -1645488000000,2022-02-22,1645142400000,1645833600000 -1646179200000,2022-03-02,1645833600000,1646524800000 -1646870400000,2022-03-10,1646524800000,1647216000000 -1647561600000,2022-03-18,1647216000000,1647907200000 -1648252800000,2022-03-26,1647907200000,1648598400000 -1648944000000,2022-04-03,1648598400000,1649289600000 -1649635200000,2022-04-11,1649289600000,1649980800000 -1650326400000,2022-04-19,1649980800000,1650672000000 -1651017600000,2022-04-27,1650672000000,1651363200000 -1651708800000,2022-05-05,1651363200000,1652054400000 -1652400000000,2022-05-13,1652054400000,1652745600000 -1653091200000,2022-05-21,1652745600000,1653436800000 -1653782400000,2022-05-29,1653436800000,1654128000000 -1654473600000,2022-06-06,1654128000000,1654819200000 -1655164800000,2022-06-14,1654819200000,1655510400000 -1655856000000,2022-06-22,1655510400000,1656201600000 -1656547200000,2022-06-30,1656201600000,1656892800000 -1657238400000,2022-07-08,1656892800000,1657584000000 -1657929600000,2022-07-16,1657584000000,1658275200000 -1658620800000,2022-07-24,1658275200000,1658966400000 -1659312000000,2022-08-01,1658966400000,1659657600000 -1660003200000,2022-08-09,1659657600000,1660348800000 -1660694400000,2022-08-17,1660348800000,1661040000000 -1661385600000,2022-08-25,1661040000000,1661731200000 -1662076800000,2022-09-02,1661731200000,1662422400000 -1662768000000,2022-09-10,1662422400000,1663113600000 -1663459200000,2022-09-18,1663113600000,1663804800000 -1664150400000,2022-09-26,1663804800000,1664496000000 -1664841600000,2022-10-04,1664496000000,1665187200000 -1665532800000,2022-10-12,1665187200000,1665878400000 -1666224000000,2022-10-20,1665878400000,1666569600000 -1666915200000,2022-10-28,1666569600000,1667260800000 -1667606400000,2022-11-05,1667260800000,1667952000000 -1668297600000,2022-11-13,1667952000000,1668643200000 -1668988800000,2022-11-21,1668643200000,1669334400000 -1669680000000,2022-11-29,1669334400000,1670025600000 -1670371200000,2022-12-07,1670025600000,1670716800000 -1671062400000,2022-12-15,1670716800000,1671408000000 -1671753600000,2022-12-23,1671408000000,1672099200000 -1672444800000,2022-12-31,1672099200000,1672790400000 diff --git a/Source/2_Inference/STAC.ipynb b/Source/2_Inference/STAC.ipynb deleted file mode 100644 index 97c6c66..0000000 --- a/Source/2_Inference/STAC.ipynb +++ /dev/null @@ -1,523 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "id": "bd032238-7e2c-4e7d-860e-5ed8c818d346", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import datetime" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "43785806-5c93-4851-b652-f24447f572c5", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import rasterio\n", - "from shapely.geometry import Polygon, mapping" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "8315815a-12d1-4aa6-b2f7-41d798ea3f8d", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import pystac\n", - "from pystac.extensions.eo import Band, EOExtension\n", - "\n", - "from pystac.extensions.scientific import ScientificExtension\n", - "from pystac.extensions.scientific import Publication\n", - "\n", - "from pystac.provider import Provider\n", - "from pystac.provider import ProviderRole\n", - "\n", - "import pystac.provider \n", - "\n", - "from pystac import Provider" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "6b41424d-d67f-495a-ad12-4148ed6633fa", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "from py_linq import Enumerable\n", - "import json" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "3b6ab76d-8e99-403a-8365-d781ed505403", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "import numpy as np\n", - "import pandas as pd" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "7e13491d-058b-4fa8-8bbd-ef43ea1baf5a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "Path.lsTif = lambda x: Enumerable(x.iterdir()).where(lambda p: p.suffix == '.tiff').to_list()\n", - "Path.lsTifName = lambda x: Enumerable(x.iterdir()).where(lambda p: p.suffix == '.tif').select(lambda p: p.name).to_list()\n", - "Path.lsTifStem = lambda x: Enumerable(x.iterdir()).where(lambda p: p.suffix == '.tif').select(lambda p: p.stem).to_list()" - ] - }, - { - "cell_type": "markdown", - "id": "2cf4730a-4fa6-4eec-9359-bab3b6ee3f4a", - "metadata": {}, - "source": [ - "### Define Root folder and Data folder" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "id": "a43b5836-d7ab-44d7-b765-6804e6168c89", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "rootPath = Path('//mule.sbs.arizona.edu/')/'btellman'/'Projects'/'NASA'/'NIP'/'Data'/'Raster'/'CVPR23'/'CVPR23FractionalInundationHistoryData'" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "id": "96aaf18d-772b-4687-9931-81fc86559c67", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "fractionInundatedFolder = 'CVPR23FractionalInundationHistory'" - ] - }, - { - "cell_type": "code", - "execution_count": 107, - "id": "38b2a5d8-49ab-4231-8faf-6eec3a61abc9", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "dataPath = rootPath/fractionInundatedFolder\n", - "stacPath = Path('RelativeSTAC')\n", - "stacPath.mkdir(exist_ok=True)" - ] - }, - { - "cell_type": "markdown", - "id": "d0226f1d-438d-4530-869f-1d3afbe6e807", - "metadata": {}, - "source": [ - "### Create STAC" - ] - }, - { - "cell_type": "markdown", - "id": "698bef26-7896-4298-95b4-944aacf8db34", - "metadata": { - "tags": [] - }, - "source": [ - "Define function to extract bounding box and footprint from open tif and do it for one file" - ] - }, - { - "cell_type": "code", - "execution_count": 108, - "id": "bcfb0b9c-398f-4aea-9835-48f89e1210e5", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "def get_bbox_and_footprint(ds):\n", - " bounds = ds.bounds\n", - " bbox = [bounds.left, bounds.bottom, bounds.right, bounds.top]\n", - " footprint = Polygon([\n", - " [bounds.left, bounds.bottom],\n", - " [bounds.left, bounds.top],\n", - " [bounds.right, bounds.top],\n", - " [bounds.right, bounds.bottom]\n", - " ])\n", - "\n", - " return (bbox, mapping(footprint))" - ] - }, - { - "cell_type": "code", - "execution_count": 109, - "id": "efaa86bc-ef2c-4968-897a-23e01fa9890b", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "filePath = dataPath.lsTif()[0]\n", - "with rasterio.open(filePath) as ds:\n", - " bbox_footprints = get_bbox_and_footprint(ds)" - ] - }, - { - "cell_type": "markdown", - "id": "825c8dd6-ccf2-4be2-8075-6a12c2507b70", - "metadata": {}, - "source": [ - "Load the modis info file" - ] - }, - { - "cell_type": "code", - "execution_count": 110, - "id": "74b60015-94b6-49e7-924e-53f3761715b2", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "df_modisInfoTimes = pd.read_csv('MODISInfo.csv')" - ] - }, - { - "cell_type": "markdown", - "id": "cdad5ab2-5041-4837-b680-3fc218ce0fea", - "metadata": {}, - "source": [ - "Define band name" - ] - }, - { - "cell_type": "code", - "execution_count": 111, - "id": "e09a1c55-ce21-49a5-9090-fd7ce8aaa8a9", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "sen1_bands = [\n", - " Band.create(name='b1', description='Fractional Inundated Area')\n", - "]" - ] - }, - { - "cell_type": "markdown", - "id": "f61bf8b3-3ad0-4eb7-a950-c569a36e8fbc", - "metadata": { - "tags": [] - }, - "source": [ - "define stac info" - ] - }, - { - "cell_type": "code", - "execution_count": 112, - "id": "eb4c66b2-babc-46b5-872e-31522f408bba", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "platform = 'Fusion Model Output'\n", - "license = 'CC-BY-SA-4.0'\n", - "items_mission_description = 'This imagery contains the fractional inundated area as inferred from the Fusion model Giezendanner et al. 2023, for the country of Bangladesh, from 2001 to 2022'\n", - "pub_doi = '10.48550/arXiv.2305.00640'\n", - "\n", - "citation = 'Giezendanner, J.; Mukherjee, R.; Purri, M.; Thomas, M.; Mauerman, M.; Islam, A. K. M. S.; Tellman, B. Inferring the Past: A Combined CNN-LSTM Deep Learning Framework to Fuse Satellites for Historical Inundation Mapping. arXiv April 30, 2023. http://arxiv.org/abs/2305.00640'\n", - "\n", - "# collection definitions\n", - "collection_id=\"FractionalInundationHistory_ts\"\n", - "collection_title='Fractional Inundation History Time Series'\n", - "collection_description='Bangladesh Historical Fractional Inundated Area Dataset'\n", - "\n", - "# top-level catalog definitions\n", - "catalog_id = 'Bangladesh Historical Fractional Inundated Area'\n", - "catalog_description = 'STAC catalog for Bangladesh Historical Fractional Inundated Area Dataset'\n", - "\n", - "#Provider information\n", - "provider = Provider(name='Jonathan Giezendanner',\n", - " description=\"Postdoctoral Researcher University of Arizona, Social [Pixel] Lab\",\n", - " roles=[ProviderRole.PRODUCER, ProviderRole.PROCESSOR],\n", - " url='https://www.jgiezendanner.com/')\n", - "\n", - "provider_dict = provider.to_dict()\n", - "\n", - "contact_email = 'jgiezendanner@arizona.edu'\n", - " \n", - "targetUrl = Path('https://data.cyverse.org/dav-anon/iplant/commons/cyverse_curated/Giezendanner_BangladeshInundationHistory_Mai2023/CVPR23FractionalInundationHistory/')\n", - "# catalogPath = Path('/dav-anon/iplant/commons/curated/Giezendanner_BangladeshInundationHistory_Mai2023/CVPR23FractionalInundationHistory/CVPR23FractionalInundationHistorySTAC/')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d983e29-755c-4f1c-8f7f-6c1f63469124", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [] - }, - { - "cell_type": "markdown", - "id": "32a62333-71f3-47e5-888b-f8f062f7644e", - "metadata": {}, - "source": [ - "Create function to convert time stamp to date time format" - ] - }, - { - "cell_type": "code", - "execution_count": 113, - "id": "94815834-03f0-4b77-8869-b271184188c1", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "def getDateFromTimeStamp(timeStamp):\n", - " return datetime.datetime.fromtimestamp(timeStamp / 1000.0, tz=datetime.timezone.utc)" - ] - }, - { - "cell_type": "markdown", - "id": "eea83e4d-6e97-42c8-b62e-d777ea537fbc", - "metadata": {}, - "source": [ - "Get item from raster URI" - ] - }, - { - "cell_type": "code", - "execution_count": 114, - "id": "c3f9ba83-cbb2-4c10-9bdc-7947c861743a", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "def getWaterItem(raster_uri):\n", - " # use stem of file as id\n", - " idx = raster_uri.stem\n", - " \n", - " # get modis info from pandas dataframe\n", - " modisInfo = df_modisInfoTimes[df_modisInfoTimes.Time == int(idx)]\n", - " \n", - " # get bbox and footprint\n", - " bbox, footprint = bbox_footprints\n", - " \n", - " # get dates from modisInfo and convert to datetime object\n", - " date = getDateFromTimeStamp(int(idx))\n", - " startDate = getDateFromTimeStamp(int(modisInfo.StartDate.values[0]))\n", - " endDate = getDateFromTimeStamp(int(modisInfo.EndDate.values[0]))\n", - " \n", - " #create item \n", - " item = pystac.Item(\n", - " id=idx,\n", - " geometry=footprint,\n", - " bbox=bbox,\n", - " stac_extensions=['https://stac-extensions.github.io/projection/v1.0.0/schema.json', 'https://stac-extensions.github.io/scientific/v1.0.0/schema.json'],\n", - " datetime=date,\n", - " start_datetime=startDate,\n", - " end_datetime=endDate,\n", - " properties=dict(\n", - " tile ='FractionalInundatedArea_' + idx,\n", - " gsd = 500,\n", - " platform= platform,\n", - " license= license,\n", - " mission= items_mission_description,\n", - " providers= [provider_dict]\n", - " )\n", - " )\n", - " \n", - " sci_ext = ScientificExtension.ext(item)\n", - " sci_ext.apply(doi=pub_doi, citation=citation)\n", - " \n", - " # apply eo extension\n", - " eo = EOExtension.ext(item, add_if_missing=True)\n", - " eo.apply(bands=sen1_bands)\n", - " \n", - " # add asset to item with raster path\n", - " item.add_asset(\n", - " key='FractionalInundatedArea',\n", - " asset=pystac.Asset(\n", - " title='Fractional Inundated Area',\n", - " href= (targetUrl/raster_uri.name).as_posix(),\n", - " media_type=pystac.MediaType.GEOTIFF\n", - " )\n", - " )\n", - " return item" - ] - }, - { - "cell_type": "markdown", - "id": "fb7f9f80-44da-4c94-a50f-c94ce551c027", - "metadata": {}, - "source": [ - "Create catalog" - ] - }, - { - "cell_type": "code", - "execution_count": 115, - "id": "94b6559d-b8bd-4be5-9ce3-6e52771d9a9d", - "metadata": { - "tags": [] - }, - "outputs": [], - "source": [ - "# create catalog\n", - "catalog = pystac.Catalog(id=catalog_id, description=catalog_description)\n", - "\n", - "# create modis collection\n", - "modisTimeSeriesCollection = pystac.Collection(id=collection_id, title=collection_title, description=collection_description, extent=bbox_footprints)\n", - "\n", - "# set the collection's catalog\n", - "modisTimeSeriesCollection.catalog = catalog\n", - "\n", - "# crawl data folder to get all files\n", - "modisFiles = dataPath.lsTif()\n", - "\n", - "# loop over all files\n", - "for file in modisFiles:\n", - " # get item for file\n", - " item = getWaterItem(file)\n", - " # set the item's collection\n", - " item.collection = modisTimeSeriesCollection\n", - " # add item to collection\n", - " modisTimeSeriesCollection.add_item(item)\n", - "\n", - "# update the collection's extent from the items\n", - "modisTimeSeriesCollection.update_extent_from_items()\n", - "\n", - "# add collection to catalogue\n", - "catalog.add_child(modisTimeSeriesCollection)\n", - "\n", - "\n", - "# normalise all paths relative to stac folder\n", - "# catalog.normalize_hrefs(catalogPath.as_posix())\n", - "# catalog.make_all_asset_hrefs_absolute()\n", - "# catalog.save(catalog_type=pystac.CatalogType.ABSOLUTE_PUBLISHED, dest_href=stacPath.as_posix())\n", - "\n", - "catalog.normalize_hrefs(stacPath.as_posix())\n", - "catalog.make_all_asset_hrefs_relative()\n", - "catalog.save(catalog_type=pystac.CatalogType.SELF_CONTAINED)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "556892c9-8fa2-4b9a-975c-49cd72352146", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 104, - "id": "b2aba5bc-6d7f-4b20-8243-09f1468337c0", - "metadata": {}, - "outputs": [], - "source": [ - "#########Combine all of the STAC item metadata into a single geojson and output to directory\n", - "####################\n", - "\n", - "# Create an empty GeoJSON FeatureCollection\n", - "geojson = {\n", - " \"type\": \"FeatureCollection\",\n", - " \"features\": []\n", - "}\n", - "\n", - "# Iterate through all the items in the collection and convert them to GeoJSON Features\n", - "for item in modisTimeSeriesCollection.get_all_items():\n", - " # Get the STAC Item as a dictionary\n", - " item_dict = item.to_dict()\n", - "\n", - " # Convert the STAC Item to a GeoJSON Feature\n", - " feature = {\n", - " \"type\": \"Feature\",\n", - " \"collection\": item_dict[\"collection\"],\n", - " \"stac_version\": item_dict[\"stac_version\"],\n", - " \"stac_extensions\": item_dict[\"stac_extensions\"],\n", - " \"id\": item_dict[\"id\"],\n", - " \"geometry\": item_dict[\"geometry\"],\n", - " \"bbox\": item_dict[\"bbox\"],\n", - " \"properties\": item_dict[\"properties\"],\n", - " \"assets\": item_dict[\"assets\"]\n", - " }\n", - "\n", - " # Add the GeoJSON Feature to the FeatureCollection\n", - " geojson[\"features\"].append(feature)\n", - "\n", - "\n", - "\n", - "output_file_path = stacPath/\"index.geojson\"\n", - "\n", - "# Write the GeoJSON FeatureCollection to a file\n", - "with open(output_file_path, \"w\") as f:\n", - " json.dump(geojson, f, indent=4) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "14eb0b5f-ef68-4b2b-8ee2-8ebb00abefce", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/Source/Helpers/GEEHelpers.py b/Source/Helpers/GEEHelpers.py new file mode 100644 index 0000000..7de26a4 --- /dev/null +++ b/Source/Helpers/GEEHelpers.py @@ -0,0 +1,7 @@ +import datetime + +def GetDateFromGEETimeStamp(timeStamp): + return datetime.datetime.fromtimestamp(timeStamp / 1000.0, tz=datetime.timezone.utc) + +def GetGEETimeStampFromDate(year,month,day): + return int(datetime.datetime.timestamp(datetime.datetime(year,month,day)) * 1000) \ No newline at end of file diff --git a/Source/Helpers/MODIS8DaysHelper.py b/Source/Helpers/MODIS8DaysHelper.py new file mode 100644 index 0000000..9127c3b --- /dev/null +++ b/Source/Helpers/MODIS8DaysHelper.py @@ -0,0 +1,63 @@ +import numpy as np +import rasterio + +bandNames = ["sur_refl_b01", "sur_refl_b02", "sur_refl_b03", "sur_refl_b04", "sur_refl_b05", "sur_refl_b06", "sur_refl_b07", "QA", "SolarZenith", "ViewZenith", "RelativeAzimuth", "StateQA", "DayOfYear"] +ranges = [[-100, 16000],[-100, 16000],[-100, 16000],[-100, 16000],[-100, 16000],[-100, 16000],[-100, 16000],[],[],[],[],[],[]] + +def getBandNumbers(names): + return [bandNames.index(entry) for entry in names] + +def getBandNumber(name): + return bandNames.index(name) + +def convertToRasterioNumbers(numbers): + return [number + 1 for number in numbers] + +def getBandRange(name): + return ranges[getBandNumber(name)] +def getBandRanges(names): + return [ranges[i] for i in getBandNumbers(names)] + +def getByteImage(image, dataRange): + return np.interp(image, dataRange, (0, 255)).astype(np.uint8) + +def getScaledImage(image, dataRange): + return np.interp(image, dataRange, (0.0, 1.0)) + +def getModisFileBandsAsUInt8Matrix(modisFile, bandNames): + banNumbers = getBandNumbers(bandNames) + with rasterio.open(modisFile) as modisImage: + bandValues = modisImage.read(convertToRasterioNumbers(banNumbers)) + bandRanges = getBandRanges(bandNames) + bOut = np.zeros(bandValues.shape).astype(np.uint8) + for i in range(len(bandRanges)): + bOut[i,:] = getByteImage(bandValues[i,:], bandRanges[i]) + return bOut + +def getScaledModisFileBands(modisFile, bandNames): + banNumbers = getBandNumbers(bandNames) + with rasterio.open(modisFile) as modisImage: + bandValues = modisImage.read(convertToRasterioNumbers(banNumbers)) + bandRanges = getBandRanges(bandNames) + bOut = np.zeros(bandValues.shape) + for i in range(len(bandRanges)): + bOut[i,:] = getScaledImage(bandValues[i,:], bandRanges[i]) + return bOut + +def getScaledModisFileBandsWithOpenRaster(modisRasterImage, bandNames): + banNumbers = getBandNumbers(bandNames) + bandValues = modisRasterImage.read(convertToRasterioNumbers(banNumbers)) + bandRanges = getBandRanges(bandNames) + bOut = np.zeros(bandValues.shape) + for i in range(len(bandRanges)): + bOut[i,:] = getScaledImage(bandValues[i,:], bandRanges[i]) + return bOut + +def getModisFileBands(modisFile, bandNames): + banNumbers = getBandNumbers(bandNames) + with rasterio.open(modisFile) as modisImage: + return modisImage.read(convertToRasterioNumbers(banNumbers)) + +def getModisFileBandsWithOpenRaster(modisRasterImage, bandNames): + banNumbers = getBandNumbers(bandNames) + return modisRasterImage.read(convertToRasterioNumbers(banNumbers)) diff --git a/Source/Helpers/StaticFeaturesHelper.py b/Source/Helpers/StaticFeaturesHelper.py new file mode 100644 index 0000000..84f72f7 --- /dev/null +++ b/Source/Helpers/StaticFeaturesHelper.py @@ -0,0 +1,20 @@ +import numpy as np +import rasterio + +def getScaledImage(image, dataRange): + return np.interp(image, dataRange, (0.0, 1.0)) + +def getScaledHAND(file): + with rasterio.open(file) as image: + bOut = getScaledImage(image.read(1), [0.0, 7.0]) + return bOut + +def getScaledElevation(file): + with rasterio.open(file) as image: + bOut = getScaledImage(image.read(1), [0.0, 100.0]) + return bOut + +def getSlope(file): + with rasterio.open(file) as image: + bOut = image.read(1) + return bOut diff --git a/Source/ModelClasses/Model.py b/Source/ModelClasses/Model.py new file mode 100644 index 0000000..d247607 --- /dev/null +++ b/Source/ModelClasses/Model.py @@ -0,0 +1,53 @@ +from fastai.vision.all import * + +class CNNLSTM(nn.Module): + def __init__(self, nbFeatures=10, initSize=32, nbLayers=1, nbTimeSteps=10, input_size=32*32, hidden_size=32*32): + super().__init__() + + self.nbFeatures = nbFeatures + self.input_size = input_size + self.nbTimeSteps = nbTimeSteps + + # define helper functions for convolutions, either single convolution, or combined with res block + def conv2_single(ni,nf): return nn.Conv2d(ni, nf, kernel_size=3, padding=1, padding_mode='reflect', stride=1) + def conv2(ni,nf): return nn.Conv2d(ni, nf, groups=nbTimeSteps, kernel_size=3, padding=1, padding_mode='reflect', stride=1) + def conv2_and_res(ni, nf): return nn.Sequential(conv2(ni,nf), ResBlock(2, ni, nf, groups=nbTimeSteps, stride=1)) + + # Create CNN A + # note that groups are defined by number of time steps, i.e. each time step is applied the same CNN separatly + self.cnn = nn.Sequential( + conv2(nbFeatures*nbTimeSteps,nbTimeSteps*initSize) + ) + + for k in range(nbLayers): + self.cnn = self.cnn.append(conv2_and_res(nbTimeSteps*initSize * 4**k, nbTimeSteps*(initSize * 2) * 4**k)) + self.cnn = self.cnn.append(conv2(nbTimeSteps*(initSize * 2) * 4**(nbLayers-1)* 2,nbTimeSteps*1)) + + # Create LSTM + self.LSTM = nn.LSTM(input_size=input_size, hidden_size=hidden_size, num_layers=1, batch_first=True, bidirectional=False) + + # Create transpose convolution + self.convTrans = nn.ConvTranspose2d(1024,1,kernel_size=32) + + # Set the output to be a single convolution and a sigmoid + self.outLayer = nn.Sequential(conv2_single(2,1), SigmoidRange(0,1)) + + def forward(self, x): + # get size of problem + batchSize = x.shape[0] + imgSize = x.shape[2:4] + + # pass all time steps through the CNN + x = self.cnn(x) + # extract time step 0 + x_now = x[:,0,::].view((batchSize,1,imgSize[0],imgSize[1])) + # pass time step -1 to -9 through lstm + x, (_,_) = self.LSTM(x[:,1:,::].view((batchSize,self.nbTimeSteps-1,-1))) + # extract result and pass through transpose convolution + x = x[:,-1,:].view((batchSize,-1,1,1)) + x = self.convTrans(x) + # concatenate lstm output and time step t + x = torch.cat((x_now,x),1) + # pass output through output layer + x = self.outLayer(x) + return x \ No newline at end of file