forked from ariyanzri/NowCasting-nets
-
Notifications
You must be signed in to change notification settings - Fork 0
/
get_data.py
103 lines (75 loc) · 3.92 KB
/
get_data.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
import numpy as np
import h5py
from sklearn.model_selection import train_test_split
def train_test(file):
with h5py.File(file, 'r') as h5:
X_train = h5['X_train'][:,:,256:,:,:]
X_valid = h5['X_test'][:,:,256:,:,:]
y_train = h5['y_train'][:,:,256:,:,:]
y_valid = h5['y_test'][:,:,256:,:,:]
return X_train, X_valid, y_train, y_valid
def train_test_feedback(file):
with h5py.File(file, 'r') as h5:
X_train = h5['X_train'][:,:,256:,:,:]
X_valid = h5['X_test'][:,:,256:,:,:]
y_train = h5['y_train'][:,:,256:,:,:]
y_valid = h5['y_test'][:,:,256:,:,:]
z_train = h5['z_train'][:,:,256:,:,:]
z_valid = h5['z_test'][:,:,256:,:,:]
return X_train, X_valid, y_train, y_valid, z_train, z_valid
def train_test_residuals(file, dic):
with h5py.File(file, 'r') as h5:
data = h5['Precipitation'][:]
data = np.expand_dims(data, axis=-1)
print(data.shape)
indx = np.arange(0, data.shape[0], dic['input_length'] + dic['output_length'])
X = [data[indx[i]:indx[i+1]-dic['output_length'], :, :, 0] for i in range(len(indx)-1)]
X = np.expand_dims(X, axis=-1)
Y = [data[indx[i]+dic['input_length']:indx[i+1], :, :, 0] - data[indx[i+1]-dic['output_length']-1, :, :, 0] for i in range(len(indx)-1)]
Y = np.expand_dims(Y, axis=-1)
Y = np.expand_dims(Y, axis=-1)
X_train, X_valid, y_train, y_valid = train_test_split(X, Y, test_size=0.2, random_state=42)
X_train = X_train[:, :, 144:400, :256, :]
y_train = y_train[:, :, 144:400, :256, :]
X_valid = X_valid[:, :, 144:400, :256, :]
y_valid = y_valid[:, :, 144:400, :256, :]
percentage = np.array([(X_train[i, 0].squeeze()>1).mean()*100 for i in range(X_train.shape[0])])
percentage_valid = np.array([(X_valid[i, 0].squeeze()>1).mean()*100 for i in range(X_valid.shape[0])])
print(X_train.shape, y_valid.shape)
return X_train[percentage>1], X_valid[percentage_valid>1], y_train[percentage>1], y_valid[percentage_valid>1]
def train_test_experiment1(file, output_length):
with h5py.File(file, 'r') as h5:
data = h5['Precipitation'][:]
data = np.expand_dims(data, axis=-1)
print(data.shape)
indx = np.arange(0, data.shape[0], 9 + output_length)
X = [data[indx[i]:indx[i+1]-output_length, :, :, 0] for i in range(len(indx)-1)]
X = np.expand_dims(X, axis=-1)
Y = [data[indx[i]+9:indx[i+1], :, :, 0] - data[indx[i+1]-output_length-1, :, :, 0] for i in range(len(indx)-1)]
Y = np.expand_dims(Y, axis=-1)
Y = np.expand_dims(Y, axis=-1)
X_train, X_valid, y_train, y_valid = train_test_split(X, Y, test_size=0.2, random_state=42)
X_train = X_train[:, :, 144:400, :256, :]
y_train = y_train[:, :, 144:400, :256, :]
X_valid = X_valid[:, :, 144:400, :256, :]
y_valid = y_valid[:, :, 144:400, :256, :]
percentage = np.array([(X_train[i, 0].squeeze()>1).mean()*100 for i in range(X_train.shape[0])])
percentage_valid = np.array([(X_valid[i, 0].squeeze()>1).mean()*100 for i in range(X_valid.shape[0])])
print(X_train.shape, y_valid.shape)
return X_train[percentage>1], X_valid[percentage_valid>1],y_train[percentage>1], y_valid[percentage_valid>1]
def train_test_west(file):
with h5py.File(file, 'r') as h5:
X_train = h5['X_train'][:,:,:256,:,:]
y_train = h5['y_train'][:,:,:256,:,:]
X_valid = h5['X_test'][:,:,:256,:,:]
y_valid = h5['y_test'][:,:,:256,:,:]
return X_train, X_valid, y_train, y_valid
def train_test_west_feedback(file):
with h5py.File(file, 'r') as h5:
X_train = h5['X_train'][:,:,:256,:,:]
X_valid = h5['X_test'][:,:,:256,:,:]
y_train = h5['y_train'][:,:,:256,:,:]
y_valid = h5['y_test'][:,:,:256,:,:]
z_train = h5['z_train'][:,:,:256,:,:]
z_valid = h5['z_test'][:,:,:256,:,:]
return X_train, X_valid, y_train, y_valid, z_train, z_valid