From ec30e29fd8846142251110d20bd7023bff21990b Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Fri, 8 Nov 2024 18:53:05 +0100 Subject: [PATCH] Add KDVH migration package --- db/flags.sql | 12 + integration_tests/src/main.rs | 1 + migrations/.gitignore | 7 + migrations/README.md | 27 + migrations/go.mod | 27 + migrations/go.sum | 62 + migrations/kdvh/cache.go | 243 ++ migrations/kdvh/dump.go | 244 ++ migrations/kdvh/dump_functions.go | 276 ++ migrations/kdvh/import.go | 325 ++ migrations/kdvh/import_functions.go | 465 +++ migrations/kdvh/import_test.go | 31 + migrations/kdvh/list_tables.go | 24 + migrations/kdvh/main.go | 53 + migrations/kdvh/product_offsets.csv | 161 + migrations/kdvh/table.go | 125 + migrations/kdvh_test.go | 71 + migrations/lard/import.go | 92 + migrations/lard/main.go | 80 + migrations/lard/timeseries.go | 62 + migrations/main.go | 43 + .../tests/T_MDATA_combined/12345/TA.csv | 2645 +++++++++++++++++ migrations/utils/email.go | 60 + migrations/utils/utils.go | 62 + 24 files changed, 5198 insertions(+) create mode 100644 migrations/.gitignore create mode 100644 migrations/README.md create mode 100644 migrations/go.mod create mode 100644 migrations/go.sum create mode 100644 migrations/kdvh/cache.go create mode 100644 migrations/kdvh/dump.go create mode 100644 migrations/kdvh/dump_functions.go create mode 100644 migrations/kdvh/import.go create mode 100644 migrations/kdvh/import_functions.go create mode 100644 migrations/kdvh/import_test.go create mode 100644 migrations/kdvh/list_tables.go create mode 100644 migrations/kdvh/main.go create mode 100644 migrations/kdvh/product_offsets.csv create mode 100644 migrations/kdvh/table.go create mode 100644 migrations/kdvh_test.go create mode 100644 migrations/lard/import.go create mode 100644 migrations/lard/main.go create mode 100644 migrations/lard/timeseries.go create mode 100644 migrations/main.go create mode 100644 migrations/tests/T_MDATA_combined/12345/TA.csv create mode 100644 migrations/utils/email.go create mode 100644 migrations/utils/utils.go diff --git a/db/flags.sql b/db/flags.sql index bdb8f50..785cf60 100644 --- a/db/flags.sql +++ b/db/flags.sql @@ -7,9 +7,21 @@ CREATE TABLE IF NOT EXISTS flags.kvdata ( corrected REAL NULL, controlinfo TEXT NULL, useinfo TEXT NULL, + -- TODO: check that this type is correct, it's stored as a string in Kvalobs? cfailed INT4 NULL, CONSTRAINT unique_kvdata_timeseries_obstime UNIQUE (timeseries, obstime) ); CREATE INDEX IF NOT EXISTS kvdata_obtime_index ON flags.kvdata (obstime); CREATE INDEX IF NOT EXISTS kvdata_timeseries_index ON flags.kvdata USING HASH (timeseries); + +CREATE TABLE IF NOT EXISTS flags.kdvh ( + timeseries INT4 REFERENCES public.timeseries, + obstime TIMESTAMPTZ NOT NULL, + controlinfo TEXT NULL, + useinfo TEXT NULL, + CONSTRAINT unique_kdvh_timeseries_obstime UNIQUE (timeseries, obstime) +); + +CREATE INDEX IF NOT EXISTS kdvh_obtime_index ON flags.kdvh (obstime); +CREATE INDEX IF NOT EXISTS kdvh_timeseries_index ON flags.kdvh USING HASH (timeseries); diff --git a/integration_tests/src/main.rs b/integration_tests/src/main.rs index 3f4cad8..a73a124 100644 --- a/integration_tests/src/main.rs +++ b/integration_tests/src/main.rs @@ -37,6 +37,7 @@ async fn main() { } }); + // NOTE: order matters let schemas = ["db/public.sql", "db/labels.sql", "db/flags.sql"]; for schema in schemas { insert_schema(&client, schema).await.unwrap(); diff --git a/migrations/.gitignore b/migrations/.gitignore new file mode 100644 index 0000000..6305aab --- /dev/null +++ b/migrations/.gitignore @@ -0,0 +1,7 @@ +*.txt +*.sh +migrate +tables*/ +test_*/ +.env +dumps/ diff --git a/migrations/README.md b/migrations/README.md new file mode 100644 index 0000000..b32cfa7 --- /dev/null +++ b/migrations/README.md @@ -0,0 +1,27 @@ +# Migrations + +Go package used to dump tables from old databases (KDVH, Kvalobs) and import them into LARD. + +## Usage + +1. Compile it with + + ```terminal + go build + ``` + +1. Dump tables from KDVH + + ```terminal + ./migrate kdvh dump --help + ``` + +1. Import dumps into LARD + + ```terminal + ./migrate kdvh import --help + ``` + +## Other notes + +Insightful talk on migrations: [here](https://www.youtube.com/watch?v=wqXqJfQMrqI&t=280s) diff --git a/migrations/go.mod b/migrations/go.mod new file mode 100644 index 0000000..d7233f7 --- /dev/null +++ b/migrations/go.mod @@ -0,0 +1,27 @@ +module migrate + +go 1.22.3 + +require ( + github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 + github.com/jackc/pgx/v5 v5.6.0 + github.com/jessevdk/go-flags v1.6.1 + github.com/joho/godotenv v1.5.1 + github.com/rickb777/period v1.0.5 + github.com/schollz/progressbar/v3 v3.16.1 +) + +require ( + github.com/govalues/decimal v0.1.29 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/rickb777/plural v1.4.2 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + golang.org/x/crypto v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/term v0.25.0 // indirect + golang.org/x/text v0.16.0 // indirect +) diff --git a/migrations/go.sum b/migrations/go.sum new file mode 100644 index 0000000..22ed9b7 --- /dev/null +++ b/migrations/go.sum @@ -0,0 +1,62 @@ +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1 h1:FWNFq4fM1wPfcK40yHE5UO3RUdSNPaBC+j3PokzA6OQ= +github.com/gocarina/gocsv v0.0.0-20240520201108-78e41c74b4b1/go.mod h1:5YoVOkjYAQumqlV356Hj3xeYh4BdZuLE0/nRkf2NKkI= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/govalues/decimal v0.1.29 h1:GKC5g9y9oWxKIy51czdHTShOABwHm/shVuOVPwG415M= +github.com/govalues/decimal v0.1.29/go.mod h1:LUlHHucpCmA4rJfNrDvMgrWibDpYnDNWqJuNU1/gxW8= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY= +github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= +github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= +github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rickb777/period v1.0.5 h1:jAzlI2knYam5VMy0X8eYgqJBl0ew57N+J1djJSBOulM= +github.com/rickb777/period v1.0.5/go.mod h1:AmEwpgIShi3EEw34qbafoPJxVeRbv9VVtjLyOeRwK6c= +github.com/rickb777/plural v1.4.2 h1:Kl/syFGLFZ5EbuV8c9SVud8s5HI2HpCCtOMw2U1kS+A= +github.com/rickb777/plural v1.4.2/go.mod h1:kdmXUpmKBJTS0FtG/TFumd//VBWsNTD7zOw7x4umxNw= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= +github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= +github.com/schollz/progressbar/v3 v3.16.1 h1:RnF1neWZFzLCoGx8yp1yF7SDl4AzNDI5y4I0aUJRrZQ= +github.com/schollz/progressbar/v3 v3.16.1/go.mod h1:I2ILR76gz5VXqYMIY/LdLecvMHDPVcQm3W/MSKi1TME= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= +golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.25.0 h1:WtHI/ltw4NvSUig5KARz9h521QvRC8RmF/cuYqifU24= +golang.org/x/term v0.25.0/go.mod h1:RPyXicDX+6vLxogjjRxjgD2TKtmAO6NZBsBRfrOLu7M= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/migrations/kdvh/cache.go b/migrations/kdvh/cache.go new file mode 100644 index 0000000..2c8d718 --- /dev/null +++ b/migrations/kdvh/cache.go @@ -0,0 +1,243 @@ +package kdvh + +import ( + "context" + "fmt" + "log/slog" + "os" + "slices" + "time" + + "github.com/gocarina/gocsv" + "github.com/jackc/pgx/v5" + "github.com/rickb777/period" +) + +// Caches all the metadata needed for import. +// If any error occurs inside here the program will exit. +func (config *ImportConfig) CacheMetadata() { + config.cacheStinfo() + config.cacheKDVH() + config.cacheParamOffsets() +} + +// StinfoKey is used for lookup of parameter offsets and metadata from Stinfosys +type StinfoKey struct { + ElemCode string + TableName string +} + +// Subset of StinfoQuery with only param info +type StinfoParam struct { + TypeID int32 + ParamID int32 + Hlevel *int32 + Sensor int32 + Fromtime time.Time + IsScalar bool +} + +// Struct holding query from Stinfosys elem_map_cfnames_param +type StinfoQuery struct { + ElemCode string `db:"elem_code"` + TableName string `db:"table_name"` + TypeID int32 `db:"typeid"` + ParamID int32 `db:"paramid"` + Hlevel *int32 `db:"hlevel"` + Sensor int32 `db:"sensor"` + Fromtime time.Time `db:"fromtime"` + IsScalar bool `db:"scalar"` +} + +func (q *StinfoQuery) toParam() StinfoParam { + return StinfoParam{ + TypeID: q.TypeID, + ParamID: q.ParamID, + Hlevel: q.Hlevel, + Sensor: q.Sensor, + Fromtime: q.Fromtime, + IsScalar: q.IsScalar, + } +} +func (q *StinfoQuery) toKey() StinfoKey { + return StinfoKey{q.ElemCode, q.TableName} +} + +// Save metadata for later use by quering Stinfosys +func (config *ImportConfig) cacheStinfo() { + cache := make(map[StinfoKey]StinfoParam) + + fmt.Println("Connecting to Stinfosys to cache metadata") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + conn, err := pgx.Connect(ctx, os.Getenv("STINFO_STRING")) + if err != nil { + slog.Error("Could not connect to Stinfosys. Make sure to be connected to the VPN. " + err.Error()) + os.Exit(1) + } + defer conn.Close(context.TODO()) + + for _, table := range KDVH { + if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { + continue + } + // select paramid, elem_code, scalar from elem_map_cfnames_param join param using(paramid) where scalar = false + query := `SELECT elem_code, table_name, typeid, paramid, hlevel, sensor, fromtime, scalar + FROM elem_map_cfnames_param + JOIN param USING(paramid) + WHERE table_name = $1 + AND ($2::text[] IS NULL OR elem_code = ANY($2))` + + rows, err := conn.Query(context.TODO(), query, table.TableName, config.Elements) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + metas, err := pgx.CollectRows(rows, pgx.RowToStructByName[StinfoQuery]) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for _, meta := range metas { + cache[meta.toKey()] = meta.toParam() + } + } + + config.StinfoMap = cache +} + +// Used for lookup of fromtime and totime from KDVH +type KDVHKey struct { + Inner StinfoKey + Station int32 +} + +func newKDVHKey(elem, table string, stnr int32) KDVHKey { + return KDVHKey{StinfoKey{ElemCode: elem, TableName: table}, stnr} +} + +// Timespan stored in KDVH for a given (table, station, element) triplet +type Timespan struct { + FromTime *time.Time `db:"fdato"` + ToTime *time.Time `db:"tdato"` +} + +// Struct used to deserialize KDVH query in cacheKDVH +type MetaKDVH struct { + ElemCode string `db:"elem_code"` + TableName string `db:"table_name"` + Station int32 `db:"stnr"` + FromTime *time.Time `db:"fdato"` + ToTime *time.Time `db:"tdato"` +} + +func (m *MetaKDVH) toTimespan() Timespan { + return Timespan{m.FromTime, m.ToTime} +} + +func (m *MetaKDVH) toKey() KDVHKey { + return KDVHKey{StinfoKey{ElemCode: m.ElemCode, TableName: m.TableName}, m.Station} +} + +func (config *ImportConfig) cacheKDVH() { + cache := make(map[KDVHKey]Timespan) + + fmt.Println("Connecting to KDVH proxy to cache metadata") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + conn, err := pgx.Connect(ctx, os.Getenv("KDVH_PROXY_CONN")) + if err != nil { + slog.Error("Could not connect to KDVH proxy. Make sure to be connected to the VPN: " + err.Error()) + os.Exit(1) + } + defer conn.Close(context.TODO()) + + for _, t := range KDVH { + if config.Tables != nil && !slices.Contains(config.Tables, t.TableName) { + continue + } + + // TODO: probably need to sanitize these inputs + query := fmt.Sprintf( + `SELECT table_name, stnr, elem_code, fdato, tdato FROM %s + WHERE ($1::bigint[] IS NULL OR stnr = ANY($1)) + AND ($2::text[] IS NULL OR elem_code = ANY($2))`, + t.ElemTableName, + ) + + rows, err := conn.Query(context.TODO(), query, config.Stations, config.Elements) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + metas, err := pgx.CollectRows(rows, pgx.RowToStructByName[MetaKDVH]) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for _, meta := range metas { + cache[meta.toKey()] = meta.toTimespan() + } + } + + config.KDVHMap = cache +} + +// Caches how to modify the obstime (in KDVH) for certain paramids +func (config *ImportConfig) cacheParamOffsets() { + cache := make(map[StinfoKey]period.Period) + + type CSVRow struct { + TableName string `csv:"table_name"` + ElemCode string `csv:"elem_code"` + ParamID int32 `csv:"paramid"` + FromtimeOffset string `csv:"fromtime_offset"` + Timespan string `csv:"timespan"` + } + + csvfile, err := os.Open("kdvh/product_offsets.csv") + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + defer csvfile.Close() + + var csvrows []CSVRow + if err := gocsv.UnmarshalFile(csvfile, &csvrows); err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + for _, row := range csvrows { + var fromtimeOffset, timespan period.Period + if row.FromtimeOffset != "" { + fromtimeOffset, err = period.Parse(row.FromtimeOffset) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + } + if row.Timespan != "" { + timespan, err = period.Parse(row.Timespan) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + } + migrationOffset, err := fromtimeOffset.Add(timespan) + if err != nil { + slog.Error(err.Error()) + os.Exit(1) + } + + cache[StinfoKey{ElemCode: row.ElemCode, TableName: row.TableName}] = migrationOffset + } + + config.OffsetMap = cache +} diff --git a/migrations/kdvh/dump.go b/migrations/kdvh/dump.go new file mode 100644 index 0000000..7affa25 --- /dev/null +++ b/migrations/kdvh/dump.go @@ -0,0 +1,244 @@ +package kdvh + +import ( + "database/sql" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strings" + + _ "github.com/jackc/pgx/v5/stdlib" + + "migrate/utils" +) + +type DumpConfig struct { + BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` + TablesCmd string `short:"t" long:"table" default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` + StationsCmd string `short:"s" long:"stnr" default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` + ElementsCmd string `short:"e" long:"elem" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` + Overwrite bool `long:"overwrite" description:"Overwrite any existing dumped files"` + Email []string `long:"email" description:"Optional email address used to notify if the program crashed"` + + Tables []string + Stations []string + Elements []string +} + +func (config *DumpConfig) setup() { + if config.TablesCmd != "" { + config.Tables = strings.Split(config.TablesCmd, ",") + } + if config.StationsCmd != "" { + config.Stations = strings.Split(config.StationsCmd, ",") + } + if config.ElementsCmd != "" { + config.Elements = strings.Split(config.ElementsCmd, ",") + } +} + +func (config *DumpConfig) Execute([]string) error { + config.setup() + + conn, err := sql.Open("pgx", os.Getenv("KDVH_PROXY_CONN")) + if err != nil { + slog.Error(err.Error()) + return nil + } + + for _, table := range KDVH { + if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { + continue + } + table.Dump(conn, config) + } + + return nil +} + +func (table *Table) Dump(conn *sql.DB, config *DumpConfig) { + defer utils.SendEmailOnPanic(fmt.Sprintf("%s dump", table.TableName), config.Email) + + table.Path = filepath.Join(config.BaseDir, table.Path) + if _, err := os.ReadDir(table.Path); err == nil && !config.Overwrite { + slog.Info(fmt.Sprint("Skipping data dump of ", table.TableName, " because dumped folder already exists")) + return + } + + utils.SetLogFile(table.TableName, "dump") + elements, err := table.getElements(conn, config) + if err != nil { + return + } + + bar := utils.NewBar(len(elements), table.TableName) + + // TODO: should be safe to spawn goroutines/waitgroup here with connection pool? + bar.RenderBlank() + for _, element := range elements { + table.dumpElement(element, conn, config) + bar.Add(1) + } +} + +// TODO: maybe we don't do this? Or can we use pgdump/copy? +// The problem is that there are no indices on the tables, that's why the queries are super slow +// Dumping the whole table might be a lot faster (for T_MDATA it's ~10 times faster!), +// but it might be more difficult to recover if something goes wrong? +// => +// copyQuery := fmt.SPrintf("\\copy (select * from t_mdata) TO '%s/%s.csv' WITH CSV HEADER", config.BaseDir, table.TableName) +// cmd := exec.Command("psql", CONN_STRING, "-c", copyQuery) +// cmd.Stderr = &bytes.Buffer{} +// err = cmd.Run() +func (table *Table) dumpElement(element string, conn *sql.DB, config *DumpConfig) { + stations, err := table.getStationsWithElement(element, conn, config) + if err != nil { + slog.Error(fmt.Sprintf("Could not fetch stations for table %s: %v", table.TableName, err)) + return + } + + for _, station := range stations { + path := filepath.Join(table.Path, string(station)) + if err := os.MkdirAll(path, os.ModePerm); err != nil { + slog.Error(err.Error()) + return + } + + err := table.dumpFunc( + path, + DumpMeta{ + element: element, + station: station, + dataTable: table.TableName, + flagTable: table.FlagTableName, + }, + conn, + ) + + // NOTE: Non-nil errors are logged inside each DumpFunc + if err == nil { + slog.Info(fmt.Sprintf("%s - %s - %s: dumped successfully", table.TableName, station, element)) + } + } +} + +// Fetches elements and filters them based on user input +func (table *Table) getElements(conn *sql.DB, config *DumpConfig) ([]string, error) { + elements, err := table.fetchElements(conn) + if err != nil { + return nil, err + } + + elements = utils.FilterSlice(config.Elements, elements, "") + return elements, nil +} + +// List of columns that we do not need to select when extracting the element codes from a KDVH table +var INVALID_COLUMNS = []string{"dato", "stnr", "typeid", "season", "xxx"} + +// Fetch column names for a given table +// We skip the columns defined in INVALID_COLUMNS and all columns that contain the 'kopi' string +// TODO: should we dump these invalid/kopi elements even if we are not importing them? +func (table *Table) fetchElements(conn *sql.DB) (elements []string, err error) { + slog.Info(fmt.Sprintf("Fetching elements for %s...", table.TableName)) + + // TODO: not sure why we only dump these two for this table + if table.TableName == "T_HOMOGEN_MONTH" { + return []string{"rr", "tam"}, nil + } + + rows, err := conn.Query( + `SELECT column_name FROM information_schema.columns + WHERE table_name = $1 + AND NOT column_name = ANY($2::text[]) + AND column_name NOT LIKE '%kopi%'`, + // NOTE: needs to be lowercase with PG + strings.ToLower(table.TableName), + INVALID_COLUMNS, + ) + if err != nil { + slog.Error(fmt.Sprintf("Could not fetch elements for table %s: %v", table.TableName, err)) + return nil, err + } + defer rows.Close() + + for rows.Next() { + var name string + if err = rows.Scan(&name); err != nil { + slog.Error(fmt.Sprintf("Could not fetch elements for table %s: %v", table.TableName, err)) + return nil, err + } + elements = append(elements, name) + } + return elements, rows.Err() +} + +// Fetches station numbers and filters them based on user input +func (table *Table) getStationsWithElement(element string, conn *sql.DB, config *DumpConfig) ([]string, error) { + stations, err := table.fetchStationsWithElement(element, conn) + if err != nil { + return nil, err + } + + msg := fmt.Sprintf("Element '%s'", element) + "not available for station '%s'" + stations = utils.FilterSlice(config.Stations, stations, msg) + return stations, nil +} + +// Fetches the unique station numbers in the table for a given element (and when that element is not null) +// NOTE: splitting by element does make it a bit better, because we avoid quering for stations that have no data or flag for that element? +func (table *Table) fetchStationsWithElement(element string, conn *sql.DB) (stations []string, err error) { + slog.Info(fmt.Sprintf("Fetching station numbers for %s (this can take a while)...", element)) + + query := fmt.Sprintf( + `SELECT DISTINCT stnr FROM %s WHERE %s IS NOT NULL`, + table.TableName, + element, + ) + + rows, err := conn.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var stnr string + if err := rows.Scan(&stnr); err != nil { + return nil, err + } + stations = append(stations, stnr) + } + + return stations, rows.Err() +} + +// Fetches all unique station numbers in the table +// FIXME: the DISTINCT query can be extremely slow +// NOTE: decided to use fetchStationsWithElement instead +func (table *Table) fetchStationNumbers(conn *sql.DB) (stations []string, err error) { + slog.Info(fmt.Sprint("Fetching station numbers (this can take a while)...")) + + query := fmt.Sprintf( + `SELECT DISTINCT stnr FROM %s`, + table.TableName, + ) + + rows, err := conn.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var stnr string + if err := rows.Scan(&stnr); err != nil { + return nil, err + } + stations = append(stations, stnr) + } + + return stations, rows.Err() +} diff --git a/migrations/kdvh/dump_functions.go b/migrations/kdvh/dump_functions.go new file mode 100644 index 0000000..8afe094 --- /dev/null +++ b/migrations/kdvh/dump_functions.go @@ -0,0 +1,276 @@ +package kdvh + +import ( + "database/sql" + "encoding/csv" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "time" +) + +// Fetch min and max year from table, needed for tables that are dumped by year +func fetchYearRange(tableName, station string, conn *sql.DB) (int64, int64, error) { + var beginStr, endStr string + query := fmt.Sprintf("SELECT min(to_char(dato, 'yyyy')), max(to_char(dato, 'yyyy')) FROM %s WHERE stnr = $1", tableName) + + if err := conn.QueryRow(query, station).Scan(&beginStr, &endStr); err != nil { + slog.Error(fmt.Sprint("Could not query row: ", err)) + return 0, 0, err + } + + begin, err := strconv.ParseInt(beginStr, 10, 64) + if err != nil { + slog.Error(fmt.Sprintf("Could not parse year '%s': %s", beginStr, err)) + return 0, 0, err + } + + end, err := strconv.ParseInt(endStr, 10, 64) + if err != nil { + slog.Error(fmt.Sprintf("Could not parse year '%s': %s", endStr, err)) + return 0, 0, err + } + + return begin, end, nil +} + +func dumpByYearDataOnly(path string, meta DumpMeta, conn *sql.DB) error { + begin, end, err := fetchYearRange(meta.dataTable, meta.station, conn) + if err != nil { + return err + } + + query := fmt.Sprintf( + `SELECT dato AS time, %[1]s AS data, '' AS flag FROM %[2]s + WHERE %[1]s IS NOT NULL + AND stnr = $1 AND TO_CHAR(dato, 'yyyy') = $2`, + meta.element, + meta.dataTable, + ) + + for year := begin; year < end; year++ { + rows, err := conn.Query(query, meta.station, year) + if err != nil { + slog.Error(fmt.Sprint("Could not query KDVH: ", err)) + return err + } + + path := filepath.Join(path, fmt.Sprint(year)) + if err := os.MkdirAll(path, os.ModePerm); err != nil { + slog.Error(err.Error()) + continue + } + + if err := dumpToFile(path, meta.element, rows); err != nil { + slog.Error(err.Error()) + return err + } + } + + return nil +} + +func dumpByYear(path string, meta DumpMeta, conn *sql.DB) error { + dataBegin, dataEnd, err := fetchYearRange(meta.dataTable, meta.station, conn) + if err != nil { + return err + } + + flagBegin, flagEnd, err := fetchYearRange(meta.flagTable, meta.station, conn) + if err != nil { + return err + } + + begin := min(dataBegin, flagBegin) + end := max(dataEnd, flagEnd) + + query := fmt.Sprintf( + `SELECT + dato AS time, + d.%[1]s AS data, + f.%[1]s AS flag + FROM + (SELECT dato, stnr, %[1]s FROM %[2]s + WHERE %[1]s IS NOT NULL AND stnr = $1 AND TO_CHAR(dato, 'yyyy') = $2) d + FULL OUTER JOIN + (SELECT dato, stnr, %[1]s FROM %[3]s + WHERE %[1]s IS NOT NULL AND stnr = $1 AND TO_CHAR(dato, 'yyyy') = $2) f + USING (dato)`, + meta.element, + meta.dataTable, + meta.flagTable, + ) + + for year := begin; year < end; year++ { + rows, err := conn.Query(query, meta.station, year) + if err != nil { + slog.Error(fmt.Sprint("Could not query KDVH: ", err)) + return err + } + + yearPath := filepath.Join(path, fmt.Sprint(year)) + if err := os.MkdirAll(path, os.ModePerm); err != nil { + slog.Error(err.Error()) + continue + } + + if err := dumpToFile(yearPath, meta.element, rows); err != nil { + slog.Error(err.Error()) + return err + } + } + + return nil +} + +func dumpHomogenMonth(path string, meta DumpMeta, conn *sql.DB) error { + query := fmt.Sprintf( + `SELECT dato AS time, %s[1]s AS data, '' AS flag FROM T_HOMOGEN_MONTH + WHERE %s[1]s IS NOT NULL AND stnr = $1 AND season BETWEEN 1 AND 12`, + // NOTE: adding a dummy argument is the only way to suppress this stupid warning + meta.element, "", + ) + + rows, err := conn.Query(query, meta.station) + if err != nil { + slog.Error(err.Error()) + return err + } + + if err := dumpToFile(path, meta.element, rows); err != nil { + slog.Error(err.Error()) + return err + } + + return nil +} + +func dumpDataOnly(path string, meta DumpMeta, conn *sql.DB) error { + query := fmt.Sprintf( + `SELECT dato AS time, %[1]s AS data, '' AS flag FROM %[2]s + WHERE %[1]s IS NOT NULL AND stnr = $1`, + meta.element, + meta.dataTable, + ) + + rows, err := conn.Query(query, meta.station) + if err != nil { + slog.Error(err.Error()) + return err + } + + if err := dumpToFile(path, meta.element, rows); err != nil { + slog.Error(err.Error()) + return err + } + + return nil +} + +func dumpDataAndFlags(path string, meta DumpMeta, conn *sql.DB) error { + query := fmt.Sprintf( + `SELECT + dato AS time, + d.%[1]s AS data, + f.%[1]s AS flag + FROM + (SELECT dato, %[1]s FROM %[2]s WHERE %[1]s IS NOT NULL AND stnr = $1) d + FULL OUTER JOIN + (SELECT dato, %[1]s FROM %[3]s WHERE %[1]s IS NOT NULL AND stnr = $1) f + USING (dato)`, + meta.element, + meta.dataTable, + meta.flagTable, + ) + + rows, err := conn.Query(query, meta.station) + if err != nil { + slog.Error(err.Error()) + return err + } + + if err := dumpToFile(path, meta.element, rows); err != nil { + slog.Error(err.Error()) + return err + } + + return nil +} + +func dumpToFile(path, element string, rows *sql.Rows) error { + filename := filepath.Join(path, element+".csv") + file, err := os.Create(filename) + if err != nil { + return err + } + + lines, err := sortRows(rows) + if err != nil { + return err + } + + err = writeElementFile(lines, file) + if closeErr := file.Close(); closeErr != nil { + return errors.Join(err, closeErr) + } + return err +} + +// Struct representing a single record in the output CSV file +type Record struct { + time time.Time + data sql.NullString + flag sql.NullString +} + +// Scans the rows and collects them in a slice of chronologically sorted lines +func sortRows(rows *sql.Rows) ([]Record, error) { + defer rows.Close() + + // TODO: if we use pgx we might be able to preallocate the right size + var records []Record + var record Record + + for rows.Next() { + if err := rows.Scan(&record.time, &record.data, &record.flag); err != nil { + return nil, errors.New("Could not scan rows: " + err.Error()) + } + records = append(records, record) + } + + slices.SortFunc(records, func(a, b Record) int { + return a.time.Compare(b.time) + }) + + return records, rows.Err() +} + +// Format string for date field in CSV files +const TIMEFORMAT string = "2006-01-02_15:04:05" + +// Writes queried (time | data | flag) columns to CSV +func writeElementFile(lines []Record, file io.Writer) error { + // Write number of lines as header + file.Write([]byte(fmt.Sprintf("%v\n", len(lines)))) + + writer := csv.NewWriter(file) + + record := make([]string, 3) + for _, l := range lines { + record[0] = l.time.Format(TIMEFORMAT) + record[1] = l.data.String + record[2] = l.flag.String + + if err := writer.Write(record); err != nil { + return errors.New("Could not write to file: " + err.Error()) + } + } + + writer.Flush() + return writer.Error() +} diff --git a/migrations/kdvh/import.go b/migrations/kdvh/import.go new file mode 100644 index 0000000..444ee9d --- /dev/null +++ b/migrations/kdvh/import.go @@ -0,0 +1,325 @@ +package kdvh + +import ( + "bufio" + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/rickb777/period" + + "migrate/lard" + "migrate/utils" +) + +type ImportConfig struct { + Verbose bool `short:"v" description:"Increase verbosity level"` + BaseDir string `short:"p" long:"path" default:"./dumps/kdvh" description:"Location the dumped data will be stored in"` + TablesCmd string `short:"t" long:"table" default:"" description:"Optional comma separated list of table names. By default all available tables are processed"` + StationsCmd string `short:"s" long:"station" default:"" description:"Optional comma separated list of stations IDs. By default all station IDs are processed"` + ElementsCmd string `short:"e" long:"elemcode" default:"" description:"Optional comma separated list of element codes. By default all element codes are processed"` + Sep string `long:"sep" default:"," description:"Separator character in the dumped files. Needs to be quoted"` + HasHeader bool `long:"header" description:"Add this flag if the dumped files have a header row"` + Skip string `long:"skip" choice:"data" choice:"flags" description:"Skip import of data or flags"` + Email []string `long:"email" description:"Optional email address used to notify if the program crashed"` + + Tables []string + Stations []string + Elements []string + + OffsetMap map[StinfoKey]period.Period // Map of offsets used to correct (?) KDVH times for specific parameters + StinfoMap map[StinfoKey]StinfoParam // Map of metadata used to query timeseries ID in LARD + KDVHMap map[KDVHKey]Timespan // Map of `from_time` and `to_time` for each (table, station, element) triplet. Not present for all parameters +} + +func (config *ImportConfig) setup() { + if len(config.Sep) > 1 { + slog.Warn("'--sep' only accepts single-byte characters. Defaulting to ','") + config.Sep = "," + } + if config.TablesCmd != "" { + config.Tables = strings.Split(config.TablesCmd, ",") + } + if config.StationsCmd != "" { + config.Stations = strings.Split(config.StationsCmd, ",") + } + if config.ElementsCmd != "" { + config.Elements = strings.Split(config.ElementsCmd, ",") + } + config.CacheMetadata() +} + +func (config *ImportConfig) Execute([]string) error { + config.setup() + + // Create connection pool for LARD + pool, err := pgxpool.New(context.TODO(), os.Getenv("LARD_STRING")) + if err != nil { + slog.Error(fmt.Sprint("Could not connect to Lard:", err)) + return err + } + defer pool.Close() + + for _, table := range KDVH { + if config.Tables != nil && !slices.Contains(config.Tables, table.TableName) { + continue + } + table.Import(pool, config) + } + + return nil +} + +func (table *Table) Import(pool *pgxpool.Pool, config *ImportConfig) (rowsInserted int64) { + defer utils.SendEmailOnPanic("importTable", config.Email) + + if table.importUntil == 0 { + if config.Verbose { + slog.Info("Skipping import of" + table.TableName + " because this table is not set for import") + } + return 0 + } + + utils.SetLogFile(table.TableName, "import") + + table.Path = filepath.Join(config.BaseDir, table.Path) + stations, err := os.ReadDir(table.Path) + if err != nil { + slog.Warn(fmt.Sprintf("Could not read directory %s: %s", table.Path, err)) + return 0 + } + + bar := utils.NewBar(len(stations), table.TableName) + bar.RenderBlank() + for _, station := range stations { + count, err := table.importStation(station, pool, config) + if err == nil { + rowsInserted += count + } + bar.Add(1) + } + + outputStr := fmt.Sprintf("%v: %v total rows inserted", table.TableName, rowsInserted) + slog.Info(outputStr) + fmt.Println(outputStr) + return rowsInserted +} + +// Loops over the element files present in the station directory and processes them concurrently +func (table *Table) importStation(station os.DirEntry, pool *pgxpool.Pool, config *ImportConfig) (totRows int64, err error) { + stnr, err := getStationNumber(station, config.Stations) + if err != nil { + if config.Verbose { + slog.Info(err.Error()) + } + return 0, err + } + + dir := filepath.Join(table.Path, station.Name()) + elements, err := os.ReadDir(dir) + if err != nil { + slog.Warn(fmt.Sprintf("Could not read directory %s: %s", dir, err)) + return 0, err + } + + var wg sync.WaitGroup + for _, element := range elements { + elemCode, err := getElementCode(element, config.Elements) + if err != nil { + if config.Verbose { + slog.Info(err.Error()) + } + continue + } + + wg.Add(1) + go func() { + defer wg.Done() + + tsInfo, err := config.NewTimeseriesInfo(table.TableName, elemCode, stnr) + if err != nil { + return + } + + tsid, err := getTimeseriesID(tsInfo, pool) + if err != nil { + slog.Error(tsInfo.logstr + "could not obtain timeseries - " + err.Error()) + return + } + + filename := filepath.Join(dir, element.Name()) + data, err := table.parseElementFile(filename, tsInfo, config) + if err != nil { + return + } + + ts := lard.NewTimeseries(tsid, data) + count, err := importData(ts, tsInfo, pool, config) + if err != nil { + return + } + totRows += count + }() + } + wg.Wait() + + return totRows, nil +} + +func (table *Table) parseElementFile(filename string, tsInfo *TimeseriesInfo, config *ImportConfig) ([]lard.Obs, error) { + file, err := os.Open(filename) + if err != nil { + slog.Warn(fmt.Sprintf("Could not open file '%s': %s", filename, err)) + return nil, err + } + defer file.Close() + + data, err := table.parseData(file, tsInfo, config) + if err != nil { + slog.Error(fmt.Sprintf("Could not parse data from '%s': %s", filename, err)) + return nil, err + } + + if len(data) == 0 { + slog.Info(tsInfo.logstr + "no rows to insert (all obstimes > max import time)") + return nil, err + } + + return data, nil +} + +func importData(ts *lard.Timeseries, tsInfo *TimeseriesInfo, pool *pgxpool.Pool, config *ImportConfig) (count int64, err error) { + if !(config.Skip == "data") { + if tsInfo.param.IsScalar { + count, err = lard.InsertData(ts, pool, tsInfo.logstr) + if err != nil { + slog.Error(tsInfo.logstr + "failed data bulk insertion - " + err.Error()) + return 0, err + } + } else { + count, err = lard.InsertNonscalarData(ts, pool, tsInfo.logstr) + if err != nil { + slog.Error(tsInfo.logstr + "failed non-scalar data bulk insertion - " + err.Error()) + return 0, err + } + // TODO: should we skip inserting flags here? In kvalobs there are no flags for text data + // return count, nil + } + } + + if !(config.Skip == "flags") { + if err := lard.InsertFlags(ts, pool, tsInfo.logstr); err != nil { + slog.Error(tsInfo.logstr + "failed flag bulk insertion - " + err.Error()) + } + } + + return count, nil + +} + +func getStationNumber(station os.DirEntry, stationList []string) (int32, error) { + if !station.IsDir() { + return 0, errors.New(fmt.Sprintf("%s is not a directory, skipping", station.Name())) + } + + if stationList != nil && !slices.Contains(stationList, station.Name()) { + return 0, errors.New(fmt.Sprintf("Station %v not in the list, skipping", station.Name())) + } + + stnr, err := strconv.ParseInt(station.Name(), 10, 32) + if err != nil { + return 0, errors.New("Error parsing station number:" + err.Error()) + } + + return int32(stnr), nil +} + +func getElementCode(element os.DirEntry, elementList []string) (string, error) { + elemCode := strings.ToUpper(strings.TrimSuffix(element.Name(), ".csv")) + + if elementList != nil && !slices.Contains(elementList, elemCode) { + return "", errors.New(fmt.Sprintf("Element '%s' not in the list, skipping", elemCode)) + } + + if elemcodeIsInvalid(elemCode) { + return "", errors.New(fmt.Sprintf("Element '%s' not set for import, skipping", elemCode)) + } + return elemCode, nil +} + +func getTimeseriesID(tsInfo *TimeseriesInfo, pool *pgxpool.Pool) (int32, error) { + label := lard.Label{ + StationID: tsInfo.station, + TypeID: tsInfo.param.TypeID, + ParamID: tsInfo.param.ParamID, + Sensor: &tsInfo.param.Sensor, + Level: tsInfo.param.Hlevel, + } + tsid, err := lard.GetTimeseriesID(label, tsInfo.param.Fromtime, pool) + if err != nil { + slog.Error(tsInfo.logstr + "could not obtain timeseries - " + err.Error()) + return 0, err + + } + return tsid, nil +} + +func (table *Table) parseData(handle *os.File, meta *TimeseriesInfo, config *ImportConfig) ([]lard.Obs, error) { + scanner := bufio.NewScanner(handle) + + var rowCount int + // Try to infer row count from header + if config.HasHeader { + scanner.Scan() + // rowCount, _ = strconv.Atoi(scanner.Text()) + if temp, err := strconv.Atoi(scanner.Text()); err == nil { + rowCount = temp + } + } + + data := make([]lard.Obs, 0, rowCount) + for scanner.Scan() { + cols := strings.Split(scanner.Text(), config.Sep) + + obsTime, err := time.Parse("2006-01-02_15:04:05", cols[0]) + if err != nil { + return nil, err + } + + // Only import data between KDVH's defined fromtime and totime + if meta.span.FromTime != nil && obsTime.Sub(*meta.span.FromTime) < 0 { + continue + } else if meta.span.ToTime != nil && obsTime.Sub(*meta.span.ToTime) > 0 { + break + } + + if obsTime.Year() >= table.importUntil { + break + } + + temp, err := table.convFunc(Obs{meta, obsTime, cols[1], cols[2]}) + if err != nil { + return nil, err + } + + data = append(data, temp) + } + + return data, nil +} + +// TODO: add CALL_SIGN? It's not in stinfosys? +var INVALID_ELEMENTS = []string{"TYPEID", "TAM_NORMAL_9120", "RRA_NORMAL_9120", "OT", "OTN", "OTX", "DD06", "DD12", "DD18"} + +func elemcodeIsInvalid(element string) bool { + return strings.Contains(element, "KOPI") || slices.Contains(INVALID_ELEMENTS, element) +} diff --git a/migrations/kdvh/import_functions.go b/migrations/kdvh/import_functions.go new file mode 100644 index 0000000..0006067 --- /dev/null +++ b/migrations/kdvh/import_functions.go @@ -0,0 +1,465 @@ +package kdvh + +import ( + "errors" + "migrate/lard" + "strconv" + + "github.com/rickb777/period" +) + +// In kvalobs a flag is a 16 char string containg QC information about the observation: +// Note: Missing numbers in the following lists are marked as reserved (not in use I guess?) +// +// CONTROLINFO FLAG: +// +// 0 - CONTROL LEVEL (not used) +// 1 - RANGE CHECK +// 0. Not checked +// 1. Check passed +// 2. Higher than HIGH +// 3. Lower than LOW +// 4. Higher than HIGHER +// 5. Lower that LOWER +// 6. Check failed, above HIGHEST or below LOWEST +// +// 2 - FORMAL CONSISTENCY CHECK +// 0. Not checked +// 1. Check passe +// 2. Inconsistency found, but not an error with the relevant parameter, no correction +// 3. Inconsistency found at the observation time, but not possible to determine which parameter, no correction +// 4. Inconsistency found at earliar/later observation times, but not possible to determine which parameter, no correction +// 6. Inconsistency found at the observation time, probably error with the relevant parameter, no correction +// 7. Inconsistency found at earliar/later observation times, probably error with relevant parameter, no correction +// 8. Inconsistency found, a parameter is missing, no correction +// A. Inconsistency found at the observation time, corrected automatically +// B. Inconsistency found at earliar/later observation times, corrected automatically +// D. Check failed +// +// 3 - JUMP CHECK (STEP, DIP, FREEZE, DRIFT) +// 0. Not checked +// 1. Check passed +// 2. Change higher than test value, no correction +// 3. No change in measured value (freeze check did not pass?), no correction +// 4. Suspected error in freeze check, no error in dip check (??), no correction +// 5. Suspected error in dip check, no error in freeze check (??), no correction +// 7. Observed drift, no correction +// 9. Change higher than test value, corrected automatically +// A. Freeze check did not pass, corrected automatically +// +// 4 - PROGNOSTIC CHECK +// 0. Not checked +// 1. Check passed +// 2. Deviation from model higher than HIGH +// 3. Deviation from model lower than LOW +// 4. Deviation from model higher than HIGHER +// 5. Deviation from model lower that LOWER +// 6. Check failed, deviation from model above HIGHEST or below LOWEST +// +// 5 - VALUE CHECK (FOR MOVING STATIONS) +// 0. Not checked +// 1. Check passed +// 3. Suspicious value, no correction +// 4. Suspicious value, corrected automatically +// 6. Check failed +// +// 6 - MISSING OBSERVATIONS +// 0. Original and corrected values exist +// 1. Original value missing, but corrected value exists +// 2. Corrected value missing, orginal value discarded +// 3. Original and corrected values missing +// +// 7 - TIMESERIES FITTING +// 0. Not checked +// 1. Interpolated with good fitness +// 2. Interpolated with unsure fitness +// 3. Intepolation not suitable +// +// 8 - WEATHER ANALYSIS +// 0. Not checked +// 1. Check passed +// 2. Suspicious value, not corrected +// 3. Suspicious value, corrected automatically +// +// 9 - STATISTICAL CHECK +// 0. Not checked +// 1. Check passed +// 2. Suspicious value, not corrected +// +// 10 - CLIMATOLOGICAL CONSISTENCY CHECK +// 0. Not checked +// 1. Check passed +// 2. Climatologically questionable, but not an error with the relevant parameter, no correction +// 3. Climatologically questionable at the observation time, but not possible to determine which parameter, no correction +// 4. Climatologically questionable at earliar/later observation times, but not possible to determine which parameter, no correction +// 6. Climatologically questionable at the observation time, probably error with the relevant parameter, no correction +// 7. Climatologically questionable at earliar/later observation times, probably error with relevant parameter, no correction +// A. Inconsistency found at the observation time, corrected automatically +// B. Inconsistency found at earliar/later observation times, corrected automatically +// D. Check failed +// +// 11 - CLIMATOLOGICAL CHECK +// 0. Not checked +// 1. Check passed +// 2. Suspicious value, not corrected +// 3. Suspicious value, corrected automatically +// +// 12 - DISTRIBUTION CHECK OF ACCUMULATED PARAMETERS (ESPECIALLY FOR PRECIPITATION) +// 0. Not checked +// 1. Not an accumulated value +// 2. Observation outside accumulated parameter range +// 3. Abnormal observation (??) +// 6. Accumulation calculated from numerical model +// 7. Accumulation calculated from weather analysis +// A. Accumulation calculated with 'steady rainfall' method +// B. Accumulation calculated with 'uneven rainfall' method +// +// 13 - PREQUALIFICATION (CERTAIN PAIRS OF 'STATIONID' AND 'PARAMID' CAN BE DISCARDED) +// 0. Not checked +// 5. Value is missing +// 6. Check failed, invalid original value +// 7. Check failed, original value is noisy +// +// 14 - COMBINATION CHECK +// 0. Not checked +// 1. Check passed +// 2. Outside test limit value, but no jumps detected and inside numerical model tolerance +// 9. Check failed. Outside test limit value, no jumps detected but outside numerical model tolerance +// A. Check failed. Outside test limit value, jumps detected but inside numerical model tolerance +// B. Check failed. Outside test limit value, jumps detected and outside numerical model tolerance +// +// 15 - MANUAL QUALITY CONTROL +// 0. Not checked +// 1. Check passed +// 2. Probably OK +// 5. Value manually interpolated +// 6. Value manually assigned +// 7. Value manually corrected +// A. Manually rejected + +const ( + VALUE_PASSED_QC = "00000" + "00000000000" + + // Corrected value is present, and original was remove by QC + VALUE_CORRECTED_AUTOMATICALLY = "00000" + "01000000000" + VALUE_MANUALLY_INTERPOLATED = "00000" + "01000000005" + VALUE_MANUALLY_ASSIGNED = "00000" + "01000000006" + + VALUE_REMOVED_BY_QC = "00000" + "02000000000" // Corrected value is missing, and original was remove by QC + VALUE_MISSING = "00000" + "03000000000" // Both original and corrected are missing + VALUE_PASSED_HQC = "00000" + "00000000001" // Value was sent to HQC for inspection, but it was OK + + // original value still exists, not exactly sure what the difference with VALUE_MANUALLY_INTERPOLATED is + INTERPOLATION_ADDED_MANUALLY = "00000" + "00000000005" +) + +// USEINFO FLAG: +// +// 0 - CONTROL LEVELS PASSED +// 1. Completed QC1, QC2 and HQC +// 2. Completed QC2 and HQC +// 3. Completed QC1 and HQC +// 4. Completed HQC +// 5. Completed QC1 and QC2 +// 6. Completed QC2 +// 7. Completed QC1 +// 9. Missing information +// +// 1 - DEVIATION FROM NORM (MEAN?) +// 0. Observation time and period are okay +// 1. Observation time deviates from norm +// 2. Observation period is shorter than norm +// 3. Observation perios is longer than norm +// 4. Observation time deviates from norm, and period is shorter than norm +// 5. Observation time deviates from norm, and period is longer than norm +// 8. Missing value +// 9. Missing status information +// +// 2 - QUALITY LEVEL OF ORIGNAL VALUE +// 0. Value is okay +// 1. Value is suspicious (probably correct) +// 2. Value is suspicious (probably wrong) +// 3. Value is wrong +// 9. Missing quality information +// +// 3 - TREATMENT OF ORIGINAL VALUE +// 0. Unchanged +// 1. Manually corrected +// 2. Manually interpolated +// 3. Automatically corrected +// 4. Automatically interpolated +// 5. Manually derived from accumulated value +// 6. Automatically derived from accumulated value +// 8. Rejected +// 9. Missing information +// +// 4 - MOST IMPORT CHECK RESULT (?) +// 0. Original value is okay +// 1. Range check +// 2. Consistency check +// 3. Jump check +// 4. Consistency check in relation with earlier/later observations +// 5. Prognostic check based on observation data +// 6. Prognostic check based on Timeseries +// 7. Prognostic check based on model data +// 8. Prognostic check based on statistics +// 9. Missing information +// +// 7 - DELAY INFORMATION +// 0. Observation carried out and reported at the right time +// 1. Observation carried out early and reported at the right time +// 2. Observation carried out late and reported at the right time +// 3. Observation reported early +// 4. Observation reported late +// 5. Observation carried out early and reported late +// 6. Observation carried out late and reported late +// 9. Missing information +// +// 8 - FIRST DIGIT OF HEXADECIMAL VALUE OF THE OBSERVATION CONFIDENCE LEVEL +// 9 - SECOND DIGIT OF HEXADECIMAL VALUE OF THE OBSERVATION CONFIDENCE LEVEL +// 13 - FIRST HQC OPERATOR DIGIT +// 14 - SECOND HQC OPERATOR DIGIT +// 15 - HEXADECIMAL DIGIT WITH NUMBER OF TESTS THAT DID NOT PASS (RETURNED A RESULT?) + +const ( + // Remaing 11 digits of `useinfo` that follow the 5 digits contained in `obs.Flags`. + // TODO: From the docs it looks like the '9' should be changed by kvalobs when + // the observation is inserted into the database but that's not the case? + DELAY_DEFAULT = "00900000000" + + INVALID_FLAGS = "99999" + DELAY_DEFAULT // Only returned when the flags are invalid + COMPLETED_HQC = "40000" + DELAY_DEFAULT // Specific to T_VDATA + DIURNAL_INTERPOLATED_USEINFO = "48925" + DELAY_DEFAULT // Specific to T_DIURNAL_INTERPOLATED +) + +func (obs *Obs) flagsAreValid() bool { + if len(obs.Flags) != 5 { + return false + } + _, err := strconv.ParseInt(obs.Flags, 10, 32) + return err == nil +} + +func (obs *Obs) Useinfo() string { + if !obs.flagsAreValid() { + return INVALID_FLAGS + } + return obs.Flags + DELAY_DEFAULT +} + +// The following functions try to recover the original pair of `controlinfo` +// and `useinfo` generated by Kvalobs for the observation, based on `Obs.Flags` and `Obs.Data` +// Different KDVH tables need different ways to perform this conversion. + +func makeDataPage(obs Obs) (lard.Obs, error) { + var valPtr *float32 + + controlinfo := VALUE_PASSED_QC + if obs.Data == "" { + controlinfo = VALUE_MISSING + } + + // NOTE: this is the only function that can return `lard.Obs` + // with non-null text data + if !obs.param.IsScalar { + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Text: &obs.Data, + Useinfo: obs.Useinfo(), + Controlinfo: controlinfo, + }, nil + } + + val, err := strconv.ParseFloat(obs.Data, 32) + if err == nil { + f32 := float32(val) + valPtr = &f32 + } + + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Useinfo: obs.Useinfo(), + Controlinfo: controlinfo, + }, nil +} + +// modify obstimes to always use totime +func makeDataPageProduct(obs Obs) (lard.Obs, error) { + obsLard, err := makeDataPage(obs) + if !obs.offset.IsZero() { + if temp, ok := obs.offset.AddTo(obsLard.Obstime); ok { + obsLard.Obstime = temp + } + } + return obsLard, err +} + +func makeDataPageEdata(obs Obs) (lard.Obs, error) { + var controlinfo string + var valPtr *float32 + + if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { + switch obs.Flags { + case "70381", "70389", "90989": + controlinfo = VALUE_REMOVED_BY_QC + default: + // Includes "70000", "70101", "99999" + controlinfo = VALUE_MISSING + } + } else { + controlinfo = VALUE_PASSED_QC + f32 := float32(val) + valPtr = &f32 + } + + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Useinfo: obs.Useinfo(), + Controlinfo: controlinfo, + }, nil +} + +func makeDataPagePdata(obs Obs) (lard.Obs, error) { + var controlinfo string + var valPtr *float32 + + if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { + switch obs.Flags { + case "20389", "30389", "40389", "50383", "70381", "71381": + controlinfo = VALUE_REMOVED_BY_QC + default: + // "00000", "10000", "10319", "30000", "30319", + // "40000", "40929", "48929", "48999", "50000", + // "50205", "60000", "70000", "70103", "70203", + // "71000", "71203", "90909", "99999" + controlinfo = VALUE_MISSING + } + } else { + f32 := float32(val) + valPtr = &f32 + + switch obs.Flags { + case "10319", "10329", "30319", "40319", "48929", "48999": + controlinfo = VALUE_MANUALLY_INTERPOLATED + case "20389", "30389", "40389", "50383", "70381", "71381", "99319": + controlinfo = VALUE_CORRECTED_AUTOMATICALLY + case "40929": + controlinfo = INTERPOLATION_ADDED_MANUALLY + default: + // "71000", "71203", "90909", "99999" + controlinfo = VALUE_PASSED_QC + } + + } + + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Useinfo: obs.Useinfo(), + Controlinfo: controlinfo, + }, nil +} + +func makeDataPageNdata(obs Obs) (lard.Obs, error) { + var controlinfo string + var valPtr *float32 + + if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { + switch obs.Flags { + case "70389": + controlinfo = VALUE_REMOVED_BY_QC + default: + // "30319", "38929", "40000", "40100", "40315" + // "40319", "43325", "48325", "49225", "49915" + // "70000", "70204", "71000", "73309", "78937" + // "90909", "93399", "98999", "99999" + controlinfo = VALUE_MISSING + } + } else { + switch obs.Flags { + case "43325", "48325": + controlinfo = VALUE_MANUALLY_ASSIGNED + case "30319", "38929", "40315", "40319": + controlinfo = VALUE_MANUALLY_INTERPOLATED + case "49225", "49915": + controlinfo = INTERPOLATION_ADDED_MANUALLY + case "70389", "73309", "78937", "93399", "98999": + controlinfo = VALUE_CORRECTED_AUTOMATICALLY + default: + // "40000", "40100", "70000", "70204", "71000", "90909", "99999" + controlinfo = VALUE_PASSED_QC + } + f32 := float32(val) + valPtr = &f32 + } + + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Useinfo: obs.Useinfo(), + Controlinfo: controlinfo, + }, nil +} + +func makeDataPageVdata(obs Obs) (lard.Obs, error) { + var useinfo, controlinfo string + var valPtr *float32 + + // set useinfo based on time + if h := obs.Obstime.Hour(); h == 0 || h == 6 || h == 12 || h == 18 { + useinfo = COMPLETED_HQC + } else { + useinfo = INVALID_FLAGS + } + + // set data and controlinfo + if val, err := strconv.ParseFloat(obs.Data, 32); err != nil { + controlinfo = VALUE_MISSING + } else { + // super special treatment clause of T_VDATA.OT_24, so it will be the same as in kvalobs + f32 := float32(val) + + if obs.element == "OT_24" { + // add custom offset, because OT_24 in KDVH has been treated differently than OT_24 in kvalobs + offset, err := period.Parse("PT18H") // fromtime_offset -PT6H, timespan P1D + if err != nil { + return lard.Obs{}, errors.New("could not parse period") + } + temp, ok := offset.AddTo(obs.Obstime) + if !ok { + return lard.Obs{}, errors.New("could not add period") + } + + obs.Obstime = temp + // convert from hours to minutes + f32 *= 60.0 + } + valPtr = &f32 + controlinfo = VALUE_PASSED_QC + } + + return lard.Obs{ + Obstime: obs.Obstime, + Data: valPtr, + Useinfo: useinfo, + Controlinfo: controlinfo, + }, nil +} + +func makeDataPageDiurnalInterpolated(obs Obs) (lard.Obs, error) { + val, err := strconv.ParseFloat(obs.Data, 32) + if err != nil { + return lard.Obs{}, err + } + f32 := float32(val) + + return lard.Obs{ + Obstime: obs.Obstime, + Data: &f32, + Useinfo: DIURNAL_INTERPOLATED_USEINFO, + Controlinfo: VALUE_MANUALLY_INTERPOLATED, + }, nil +} diff --git a/migrations/kdvh/import_test.go b/migrations/kdvh/import_test.go new file mode 100644 index 0000000..ba6e574 --- /dev/null +++ b/migrations/kdvh/import_test.go @@ -0,0 +1,31 @@ +package kdvh + +import "testing" + +func TestFlagsAreValid(t *testing.T) { + type testCase struct { + input Obs + expected bool + } + + cases := []testCase{ + {Obs{Flags: "12309"}, true}, + {Obs{Flags: "984.3"}, false}, + {Obs{Flags: ".1111"}, false}, + {Obs{Flags: "1234."}, false}, + {Obs{Flags: "12.2.4"}, false}, + {Obs{Flags: "12.343"}, false}, + {Obs{Flags: ""}, false}, + {Obs{Flags: "asdas"}, false}, + {Obs{Flags: "12a3a"}, false}, + {Obs{Flags: "1sdfl"}, false}, + } + + for _, c := range cases { + t.Log("Testing flag:", c.input.Flags) + + if result := c.input.flagsAreValid(); result != c.expected { + t.Errorf("Got %v, wanted %v", result, c.expected) + } + } +} diff --git a/migrations/kdvh/list_tables.go b/migrations/kdvh/list_tables.go new file mode 100644 index 0000000..c030a6c --- /dev/null +++ b/migrations/kdvh/list_tables.go @@ -0,0 +1,24 @@ +package kdvh + +import ( + "fmt" + "slices" +) + +type ListConfig struct{} + +func (config *ListConfig) Execute(_ []string) error { + fmt.Println("Available tables in KDVH:") + + var tables []string + for table := range KDVH { + tables = append(tables, table) + } + + slices.Sort(tables) + for _, table := range tables { + fmt.Println(" -", table) + } + + return nil +} diff --git a/migrations/kdvh/main.go b/migrations/kdvh/main.go new file mode 100644 index 0000000..af10422 --- /dev/null +++ b/migrations/kdvh/main.go @@ -0,0 +1,53 @@ +package kdvh + +// Command line arguments for KDVH migrations +type Cmd struct { + Dump DumpConfig `command:"dump" description:"Dump tables from KDVH to CSV"` + Import ImportConfig `command:"import" description:"Import CSV file dumped from KDVH"` + List ListConfig `command:"list" description:"List available KDVH tables"` +} + +// The KDVH database simply contains a map of "table name" to `Table` +var KDVH map[string]*Table = map[string]*Table{ + // Section 1: tables that need to be migrated entirely + // TODO: figure out if we need to use the elem_code_paramid_level_sensor_t_edata table? + "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetConvFunc(makeDataPageEdata).SetImport(3000), + // NOTE(1): there is a T_METARFLAG, but it's empty + // NOTE(2): already dumped, but with wrong format? + "T_METARDATA": NewTable("T_METARDATA", "", "T_ELEM_METARDATA").SetDumpFunc(dumpDataOnly).SetImport(3000), // already dumped + + // Section 2: tables with some data in kvalobs, import only up to 2005-12-31 + "T_ADATA": NewTable("T_ADATA", "T_AFLAG", "T_ELEM_OBS").SetImport(2006), + "T_MDATA": NewTable("T_MDATA", "T_MFLAG", "T_ELEM_OBS").SetImport(2006), // already dumped + "T_TJ_DATA": NewTable("T_TJ_DATA", "T_TJ_FLAG", "T_ELEM_OBS").SetImport(2006), // already dumped + "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPagePdata).SetImport(2006), // already dumped + "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageNdata).SetImport(2006), // already dumped + "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageVdata).SetImport(2006), // already dumped + "T_UTLANDDATA": NewTable("T_UTLANDDATA", "T_UTLANDFLAG", "T_ELEM_OBS").SetImport(2006), // already dumped + + // Section 3: tables that should only be dumped + "T_10MINUTE_DATA": NewTable("T_10MINUTE_DATA", "T_10MINUTE_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), + "T_ADATA_LEVEL": NewTable("T_ADATA_LEVEL", "T_AFLAG_LEVEL", "T_ELEM_OBS"), + + // TODO: T_AVINOR, T_PROJDATA have a bunch of parameters that are not in Stinfosys? + // But it shouldn't be a problem if the goal is to only dump them? + "T_AVINOR": NewTable("T_AVINOR", "T_AVINOR_FLAG", "T_ELEM_OBS"), + // TODO: T_PROJFLAG is not in the proxy! And T_PROJDATA is not readable from the proxy + // "T_PROJDATA": newTable("T_PROJDATA", "T_PROJFLAG", "T_ELEM_PROJ"), + "T_MINUTE_DATA": NewTable("T_MINUTE_DATA", "T_MINUTE_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), // already dumped + "T_SECOND_DATA": NewTable("T_SECOND_DATA", "T_SECOND_FLAG", "T_ELEM_OBS").SetDumpFunc(dumpByYear), // already dumped + "T_CDCV_DATA": NewTable("T_CDCV_DATA", "T_CDCV_FLAG", "T_ELEM_EDATA"), // already dumped + "T_MERMAID": NewTable("T_MERMAID", "T_MERMAID_FLAG", "T_ELEM_EDATA"), // already dumped + "T_SVVDATA": NewTable("T_SVVDATA", "T_SVVFLAG", "T_ELEM_OBS"), // already dumped + + // Section 4: other special cases + // TODO: do we need to import these? + "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetConvFunc(makeDataPageProduct).SetImport(1957), + "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetConvFunc(makeDataPageProduct), + "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH").SetDumpFunc(dumpDataOnly).SetConvFunc(makeDataPageProduct), + "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", "").SetDumpFunc(dumpHomogenMonth).SetConvFunc(makeDataPageProduct), + + // TODO: these two are the only tables seemingly missing from the KDVH proxy + // {TableName: "T_DIURNAL_INTERPOLATED", DataFunction: makeDataPageDiurnalInterpolated, ImportUntil: 3000}, + // {TableName: "T_MONTH_INTERPOLATED", DataFunction: makeDataPageDiurnalInterpolated, ImportUntil: 3000}, +} diff --git a/migrations/kdvh/product_offsets.csv b/migrations/kdvh/product_offsets.csv new file mode 100644 index 0000000..7ee4fdb --- /dev/null +++ b/migrations/kdvh/product_offsets.csv @@ -0,0 +1,161 @@ +table_name,elem_code,paramid,fromtime_offset,timespan +T_DIURNAL,EE,129,PT6H, +T_DIURNAL,EM,7,PT6H, +T_DIURNAL,FF2M,3044,-PT1H,P1D +T_DIURNAL,FF2N,3046,-PT1H,P1D +T_DIURNAL,FF2X,3049,-PT1H,P1D +T_DIURNAL,FFM,3050,-PT1H,P1D +T_DIURNAL,FFN,3052,-PT1H,P1D +T_DIURNAL,FFX,3054,-PT1H,P1D +T_DIURNAL,FGM,3056,-PT6H,P1D +T_DIURNAL,FGN,3058,-PT6H,P1D +T_DIURNAL,FGX,3060,-PT6H,P1D +T_DIURNAL,FLRR,,-PT18H,P1D +T_DIURNAL,FXM,3063,-PT6H,P1D +T_DIURNAL,FXN,3065,-PT6H,P1D +T_DIURNAL,FXX,3067,-PT6H,P1D +T_DIURNAL,GD17,3073,-PT1H,P1D +T_DIURNAL,HWAM,3147,-PT1H,P1D +T_DIURNAL,HWAN,3151,-PT1H,P1D +T_DIURNAL,HWAX,3149,-PT1H,P1D +T_DIURNAL,MR,3197,-PT1H,P1D +T_DIURNAL,NN04,3075,-PT1H,P1D +T_DIURNAL,NN09,3077,-PT1H,P1D +T_DIURNAL,NN20,3079,-PT1H,P1D +T_DIURNAL,NNM,3081,-PT1H,P1D +T_DIURNAL,NNN,3086,-PT1H,P1D +T_DIURNAL,NNX,3088,-PT1H,P1D +T_DIURNAL,OT,122,-P1D,P1D +T_DIURNAL,POM,3032,-PT1H,P1D +T_DIURNAL,PON,3035,-PT1H,P1D +T_DIURNAL,POX,3037,-PT1H,P1D +T_DIURNAL,PRM,3093,-PT1H,P1D +T_DIURNAL,PRN,3095,-PT1H,P1D +T_DIURNAL,PRX,3097,-PT1H,P1D +T_DIURNAL,PWAM,3141,-PT1H,P1D +T_DIURNAL,PWAN,3145,-PT1H,P1D +T_DIURNAL,PWAX,3143,-PT1H,P1D +T_DIURNAL,RR,110,-PT18H,P1D +T_DIURNAL,RR_720,3243,-P29DT18H,P30D +T_DIURNAL,RRID,117,PT6H, +T_DIURNAL,RRTA,3241,-PT6H,P1D +T_DIURNAL,SA,112,PT6H, +T_DIURNAL,SAE,3109,-PT18H,P1D +T_DIURNAL,SD,18,PT6H, +T_DIURNAL,SGN,3119,-PT1H,P1D +T_DIURNAL,SGX,3121,-PT1H,P1D +T_DIURNAL,SH,3123,-PT1H,P1D +T_DIURNAL,SLAG,10051,-PT18H,P1D +T_DIURNAL,SLAGV,10052,-PT18H,P1D +T_DIURNAL,SLAGW,10053,-PT18H,P1D +T_DIURNAL,SLAGWA,10054,-PT18H,P1D +T_DIURNAL,SS_24,114,-PT18H,P1D +T_DIURNAL,TAM,3016,-PT1H,P1D +T_DIURNAL,X1TAM,3016,-PT1H,P1D +T_DIURNAL,TAM_K,3125,-PT1H,P1D +T_DIURNAL,TAM10,3016,-PT1H,P1D +T_DIURNAL,TAMRR,3300,-PT18H,P1D +T_DIURNAL,TAN,3304,-PT6H,P1D +T_DIURNAL,X1TAN,3304,-PT6H,P1D +T_DIURNAL,TAND,3302,-PT1H,P1D +T_DIURNAL,TAX,3305,-PT6H,P1D +T_DIURNAL,X1TAX,3305,-PT6H,P1D +T_DIURNAL,TAXD,3303,-PT1H,P1D +T_DIURNAL,TD,3026,-PT1H,P1D +T_DIURNAL,TGN,3028,-PT1H,P1D +T_DIURNAL,TW,3306,-PT1H,P1D +T_DIURNAL,UM,266,-PT1H,P1D +T_DIURNAL,UM10,266,-PT1H,P1D +T_DIURNAL,UN,3006,-PT1H,P1D +T_DIURNAL,UX,3004,-PT1H,P1D +T_DIURNAL,VEKST,3134,-PT1H,P1D +T_DIURNAL,VP,3136,-PT1H,P1D +T_DIURNAL,VSUM,3138,-PT1H,P1D +T_DIURNAL,VVN,3002,-PT1H,P1D +T_DIURNAL,VVX,3000,-PT1H,P1D +T_MONTH,DRR_GE1,3194,-PT1H,P1M +T_MONTH,FF2M,3045,-PT1H,P1M +T_MONTH,FF2N,3047,-PT1H,P1M +T_MONTH,FF2X,3048,-PT1H,P1M +T_MONTH,FFM,3051,-PT1H,P1M +T_MONTH,FFN,3053,-PT1H,P1M +T_MONTH,FFX,3055,-PT1H,P1M +T_MONTH,FGM,3057,-PT6H,P1M +T_MONTH,FGN,3059,-PT6H,P1M +T_MONTH,FGX,3061,-PT6H,P1M +T_MONTH,FXM,3062,-PT6H,P1M +T_MONTH,FXN,3064,-PT6H,P1M +T_MONTH,FXX,3066,-PT6H,P1M +T_MONTH,FXXDT,3068,-PT1H,P1M +T_MONTH,GD17,3069,-PT1H,P1M +T_MONTH,GD17_I,3074,-PT1H,P1M +T_MONTH,HWAM,3148,-PT1H,P1M +T_MONTH,HWAN,3152,-PT1H,P1M +T_MONTH,HWAX,3150,-PT1H,P1M +T_MONTH,MRM,3193,-PT1H,P1M +T_MONTH,NN04,3076,-PT1H,P1M +T_MONTH,NN09,3078,-PT1H,P1M +T_MONTH,NN20,3080,-PT1H,P1M +T_MONTH,NNM,3082,-PT1H,P1M +T_MONTH,NNN,3087,-PT1H,P1M +T_MONTH,NNX,3089,-PT1H,P1M +T_MONTH,OT,3090,-PT1H,P1M +T_MONTH,OTN,3091,-PT1H,P1M +T_MONTH,OTX,3092,-PT1H,P1M +T_MONTH,POM,3033,-PT1H,P1M +T_MONTH,PON,3036,-PT1H,P1M +T_MONTH,POX,3038,-PT1H,P1M +T_MONTH,PRM,3094,-PT1H,P1M +T_MONTH,PRN,3096,-PT1H,P1M +T_MONTH,PRX,3098,-PT1H,P1M +T_MONTH,PWAM,3142,-PT1H,P1M +T_MONTH,PWAN,3146,-PT1H,P1M +T_MONTH,PWAX,3144,-PT1H,P1M +T_MONTH,RR,3102,-PT18H,P1M +T_MONTH,RR_24X,3196,-PT18H,P1M +T_MONTH,RR_24XDT,3195,-PT18H,P1M +T_MONTH,RRA,3175,-PT18H,P1M +T_MONTH,RRA_6190,3175,-PT18H,P1M +T_MONTH,RRA_9120,3163,-PT18H,P1M +T_MONTH,RRID,117,PT6H, +T_MONTH,RRTA,3240,-PT6H,P1M +T_MONTH,SAM,3110,-PT18H,P1M +T_MONTH,SAN,3112,-PT18H,P1M +T_MONTH,SAX,3114,-PT18H,P1M +T_MONTH,SDM,3116,-PT18H,P1M +T_MONTH,SDN,3117,-PT18H,P1M +T_MONTH,SDX,3118,-PT18H,P1M +T_MONTH,SGN,3120,-PT1H,P1M +T_MONTH,SGX,3122,-PT1H,P1M +T_MONTH,SHM,3124,-PT1H,P1M +T_MONTH,TAM,3015,-PT1H,P1M +T_MONTH,TAM_K,3126,-PT1H,P1M +T_MONTH,TAMA,3170,-PT1H,P1M +T_MONTH,TAMA_6190,3170,-PT1H,P1M +T_MONTH,TAMA_9120,3159,-PT1H,P1M +T_MONTH,TAMRR,3301,-PT18H,P1M +T_MONTH,TAN,3018,-PT6H,P1M +T_MONTH,TANDT,3127,-PT6H,P1M +T_MONTH,TANM,3128,-PT6H,P1M +T_MONTH,TAX,3022,-PT6H,P1M +T_MONTH,TAXDT,3129,-PT6H,P1M +T_MONTH,TAXM,3130,-PT6H,P1M +T_MONTH,TD,3027,-PT1H,P1M +T_MONTH,TGNM,3131,-PT1H,P1M +T_MONTH,TGNN,3132,-PT1H,P1M +T_MONTH,TGNX,3133,-PT1H,P1M +T_MONTH,TWM,3029,-PT1H,P1M +T_MONTH,TWN,3030,-PT1H,P1M +T_MONTH,TWX,3031,-PT1H,P1M +T_MONTH,UM,3008,-PT1H,P1M +T_MONTH,UN,3007,-PT1H,P1M +T_MONTH,UX,3005,-PT1H,P1M +T_MONTH,VEKST,3135,-PT1H,P1M +T_MONTH,VP,3137,-PT1H,P1M +T_MONTH,VSUM,3139,-PT1H,P1M +T_MONTH,VVN,3003,-PT1H,P1M +T_MONTH,VVX,3001,-PT1H,P1M +T_HOMOGEN_DIURNAL,TAM,3009,-PT1H,P1D +T_HOMOGEN_DIURNAL,RR,3247,-PT18H,P1D +T_HOMOGEN_MONTH,TAM,3010,-PT1H,P1M +T_HOMOGEN_MONTH,RR,3099,-PT18H,P1M \ No newline at end of file diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go new file mode 100644 index 0000000..6a15574 --- /dev/null +++ b/migrations/kdvh/table.go @@ -0,0 +1,125 @@ +package kdvh + +import ( + "database/sql" + "errors" + "fmt" + "log/slog" + "migrate/lard" + "time" + + "github.com/rickb777/period" +) + +// In KDVH for each table name we usually have three separate tables: +// 1. A DATA table containing observation values; +// 2. A FLAG table containing quality control (QC) flags; +// 3. A ELEM table containing metadata about the validity of the timeseries. +// +// DATA and FLAG tables have the same schema: +// | dato | stnr | ... | +// where 'dato' is the timestamp of the observation, 'stnr' is the station +// where the observation was measured, and '...' is a varying number of columns +// each with different observations, where the column name is the 'elem_code' +// (e.g. for air temperature, 'ta'). +// +// TODO: are the timestamps UTC? Otherwise we probably need to convert them during import +// +// The ELEM tables have the following schema: +// | stnr | elem_code | fdato | tdato | table_name | flag_table_name | audit_dato + +// Table contains metadata on how to treat different tables in KDVH +type Table struct { + TableName string // Name of the DATA table + FlagTableName string // Name of the FLAG table + ElemTableName string // Name of the ELEM table + Path string // Directory name of where the dumped table is stored + dumpFunc DumpFunction // Function used to dump the KDVH table (found in `dump_functions.go`) + convFunc ConvertFunction // Function that converts KDVH obs to LARD obs (found in `import_functions.go`) + importUntil int // Import data only until the year specified by this field +} + +type DumpFunction func(path string, meta DumpMeta, conn *sql.DB) error +type DumpMeta struct { + element string + station string + dataTable string + flagTable string +} + +type ConvertFunction func(Obs) (lard.Obs, error) +type Obs struct { + *TimeseriesInfo + Obstime time.Time + Data string + Flags string +} + +// Convenience struct that holds information for a specific timeseries +type TimeseriesInfo struct { + station int32 + element string + offset period.Period + param StinfoParam + span Timespan + logstr string +} + +func (config *ImportConfig) NewTimeseriesInfo(table, element string, station int32) (*TimeseriesInfo, error) { + logstr := fmt.Sprintf("%v - %v - %v: ", table, station, element) + key := newKDVHKey(element, table, station) + + meta, ok := config.StinfoMap[key.Inner] + if !ok { + // TODO: should it fail here? How do we deal with data without metadata? + slog.Error(logstr + "Missing metadata in Stinfosys") + return nil, errors.New("") + } + + // No need to check for `!ok`, will default to 0 offset + offset := config.OffsetMap[key.Inner] + + // No need to check for `!ok`, timespan will be ignored if not in the map + span := config.KDVHMap[key] + + return &TimeseriesInfo{ + station: station, + element: element, + offset: offset, + param: meta, + span: span, + logstr: logstr, + }, nil +} + +// Creates default Table +func NewTable(data, flag, elem string) *Table { + return &Table{ + TableName: data, + FlagTableName: flag, + ElemTableName: elem, + Path: data + "_combined", // NOTE: '_combined' kept for backward compatibility with original scripts + dumpFunc: dumpDataAndFlags, + convFunc: makeDataPage, + } +} + +// Sets the `ImportUntil` field if the year is greater than 0 +func (t *Table) SetImport(year int) *Table { + if year > 0 { + t.importUntil = year + } + return t +} + +// Sets the function used to dump the Table +func (t *Table) SetDumpFunc(fn DumpFunction) *Table { + t.dumpFunc = fn + return t +} + +// Sets the function used to convert observations from the table to LARD observations +func (t *Table) SetConvFunc(fn ConvertFunction) *Table { + t.convFunc = fn + return t +} diff --git a/migrations/kdvh_test.go b/migrations/kdvh_test.go new file mode 100644 index 0000000..3f564e8 --- /dev/null +++ b/migrations/kdvh_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "fmt" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/joho/godotenv" + // "github.com/rickb777/period" + + "migrate/kdvh" +) + +const LARD_STRING string = "host=localhost user=postgres dbname=postgres password=postgres" + +func mockConfig(t *ImportTest) *kdvh.ImportConfig { + config := kdvh.ImportConfig{ + Tables: []string{t.table}, + Stations: []string{fmt.Sprint(t.station)}, + Elements: []string{t.elem}, + BaseDir: "./tests", + HasHeader: true, + Sep: ";", + } + + config.CacheMetadata() + return &config + +} + +type ImportTest struct { + table string + station int32 + elem string + expectedRows int64 +} + +func TestImportKDVH(t *testing.T) { + err := godotenv.Load() + if err != nil { + fmt.Println(err) + return + } + + // TODO: could also define a smaller version just for tests + db := kdvh.KDVH + + pool, err := pgxpool.New(context.TODO(), LARD_STRING) + if err != nil { + t.Log("Could not connect to Lard:", err) + } + defer pool.Close() + + testCases := []ImportTest{ + {table: "T_MDATA", station: 12345, elem: "TA", expectedRows: 2644}, + } + + for _, c := range testCases { + config := mockConfig(&c) + table, ok := db[c.table] + if !ok { + t.Fatal("Table does not exist in database") + } + + insertedRows := table.Import(pool, config) + if insertedRows != c.expectedRows { + t.Fail() + } + } +} diff --git a/migrations/lard/import.go b/migrations/lard/import.go new file mode 100644 index 0000000..408cc64 --- /dev/null +++ b/migrations/lard/import.go @@ -0,0 +1,92 @@ +package lard + +import ( + "context" + "fmt" + "log/slog" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +func InsertData(data DataInserter, pool *pgxpool.Pool, logStr string) (int64, error) { + size := data.Len() + count, err := pool.CopyFrom( + context.TODO(), + pgx.Identifier{"public", "data"}, + []string{"timeseries", "obstime", "obsvalue"}, + pgx.CopyFromSlice(size, func(i int) ([]any, error) { + return []any{ + data.ID(), + data.Obstime(i), + data.Data(i), + }, nil + }), + ) + if err != nil { + return count, err + } + + logStr += fmt.Sprintf("%v/%v data rows inserted", count, size) + if int(count) != size { + slog.Warn(logStr) + } else { + slog.Info(logStr) + } + return count, nil +} + +func InsertNonscalarData(data TextInserter, pool *pgxpool.Pool, logStr string) (int64, error) { + size := data.Len() + count, err := pool.CopyFrom( + context.TODO(), + pgx.Identifier{"public", "nonscalar_data"}, + []string{"timeseries", "obstime", "obsvalue"}, + pgx.CopyFromSlice(size, func(i int) ([]any, error) { + return []any{ + data.ID(), + data.Obstime(i), + data.Text(i), + }, nil + }), + ) + if err != nil { + return count, err + } + + logStr += fmt.Sprintf("%v/%v non-scalar data rows inserted", count, size) + if int(count) != size { + slog.Warn(logStr) + } else { + slog.Info(logStr) + } + return count, nil +} + +func InsertFlags(data FlagInserter, pool *pgxpool.Pool, logStr string) error { + size := data.Len() + count, err := pool.CopyFrom( + context.TODO(), + pgx.Identifier{"flags", "kdvh"}, + []string{"timeseries", "obstime", "controlinfo", "useinfo"}, + pgx.CopyFromSlice(size, func(i int) ([]any, error) { + return []any{ + data.ID(), + data.Obstime(i), + data.Controlinfo(i), + data.Useinfo(i), + }, nil + }), + ) + if err != nil { + return err + } + + logStr += fmt.Sprintf("%v/%v flag rows inserted", count, size) + if int(count) != size { + slog.Warn(logStr) + } else { + slog.Info(logStr) + } + return nil +} diff --git a/migrations/lard/main.go b/migrations/lard/main.go new file mode 100644 index 0000000..d245f5d --- /dev/null +++ b/migrations/lard/main.go @@ -0,0 +1,80 @@ +package lard + +import "time" + +// Timeseries in LARD have and ID and associated observations +type Timeseries struct { + id int32 + data []Obs +} + +func NewTimeseries(id int32, data []Obs) *Timeseries { + return &Timeseries{id, data} +} + +func (ts *Timeseries) Len() int { + return len(ts.data) +} + +func (ts *Timeseries) ID() int32 { + return ts.id +} + +func (ts *Timeseries) Obstime(i int) time.Time { + return ts.data[i].Obstime +} + +func (ts *Timeseries) Text(i int) string { + return *ts.data[i].Text +} + +func (ts *Timeseries) Data(i int) float32 { + return *ts.data[i].Data +} + +func (ts *Timeseries) Controlinfo(i int) string { + return ts.data[i].Controlinfo +} + +func (ts *Timeseries) Useinfo(i int) string { + return ts.data[i].Useinfo +} + +// Struct containg all the fields we want to save in LARD +type Obs struct { + // Time of observation + Obstime time.Time + // Observation data formatted as a single precision floating point number + Data *float32 + // Observation data that cannot be represented as a float, therefore stored as a string + Text *string + // Flag encoding quality control status + Controlinfo string + // Flag encoding quality control status + Useinfo string +} + +// TODO: I'm not sure I like the interface solution +type DataInserter interface { + Obstime(i int) time.Time + Data(i int) float32 + ID() int32 + Len() int +} + +type TextInserter interface { + Obstime(i int) time.Time + Text(i int) string + ID() int32 + Len() int +} + +// TODO: This maybe needs different implementation for each system +// i.e. insert to different tables and different columns +type FlagInserter interface { + ID() int32 + Obstime(i int) time.Time + Controlinfo(i int) string + Useinfo(i int) string + Len() int +} diff --git a/migrations/lard/timeseries.go b/migrations/lard/timeseries.go new file mode 100644 index 0000000..5629b3c --- /dev/null +++ b/migrations/lard/timeseries.go @@ -0,0 +1,62 @@ +package lard + +import ( + "context" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// Struct that mimics `labels.met` table structure +type Label struct { + StationID int32 + TypeID int32 + ParamID int32 + Sensor *int32 + Level *int32 +} + +func GetTimeseriesID(label Label, fromtime time.Time, pool *pgxpool.Pool) (tsid int32, err error) { + // Query LARD labels table + err = pool.QueryRow( + context.TODO(), + `SELECT timeseries FROM labels.met + WHERE station_id = $1 + AND param_id = $2 + AND type_id = $3 + AND (($4::int IS NULL AND lvl IS NULL) OR (lvl = $4)) + AND (($5::int IS NULL AND sensor IS NULL) OR (sensor = $5))`, + label.StationID, label.ParamID, label.TypeID, label.Level, label.Sensor).Scan(&tsid) + + // If timeseries exists, return its ID + if err == nil { + return tsid, nil + } + + // Otherwise insert new timeseries + transaction, err := pool.Begin(context.TODO()) + if err != nil { + return tsid, err + } + + err = transaction.QueryRow( + context.TODO(), + `INSERT INTO public.timeseries (fromtime) VALUES ($1) RETURNING id`, + fromtime, + ).Scan(&tsid) + if err != nil { + return tsid, err + } + + _, err = transaction.Exec( + context.TODO(), + `INSERT INTO labels.met (timeseries, station_id, param_id, type_id, lvl, sensor) + VALUES ($1, $2, $3, $4, $5, $6)`, + tsid, label.StationID, label.ParamID, label.TypeID, label.Level, label.Sensor) + if err != nil { + return tsid, err + } + + err = transaction.Commit(context.TODO()) + return tsid, err +} diff --git a/migrations/main.go b/migrations/main.go new file mode 100644 index 0000000..78ae62c --- /dev/null +++ b/migrations/main.go @@ -0,0 +1,43 @@ +package main + +import ( + "fmt" + "log" + + "github.com/jessevdk/go-flags" + "github.com/joho/godotenv" + + "migrate/kdvh" +) + +type CmdArgs struct { + KDVH kdvh.Cmd `command:"kdvh" description:"Perform KDVH migrations"` +} + +func main() { + log.SetFlags(log.LstdFlags | log.Lshortfile) + + // The following env variables are needed: + // 1. Dump + // - kdvh: "KDVH_PROXY_CONN" + // + // 2. Import + // - kdvh: "LARD_STRING", "STINFO_STRING", "KDVH_PROXY_CONN" + err := godotenv.Load() + if err != nil { + fmt.Println(err) + return + } + + // NOTE: go-flags calls the Execute method on the parsed subcommand + _, err = flags.Parse(&CmdArgs{}) + if err != nil { + if flagsErr, ok := err.(*flags.Error); ok { + if flagsErr.Type == flags.ErrHelp { + return + } + } + fmt.Println("Type './migrate -h' for help") + return + } +} diff --git a/migrations/tests/T_MDATA_combined/12345/TA.csv b/migrations/tests/T_MDATA_combined/12345/TA.csv new file mode 100644 index 0000000..dd6cb26 --- /dev/null +++ b/migrations/tests/T_MDATA_combined/12345/TA.csv @@ -0,0 +1,2645 @@ +2644 +2001-07-01_09:00:00;12.9;70000 +2001-07-01_10:00:00;13;70000 +2001-07-01_11:00:00;13;70000 +2001-07-01_12:00:00;13.1;70000 +2001-07-01_13:00:00;13.1;70000 +2001-07-01_14:00:00;13;70000 +2001-07-01_15:00:00;12.9;70000 +2001-07-01_16:00:00;12.8;70000 +2001-07-01_17:00:00;12.8;70000 +2001-07-01_18:00:00;12.7;70000 +2001-07-01_19:00:00;12.8;70000 +2001-07-01_20:00:00;12.6;70000 +2001-07-01_21:00:00;12.6;70000 +2001-07-01_22:00:00;12.6;70000 +2001-07-01_23:00:00;12.6;70000 +2001-07-02_00:00:00;12.5;70000 +2001-07-02_01:00:00;12.4;70000 +2001-07-02_02:00:00;12.4;70000 +2001-07-02_03:00:00;12.3;70000 +2001-07-02_04:00:00;12.3;70000 +2001-07-02_05:00:00;12.3;70000 +2001-07-02_06:00:00;12.4;70000 +2001-07-02_07:00:00;12.5;70000 +2001-07-02_08:00:00;12.6;70000 +2001-07-02_09:00:00;12.7;70000 +2001-07-02_10:00:00;12.9;70000 +2001-07-02_11:00:00;13;70000 +2001-07-02_12:00:00;13.2;70000 +2001-07-02_13:00:00;13.3;70000 +2001-07-02_14:00:00;13.3;70000 +2001-07-02_15:00:00;13.4;70000 +2001-07-02_16:00:00;13.3;70000 +2001-07-02_17:00:00;13.3;70000 +2001-07-02_18:00:00;13.2;70000 +2001-07-02_19:00:00;13.2;70000 +2001-07-02_20:00:00;13.1;70000 +2001-07-02_21:00:00;12.9;70000 +2001-07-02_22:00:00;12.9;70000 +2001-07-02_23:00:00;12.9;70000 +2001-07-03_00:00:00;12.8;58927 +2001-07-03_01:00:00;12.7;70000 +2001-07-03_02:00:00;12.6;70000 +2001-07-03_03:00:00;12.7;70000 +2001-07-03_04:00:00;12.5;70000 +2001-07-03_05:00:00;12.2;70000 +2001-07-03_06:00:00;12.3;70000 +2001-07-03_07:00:00;12.4;70000 +2001-07-03_08:00:00;12.5;70000 +2001-07-03_09:00:00;12.6;70000 +2001-07-03_10:00:00;12.6;70000 +2001-07-03_11:00:00;12.8;70000 +2001-07-03_12:00:00;12.8;70000 +2001-07-03_13:00:00;13;70000 +2001-07-03_14:00:00;13.1;70000 +2001-07-03_15:00:00;13.2;70000 +2001-07-03_16:00:00;13.2;70000 +2001-07-03_17:00:00;13.1;70000 +2001-07-03_18:00:00;13.1;70000 +2001-07-03_19:00:00;13.1;70000 +2001-07-03_20:00:00;12.8;70000 +2001-07-03_21:00:00;12.8;70000 +2001-07-03_22:00:00;12.8;70000 +2001-07-03_23:00:00;12.8;70000 +2001-07-04_00:00:00;12.6;70000 +2001-07-04_01:00:00;12.8;70000 +2001-07-04_02:00:00;12.3;70000 +2001-07-04_03:00:00;12.6;70000 +2001-07-04_04:00:00;12.5;70000 +2001-07-04_05:00:00;12.5;70000 +2001-07-04_06:00:00;12.5;70000 +2001-07-04_07:00:00;12.5;70000 +2001-07-04_08:00:00;12.4;70000 +2001-07-04_09:00:00;12.5;70000 +2001-07-04_10:00:00;12.6;70000 +2001-07-04_11:00:00;12.6;70000 +2001-07-04_12:00:00;12.6;58927 +2001-07-04_13:00:00;12.5;70000 +2001-07-04_14:00:00;12.6;70000 +2001-07-04_15:00:00;12.5;70000 +2001-07-04_16:00:00;12.6;70000 +2001-07-04_17:00:00;12.6;70000 +2001-07-04_18:00:00;12.6;70000 +2001-07-04_19:00:00;12.5;70000 +2001-07-04_20:00:00;12.5;70000 +2001-07-04_21:00:00;12.5;70000 +2001-07-04_22:00:00;12.4;70000 +2001-07-04_23:00:00;12.4;70000 +2001-07-05_00:00:00;12.5;70000 +2001-07-05_01:00:00;12.4;70000 +2001-07-05_02:00:00;12.1;70000 +2001-07-05_03:00:00;11.9;70000 +2001-07-05_04:00:00;12;70000 +2001-07-05_05:00:00;12;70000 +2001-07-05_06:00:00;12.1;70000 +2001-07-05_07:00:00;12.3;70000 +2001-07-05_08:00:00;12.6;70000 +2001-07-05_09:00:00;12.9;70000 +2001-07-05_10:00:00;13;70000 +2001-07-05_11:00:00;13.2;70000 +2001-07-05_12:00:00;13.5;70000 +2001-07-05_13:00:00;13.8;70000 +2001-07-05_14:00:00;13.9;70000 +2001-07-05_15:00:00;13.4;70000 +2001-07-05_16:00:00;13.9;70000 +2001-07-05_17:00:00;13.8;70000 +2001-07-05_18:00:00;13.7;70000 +2001-07-05_19:00:00;13.6;70000 +2001-07-05_20:00:00;13.5;70000 +2001-07-05_21:00:00;13.3;70000 +2001-07-05_22:00:00;13.2;70000 +2001-07-05_23:00:00;13.1;70000 +2001-07-06_00:00:00;13.1;70000 +2001-07-06_01:00:00;13;70000 +2001-07-06_02:00:00;12.9;70000 +2001-07-06_03:00:00;12.8;70000 +2001-07-06_04:00:00;12.9;58927 +2001-07-06_05:00:00;12.9;70000 +2001-07-06_06:00:00;13.2;70000 +2001-07-06_07:00:00;13.2;70000 +2001-07-06_08:00:00;13.3;70000 +2001-07-06_09:00:00;13.8;70000 +2001-07-06_10:00:00;14.3;70000 +2001-07-06_11:00:00;14.7;70000 +2001-07-06_12:00:00;15.8;70000 +2001-07-06_13:00:00;14.9;70000 +2001-07-06_14:00:00;14.6;70000 +2001-07-06_15:00:00;14.7;70000 +2001-07-06_16:00:00;14.6;70000 +2001-07-06_17:00:00;15.5;70000 +2001-07-06_18:00:00;16.6;70000 +2001-07-06_19:00:00;15.5;70000 +2001-07-06_20:00:00;14.8;70000 +2001-07-06_21:00:00;14.7;70000 +2001-07-06_22:00:00;16.2;70000 +2001-07-06_23:00:00;15.6;70000 +2001-07-07_00:00:00;15.1;70000 +2001-07-07_01:00:00;14.4;70000 +2001-07-07_02:00:00;13.8;70000 +2001-07-07_03:00:00;13.2;70000 +2001-07-07_04:00:00;13.3;70000 +2001-07-07_05:00:00;13.6;70000 +2001-07-07_06:00:00;14;70000 +2001-07-07_07:00:00;14.1;70000 +2001-07-07_08:00:00;14.1;70000 +2001-07-07_09:00:00;14.3;70000 +2001-07-07_10:00:00;14.4;70000 +2001-07-07_11:00:00;14.5;70000 +2001-07-07_12:00:00;14.6;70000 +2001-07-07_13:00:00;14.9;70000 +2001-07-07_14:00:00;15;70000 +2001-07-07_15:00:00;14.9;70000 +2001-07-07_16:00:00;15;70000 +2001-07-07_17:00:00;14.9;70000 +2001-07-07_18:00:00;14.9;70000 +2001-07-07_19:00:00;14.8;70000 +2001-07-07_20:00:00;14.8;70000 +2001-07-07_21:00:00;15;70000 +2001-07-07_22:00:00;15;70000 +2001-07-07_23:00:00;15.3;70000 +2001-07-08_00:00:00;14.9;70000 +2001-07-08_01:00:00;14.6;70000 +2001-07-08_02:00:00;14.5;70000 +2001-07-08_03:00:00;14.4;70000 +2001-07-08_04:00:00;14.4;70000 +2001-07-08_05:00:00;14.7;70000 +2001-07-08_06:00:00;14.6;70000 +2001-07-08_07:00:00;14.3;70000 +2001-07-08_08:00:00;14.5;70000 +2001-07-08_09:00:00;14.5;70000 +2001-07-08_10:00:00;14.5;70000 +2001-07-08_11:00:00;15.1;70000 +2001-07-08_12:00:00;15.2;70000 +2001-07-08_13:00:00;15.5;70000 +2001-07-08_14:00:00;14.6;70000 +2001-07-08_15:00:00;16.9;78947 +2001-07-08_16:00:00;17.1;78947 +2001-07-08_17:00:00;16.9;78947 +2001-07-08_18:00:00;16;78947 +2001-07-08_19:00:00;15.5;78947 +2001-07-08_20:00:00;15.1;78947 +2001-07-08_21:00:00;14.9;78947 +2001-07-08_22:00:00;14.6;78947 +2001-07-08_23:00:00;14.3;78947 +2001-07-09_00:00:00;14.1;78947 +2001-07-09_01:00:00;14.2;78947 +2001-07-09_02:00:00;14.3;78947 +2001-07-09_03:00:00;14;78947 +2001-07-09_04:00:00;14;78947 +2001-07-09_05:00:00;14.2;78947 +2001-07-09_06:00:00;14;78947 +2001-07-09_07:00:00;14.6;78947 +2001-07-09_08:00:00;14.5;78947 +2001-07-09_09:00:00;15.3;78947 +2001-07-09_10:00:00;16.3;78947 +2001-07-09_11:00:00;15.1;78947 +2001-07-09_12:00:00;16.2;78947 +2001-07-09_13:00:00;15.2;78947 +2001-07-09_14:00:00;15.6;78947 +2001-07-09_15:00:00;15.4;78947 +2001-07-09_16:00:00;15.6;78947 +2001-07-09_17:00:00;15;78947 +2001-07-09_18:00:00;14.2;78947 +2001-07-09_19:00:00;13.7;78947 +2001-07-09_20:00:00;13.5;78947 +2001-07-09_21:00:00;13.2;78947 +2001-07-09_22:00:00;13.4;78947 +2001-07-09_23:00:00;13.5;78947 +2001-07-10_00:00:00;12.8;78947 +2001-07-10_01:00:00;12.9;78947 +2001-07-10_02:00:00;12.9;78947 +2001-07-10_03:00:00;13.2;78947 +2001-07-10_04:00:00;13.1;78947 +2001-07-10_05:00:00;13.3;78947 +2001-07-10_06:00:00;13.8;78947 +2001-07-10_07:00:00;13.9;78947 +2001-07-10_08:00:00;14.3;78947 +2001-07-10_09:00:00;14.7;78947 +2001-07-10_10:00:00;15.1;78947 +2001-07-10_11:00:00;15.3;78947 +2001-07-10_12:00:00;15.3;78947 +2001-07-10_13:00:00;16;78947 +2001-07-10_14:00:00;16.1;78947 +2001-07-10_15:00:00;15.6;78947 +2001-07-10_16:00:00;15;78947 +2001-07-10_17:00:00;14.5;78947 +2001-07-10_18:00:00;14.3;78947 +2001-07-10_19:00:00;13.5;78947 +2001-07-10_20:00:00;13.3;78947 +2001-07-10_21:00:00;12.9;78947 +2001-07-10_22:00:00;12.2;78947 +2001-07-10_23:00:00;11.9;78947 +2001-07-11_00:00:00;13;78947 +2001-07-11_01:00:00;12.7;78947 +2001-07-11_02:00:00;12.7;78947 +2001-07-11_03:00:00;12.6;78947 +2001-07-11_04:00:00;12.7;78947 +2001-07-11_05:00:00;12.8;78947 +2001-07-11_06:00:00;13.7;78947 +2001-07-11_07:00:00;13.7;78947 +2001-07-11_08:00:00;13.7;78947 +2001-07-11_09:00:00;14.4;78947 +2001-07-11_10:00:00;14.7;78947 +2001-07-11_11:00:00;15.2;78947 +2001-07-11_12:00:00;15.3;78947 +2001-07-11_13:00:00;13.7;78947 +2001-07-11_14:00:00;14.5;78947 +2001-07-11_15:00:00;15;78947 +2001-07-11_16:00:00;13.2;78947 +2001-07-11_17:00:00;12.9;78947 +2001-07-11_18:00:00;12.5;78947 +2001-07-11_19:00:00;12.3;78947 +2001-07-11_20:00:00;12.4;78947 +2001-07-11_21:00:00;12.4;78947 +2001-07-11_22:00:00;12.4;78947 +2001-07-11_23:00:00;12.5;78947 +2001-07-12_00:00:00;12;78947 +2001-07-12_01:00:00;12.1;78947 +2001-07-12_02:00:00;12.2;78947 +2001-07-12_03:00:00;12.2;78947 +2001-07-12_04:00:00;12.3;78947 +2001-07-12_05:00:00;12.3;78947 +2001-07-12_06:00:00;12.1;78947 +2001-07-12_07:00:00;12.4;78947 +2001-07-12_08:00:00;13.5;78947 +2001-07-12_09:00:00;13.1;78947 +2001-07-12_10:00:00;14;78947 +2001-07-12_11:00:00;15.2;78947 +2001-07-12_12:00:00;14.3;78947 +2001-07-12_13:00:00;13.9;78947 +2001-07-12_14:00:00;14.3;78947 +2001-07-12_15:00:00;14;78947 +2001-07-12_16:00:00;13.9;78947 +2001-07-12_17:00:00;13.7;78947 +2001-07-12_18:00:00;13.5;78947 +2001-07-12_19:00:00;13.1;78947 +2001-07-12_20:00:00;12.6;78947 +2001-07-12_21:00:00;12.2;78947 +2001-07-12_22:00:00;11.9;78947 +2001-07-12_23:00:00;11.9;78947 +2001-07-13_00:00:00;11.7;78947 +2001-07-13_01:00:00;11.5;78947 +2001-07-13_02:00:00;11.3;78947 +2001-07-13_03:00:00;11.1;78947 +2001-07-13_04:00:00;11.3;78947 +2001-07-13_05:00:00;12;78947 +2001-07-13_06:00:00;13.4;78947 +2001-07-13_08:00:00;15.5;78947 +2001-07-13_09:00:00;16.5;78947 +2001-07-13_10:00:00;17.4;78947 +2001-07-13_11:00:00;17.7;78947 +2001-07-13_12:00:00;17.3;78947 +2001-07-13_13:00:00;17.3;78947 +2001-07-13_14:00:00;17.3;78947 +2001-07-13_15:00:00;17;78947 +2001-07-13_16:00:00;16.4;78947 +2001-07-13_17:00:00;15.5;78947 +2001-07-13_18:00:00;14.9;78947 +2001-07-13_19:00:00;14.1;78947 +2001-07-13_20:00:00;13.2;78947 +2001-07-13_21:00:00;12.3;78947 +2001-07-13_23:00:00;11.5;78947 +2001-07-14_00:00:00;11.2;78947 +2001-07-14_01:00:00;10.9;78947 +2001-07-14_02:00:00;10.7;78947 +2001-07-14_03:00:00;10.6;78947 +2001-07-14_04:00:00;10.7;78947 +2001-07-14_05:00:00;11.6;78947 +2001-07-14_06:00:00;13.1;78947 +2001-07-14_07:00:00;14.5;78947 +2001-07-14_08:00:00;15.9;78947 +2001-07-14_09:00:00;17.2;78947 +2001-07-14_10:00:00;18.3;78947 +2001-07-14_11:00:00;18.8;78947 +2001-07-14_12:00:00;18.5;78947 +2001-07-14_13:00:00;17.9;78947 +2001-07-14_14:00:00;17.4;78947 +2001-07-14_15:00:00;17.1;78947 +2001-07-14_16:00:00;17;78947 +2001-07-14_17:00:00;16.6;78947 +2001-07-14_18:00:00;16.4;78947 +2001-07-14_19:00:00;15.4;78947 +2001-07-14_20:00:00;14.6;78947 +2001-07-14_21:00:00;13.7;78947 +2001-07-14_23:00:00;12.9;78947 +2001-07-15_00:00:00;12.7;78947 +2001-07-15_01:00:00;12.5;78947 +2001-07-15_02:00:00;12.5;78947 +2001-07-15_03:00:00;12.4;78947 +2001-07-15_05:00:00;12.8;78947 +2001-07-15_06:00:00;13.6;78947 +2001-07-15_10:00:00;16.7;78947 +2001-07-15_11:00:00;16.6;78947 +2001-07-15_13:00:00;16.1;78947 +2001-07-15_15:00:00;15.9;78947 +2001-07-15_16:00:00;15.5;78947 +2001-07-15_17:00:00;15.1;78947 +2001-07-15_18:00:00;14.7;78947 +2001-07-15_19:00:00;14.2;78947 +2001-07-15_20:00:00;13.5;78947 +2001-07-15_21:00:00;12.4;78947 +2001-07-15_22:00:00;11.5;78947 +2001-07-15_23:00:00;10.9;78947 +2001-07-16_00:00:00;10.1;78947 +2001-07-16_01:00:00;10.6;78947 +2001-07-16_02:00:00;11.8;78947 +2001-07-16_03:00:00;12.7;78947 +2001-07-16_04:00:00;13.2;78947 +2001-07-16_05:00:00;13.6;78947 +2001-07-16_06:00:00;14;78947 +2001-07-16_07:00:00;15.2;78947 +2001-07-16_08:00:00;16.6;78947 +2001-07-16_09:00:00;17.7;78947 +2001-07-16_10:00:00;18.8;78947 +2001-07-16_11:00:00;19.5;78947 +2001-07-16_12:00:00;20.5;78947 +2001-07-16_13:00:00;20.6;78947 +2001-07-16_14:00:00;20.7;78947 +2001-07-16_15:00:00;19.3;78947 +2001-07-16_16:00:00;19.4;78947 +2001-07-16_17:00:00;18.5;78947 +2001-07-16_18:00:00;17.1;78947 +2001-07-16_19:00:00;15.8;78947 +2001-07-16_20:00:00;15.1;78947 +2001-07-16_21:00:00;15.1;78947 +2001-07-16_22:00:00;15.5;78947 +2001-07-16_23:00:00;15.1;78947 +2001-07-17_00:00:00;15.3;78947 +2001-07-17_01:00:00;15.1;78947 +2001-07-17_02:00:00;15.3;78947 +2001-07-17_03:00:00;15.3;78947 +2001-07-17_04:00:00;15.2;78947 +2001-07-17_05:00:00;15.1;78947 +2001-07-17_06:00:00;14.8;78947 +2001-07-17_07:00:00;14.8;78947 +2001-07-17_08:00:00;14.8;78947 +2001-07-17_09:00:00;15;78947 +2001-07-17_10:00:00;15.2;78947 +2001-07-17_11:00:00;15.2;78947 +2001-07-17_12:00:00;15.3;78947 +2001-07-17_13:00:00;15.4;78947 +2001-07-17_14:00:00;15.5;78947 +2001-07-17_15:00:00;15.5;78947 +2001-07-17_16:00:00;16.4;78947 +2001-07-17_17:00:00;16.2;78947 +2001-07-17_18:00:00;14.6;78947 +2001-07-17_19:00:00;14.7;78947 +2001-07-17_20:00:00;14.8;78947 +2001-07-17_21:00:00;14.1;78947 +2001-07-17_22:00:00;13.7;78947 +2001-07-17_23:00:00;13.7;78947 +2001-07-18_00:00:00;13.4;78947 +2001-07-18_01:00:00;14;78947 +2001-07-18_02:00:00;14.3;78947 +2001-07-18_03:00:00;13.9;78947 +2001-07-18_04:00:00;13.4;78947 +2001-07-18_05:00:00;13.7;78947 +2001-07-18_06:00:00;16.2;78947 +2001-07-18_07:00:00;17.4;78947 +2001-07-18_08:00:00;18.3;78947 +2001-07-18_11:00:00;20.1;78947 +2001-07-18_12:00:00;19.7;78947 +2001-07-18_13:00:00;18.6;78947 +2001-07-18_14:00:00;19.3;78947 +2001-07-18_15:00:00;18.3;78947 +2001-07-18_16:00:00;16.6;78947 +2001-07-18_17:00:00;17;78947 +2001-07-18_18:00:00;16.9;78947 +2001-07-18_19:00:00;16.5;78947 +2001-07-18_20:00:00;15.1;78947 +2001-07-18_21:00:00;14.7;78947 +2001-07-18_22:00:00;14.3;78947 +2001-07-18_23:00:00;14;78947 +2001-07-19_00:00:00;14.1;78947 +2001-07-19_01:00:00;14;78947 +2001-07-19_02:00:00;14.1;78947 +2001-07-19_03:00:00;14.2;78947 +2001-07-19_04:00:00;13.9;78947 +2001-07-19_05:00:00;13.8;78947 +2001-07-19_06:00:00;14.7;78947 +2001-07-19_07:00:00;15.7;78947 +2001-07-19_08:00:00;15.7;78947 +2001-07-19_09:00:00;17.2;78947 +2001-07-19_10:00:00;18.4;78947 +2001-07-19_11:00:00;18.3;78947 +2001-07-19_12:00:00;16.1;78947 +2001-07-19_13:00:00;15.5;78947 +2001-07-19_14:00:00;16;78947 +2001-07-19_15:00:00;16.5;78947 +2001-07-19_16:00:00;15.3;78947 +2001-07-19_17:00:00;15.4;78947 +2001-07-19_18:00:00;15;78947 +2001-07-19_19:00:00;14.4;78947 +2001-07-19_20:00:00;14.2;78947 +2001-07-19_21:00:00;14.1;78947 +2001-07-19_22:00:00;14;78947 +2001-07-19_23:00:00;13.6;78947 +2001-07-20_00:00:00;13.8;78947 +2001-07-20_01:00:00;13.8;78947 +2001-07-20_02:00:00;13.6;78947 +2001-07-20_03:00:00;13.7;78947 +2001-07-20_04:00:00;13.6;78947 +2001-07-20_05:00:00;14;78947 +2001-07-20_06:00:00;15.1;78947 +2001-07-20_07:00:00;15.6;78947 +2001-07-20_08:00:00;15.4;78947 +2001-07-20_09:00:00;16;78947 +2001-07-20_10:00:00;16.6;78947 +2001-07-20_11:00:00;17.1;78947 +2001-07-20_12:00:00;17.3;78947 +2001-07-20_13:00:00;17;78947 +2001-07-20_14:00:00;16.5;78947 +2001-07-20_15:00:00;16.4;78947 +2001-07-20_16:00:00;15.7;78947 +2001-07-20_17:00:00;14.9;78947 +2001-07-20_18:00:00;14.4;78947 +2001-07-20_19:00:00;14.1;78947 +2001-07-20_20:00:00;13.8;78947 +2001-07-20_21:00:00;13.7;78947 +2001-07-20_22:00:00;13.5;78947 +2001-07-20_23:00:00;13.4;78947 +2001-07-21_00:00:00;13.4;78947 +2001-07-21_01:00:00;13.4;78947 +2001-07-21_02:00:00;13.4;78947 +2001-07-21_03:00:00;13.3;78947 +2001-07-21_04:00:00;13.2;78947 +2001-07-21_05:00:00;13.2;78947 +2001-07-21_06:00:00;13.2;78947 +2001-07-21_07:00:00;13.4;78947 +2001-07-21_08:00:00;14;78947 +2001-07-21_09:00:00;14.6;78947 +2001-07-21_10:00:00;15.2;78947 +2001-07-21_11:00:00;15.4;78947 +2001-07-21_12:00:00;16.5;78947 +2001-07-21_13:00:00;16.2;78947 +2001-07-21_14:00:00;15.8;78947 +2001-07-21_15:00:00;15.4;78947 +2001-07-21_16:00:00;15.1;78947 +2001-07-21_17:00:00;14.7;78947 +2001-07-21_18:00:00;13.9;78947 +2001-07-21_19:00:00;13.4;78947 +2001-07-21_20:00:00;13;78947 +2001-07-21_21:00:00;12.8;78947 +2001-07-21_22:00:00;12.8;78947 +2001-07-21_23:00:00;12.9;78947 +2001-07-22_00:00:00;13;78947 +2001-07-22_01:00:00;13.1;78947 +2001-07-22_02:00:00;13.2;78947 +2001-07-22_03:00:00;13.2;78947 +2001-07-22_04:00:00;13.3;78947 +2001-07-22_05:00:00;13.5;78947 +2001-07-22_06:00:00;14;78947 +2001-07-22_07:00:00;14.7;78947 +2001-07-22_08:00:00;15.5;78947 +2001-07-22_09:00:00;15.8;78947 +2001-07-22_10:00:00;16.7;78947 +2001-07-22_11:00:00;17;78947 +2001-07-22_12:00:00;16.5;78947 +2001-07-22_13:00:00;17.4;78947 +2001-07-22_14:00:00;17.3;78947 +2001-07-22_15:00:00;17.5;78947 +2001-07-22_16:00:00;17;78947 +2001-07-22_17:00:00;16.7;78947 +2001-07-22_18:00:00;15.8;78947 +2001-07-22_19:00:00;15.4;78947 +2001-07-22_20:00:00;15.5;78947 +2001-07-22_21:00:00;15.2;78947 +2001-07-22_22:00:00;15.2;78947 +2001-07-22_23:00:00;15.1;78947 +2001-07-23_00:00:00;14.9;78947 +2001-07-23_01:00:00;14.8;78947 +2001-07-23_02:00:00;14.8;78947 +2001-07-23_03:00:00;14.6;78947 +2001-07-23_04:00:00;14.5;78947 +2001-07-23_05:00:00;14.7;78947 +2001-07-23_06:00:00;15.1;78947 +2001-07-23_07:00:00;15.5;78947 +2001-07-23_08:00:00;15.7;78947 +2001-07-23_09:00:00;16.6;78947 +2001-07-23_10:00:00;18.7;78947 +2001-07-23_11:00:00;19.6;78947 +2001-07-23_12:00:00;16.8;78947 +2001-07-23_13:00:00;17.3;78947 +2001-07-23_14:00:00;17.6;78947 +2001-07-23_15:00:00;19.6;78947 +2001-07-23_16:00:00;17.4;78947 +2001-07-23_17:00:00;17.5;78947 +2001-07-23_18:00:00;16.7;78947 +2001-07-23_19:00:00;16.1;78947 +2001-07-23_20:00:00;15.4;78947 +2001-07-23_21:00:00;15.2;78947 +2001-07-23_22:00:00;14.9;78947 +2001-07-23_23:00:00;15.2;78947 +2001-07-24_00:00:00;15.4;78947 +2001-07-24_01:00:00;15;78947 +2001-07-24_02:00:00;14.1;78947 +2001-07-24_03:00:00;14.1;78947 +2001-07-24_04:00:00;14.5;78947 +2001-07-24_05:00:00;15;78947 +2001-07-24_06:00:00;15.6;78947 +2001-07-24_07:00:00;16.1;78947 +2001-07-24_08:00:00;17.4;78947 +2001-07-24_09:00:00;18.6;78947 +2001-07-24_10:00:00;19.9;78947 +2001-07-24_11:00:00;19.9;78947 +2001-07-24_12:00:00;18.2;78947 +2001-07-24_13:00:00;17.6;78947 +2001-07-24_14:00:00;17.9;78947 +2001-07-24_15:00:00;18.4;78947 +2001-07-24_16:00:00;17.9;78947 +2001-07-24_17:00:00;17.6;78947 +2001-07-24_18:00:00;17.3;78947 +2001-07-24_19:00:00;16.4;78947 +2001-07-24_20:00:00;15.4;78947 +2001-07-24_21:00:00;15.1;78947 +2001-07-24_22:00:00;15.2;78947 +2001-07-24_23:00:00;15.1;78947 +2001-07-25_00:00:00;15.1;78947 +2001-07-25_01:00:00;15;78947 +2001-07-25_02:00:00;14.9;78947 +2001-07-25_03:00:00;15;78947 +2001-07-25_04:00:00;14.9;78947 +2001-07-25_05:00:00;15.2;78947 +2001-07-25_06:00:00;15.7;78947 +2001-07-25_07:00:00;16.3;78947 +2001-07-25_09:00:00;17.7;78947 +2001-07-25_10:00:00;18.6;78947 +2001-07-25_11:00:00;19.1;78947 +2001-07-25_12:00:00;18.1;78947 +2001-07-25_13:00:00;18.7;78947 +2001-07-25_14:00:00;18.8;78947 +2001-07-25_15:00:00;18.9;78947 +2001-07-25_16:00:00;18.7;78947 +2001-07-25_17:00:00;17.8;78947 +2001-07-25_18:00:00;16.9;78947 +2001-07-25_19:00:00;16.4;78947 +2001-07-25_20:00:00;16;78947 +2001-07-25_21:00:00;15.7;78947 +2001-07-25_22:00:00;15.4;78947 +2001-07-25_23:00:00;15.1;78947 +2001-07-26_00:00:00;14.7;78947 +2001-07-26_01:00:00;14.7;78947 +2001-07-26_02:00:00;14.6;78947 +2001-07-26_03:00:00;14.6;78947 +2001-07-26_04:00:00;14.7;78947 +2001-07-26_05:00:00;14.7;78947 +2001-07-26_06:00:00;14.6;78947 +2001-07-26_07:00:00;14.7;78947 +2001-07-26_09:00:00;15.2;78947 +2001-07-26_10:00:00;15.7;78947 +2001-07-26_11:00:00;15.8;78947 +2001-07-26_12:00:00;14.8;78947 +2001-07-26_13:00:00;14.9;78947 +2001-07-26_14:00:00;15.4;78947 +2001-07-26_15:00:00;15.7;78947 +2001-07-26_16:00:00;15.5;78947 +2001-07-26_17:00:00;15.3;78947 +2001-07-26_18:00:00;15.2;78947 +2001-07-26_19:00:00;14.6;78947 +2001-07-26_20:00:00;13.9;78947 +2001-07-26_21:00:00;13.3;78947 +2001-07-26_22:00:00;13.2;78947 +2001-07-26_23:00:00;13.2;78947 +2001-07-27_01:00:00;13.4;78947 +2001-07-27_02:00:00;13.4;78947 +2001-07-27_03:00:00;13.2;78947 +2001-07-27_04:00:00;13;78947 +2001-07-27_05:00:00;13.1;78947 +2001-07-27_06:00:00;13.8;78947 +2001-07-27_07:00:00;14.4;78947 +2001-07-27_08:00:00;15.1;78947 +2001-07-27_09:00:00;16;78947 +2001-07-27_10:00:00;16.7;78947 +2001-07-27_11:00:00;16.7;78947 +2001-07-27_12:00:00;16.8;78947 +2001-07-27_13:00:00;16.5;78947 +2001-07-27_14:00:00;16.1;78947 +2001-07-27_15:00:00;15.6;78947 +2001-07-27_16:00:00;15;78947 +2001-07-27_17:00:00;14.5;78947 +2001-07-27_18:00:00;14.1;78947 +2001-07-27_19:00:00;13.5;78947 +2001-07-27_20:00:00;12.9;78947 +2001-07-27_21:00:00;12.6;78947 +2001-07-27_22:00:00;12.5;78947 +2001-07-27_23:00:00;12.4;78947 +2001-07-28_00:00:00;12.6;78947 +2001-07-28_01:00:00;12.8;78947 +2001-07-28_02:00:00;12.9;78947 +2001-07-28_03:00:00;12.9;78947 +2001-07-28_04:00:00;13;78947 +2001-07-28_05:00:00;13.2;78947 +2001-07-28_06:00:00;13.4;78947 +2001-07-28_07:00:00;13.7;78947 +2001-07-28_09:00:00;14.1;78947 +2001-07-28_10:00:00;14.4;78947 +2001-07-28_11:00:00;14.7;78947 +2001-07-28_12:00:00;15.4;78947 +2001-07-28_13:00:00;15.3;78947 +2001-07-28_14:00:00;14.8;78947 +2001-07-28_15:00:00;14.5;78947 +2001-07-28_16:00:00;14.3;78947 +2001-07-28_17:00:00;14.1;78947 +2001-07-28_18:00:00;13.4;78947 +2001-07-28_19:00:00;12.8;78947 +2001-07-28_20:00:00;12.4;78947 +2001-07-28_21:00:00;12.3;78947 +2001-07-28_22:00:00;12.5;78947 +2001-07-28_23:00:00;12.7;78947 +2001-07-29_00:00:00;12.4;78947 +2001-07-29_01:00:00;12.3;78947 +2001-07-29_02:00:00;12.1;78947 +2001-07-29_03:00:00;12;78947 +2001-07-29_04:00:00;12.1;78947 +2001-07-29_05:00:00;12.3;78947 +2001-07-29_06:00:00;12.9;78947 +2001-07-29_07:00:00;13.5;78947 +2001-07-29_09:00:00;14.6;78947 +2001-07-29_10:00:00;15;78947 +2001-07-29_11:00:00;15.2;78947 +2001-07-29_12:00:00;15.5;78947 +2001-07-29_13:00:00;15.4;78947 +2001-07-29_14:00:00;15.2;78947 +2001-07-29_15:00:00;14.8;78947 +2001-07-29_16:00:00;14.4;78947 +2001-07-29_17:00:00;14;78947 +2001-07-29_18:00:00;13.6;78947 +2001-07-29_19:00:00;13.1;78947 +2001-07-29_20:00:00;12.7;78947 +2001-07-29_21:00:00;12.5;78947 +2001-07-29_22:00:00;12.4;78947 +2001-07-29_23:00:00;12.3;78947 +2001-07-30_00:00:00;12.3;78947 +2001-07-30_01:00:00;12.2;78947 +2001-07-30_02:00:00;12.3;78947 +2001-07-30_03:00:00;12.3;78947 +2001-07-30_05:00:00;12.6;78947 +2001-07-30_06:00:00;13.4;78947 +2001-07-30_07:00:00;14;78947 +2001-07-30_08:00:00;14.8;78947 +2001-07-30_09:00:00;15.4;78947 +2001-07-30_10:00:00;15.8;78947 +2001-07-30_11:00:00;16.1;78947 +2001-07-30_12:00:00;16.6;78947 +2001-07-30_13:00:00;16.5;78947 +2001-07-30_14:00:00;16.3;78947 +2001-07-30_15:00:00;16;78947 +2001-07-30_16:00:00;15.6;78947 +2001-07-30_17:00:00;15;78947 +2001-07-30_18:00:00;14.4;78947 +2001-07-30_19:00:00;13.9;78947 +2001-07-30_20:00:00;13.3;78947 +2001-07-30_21:00:00;12.8;78947 +2001-07-30_22:00:00;12.5;78947 +2001-07-30_23:00:00;12.6;78947 +2001-07-31_00:00:00;13;78947 +2001-07-31_01:00:00;13.1;78947 +2001-07-31_02:00:00;13.3;78947 +2001-07-31_03:00:00;13.3;78947 +2001-07-31_04:00:00;13.2;78947 +2001-07-31_05:00:00;13.5;78947 +2001-07-31_06:00:00;14.2;78947 +2001-07-31_07:00:00;14.6;78947 +2001-07-31_08:00:00;15;78947 +2001-07-31_09:00:00;15.8;78947 +2001-07-31_10:00:00;16.4;78947 +2001-07-31_11:00:00;16.8;78947 +2001-07-31_12:00:00;17.1;78947 +2001-07-31_13:00:00;17.1;78947 +2001-07-31_14:00:00;16.6;78947 +2001-07-31_15:00:00;16.2;78947 +2001-07-31_16:00:00;15.8;78947 +2001-07-31_17:00:00;15.4;78947 +2001-07-31_18:00:00;14.8;78947 +2001-07-31_19:00:00;14;78947 +2001-07-31_20:00:00;13.2;78947 +2001-07-31_21:00:00;12.6;78947 +2001-07-31_22:00:00;12;78947 +2001-07-31_23:00:00;12.3;78947 +2001-08-01_00:00:00;13.1;78947 +2001-08-01_01:00:00;13.2;78947 +2001-08-01_02:00:00;13.3;78947 +2001-08-01_03:00:00;13.3;78947 +2001-08-01_04:00:00;13.3;78947 +2001-08-01_05:00:00;13.6;78947 +2001-08-01_06:00:00;14.4;78947 +2001-08-01_07:00:00;14.9;78947 +2001-08-01_09:00:00;16.2;78947 +2001-08-01_10:00:00;16.6;78947 +2001-08-01_11:00:00;17;78947 +2001-08-01_12:00:00;17.2;78947 +2001-08-01_13:00:00;17.4;78947 +2001-08-01_14:00:00;17.4;78947 +2001-08-01_15:00:00;17.1;78947 +2001-08-01_16:00:00;16.7;78947 +2001-08-01_17:00:00;16.2;78947 +2001-08-01_18:00:00;15.4;78947 +2001-08-01_19:00:00;14.6;78947 +2001-08-01_20:00:00;13.7;78947 +2001-08-01_21:00:00;13;78947 +2001-08-01_22:00:00;12.5;78947 +2001-08-01_23:00:00;12.3;78947 +2001-08-02_00:00:00;12.6;78947 +2001-08-02_01:00:00;12.6;78947 +2001-08-02_02:00:00;12.6;78947 +2001-08-02_03:00:00;12.8;78947 +2001-08-02_04:00:00;13.2;78947 +2001-08-02_05:00:00;14;78947 +2001-08-02_06:00:00;15.6;78947 +2001-08-02_07:00:00;16.1;78947 +2001-08-02_09:00:00;17;78947 +2001-08-02_10:00:00;17;78947 +2001-08-02_11:00:00;17.1;78947 +2001-08-02_12:00:00;17.2;78947 +2001-08-02_13:00:00;18.2;78947 +2001-08-02_14:00:00;19.2;78947 +2001-08-02_15:00:00;19.4;78947 +2001-08-02_16:00:00;19.8;78947 +2001-08-02_17:00:00;20;78947 +2001-08-02_18:00:00;18.4;78947 +2001-08-02_19:00:00;17.8;78947 +2001-08-02_20:00:00;16.6;78947 +2001-08-02_21:00:00;15.6;78947 +2001-08-02_22:00:00;15.6;78947 +2001-08-02_23:00:00;15.7;78947 +2001-08-03_00:00:00;15.6;78947 +2001-08-03_01:00:00;16;78947 +2001-08-03_02:00:00;16.5;78947 +2001-08-03_03:00:00;16.7;78947 +2001-08-03_04:00:00;16.3;78947 +2001-08-03_05:00:00;16.5;78947 +2001-08-03_06:00:00;17.5;78947 +2001-08-03_07:00:00;17;78947 +2001-08-03_09:00:00;17.4;78947 +2001-08-03_10:00:00;18.7;78947 +2001-08-03_11:00:00;20.4;78947 +2001-08-03_12:00:00;21.9;78947 +2001-08-03_13:00:00;21.3;78947 +2001-08-03_14:00:00;21.9;78947 +2001-08-03_15:00:00;21.7;78947 +2001-08-03_16:00:00;19.8;78947 +2001-08-03_17:00:00;17.8;78947 +2001-08-03_18:00:00;19.1;78947 +2001-08-03_19:00:00;18.6;78947 +2001-08-03_20:00:00;18;78947 +2001-08-03_21:00:00;17.7;78947 +2001-08-03_22:00:00;17.7;78947 +2001-08-03_23:00:00;17.8;78947 +2001-08-04_00:00:00;17;78947 +2001-08-04_01:00:00;16.8;78947 +2001-08-04_02:00:00;16.9;78947 +2001-08-04_03:00:00;16.9;78947 +2001-08-04_04:00:00;16.9;78947 +2001-08-04_05:00:00;16.6;78947 +2001-08-04_06:00:00;17.5;78947 +2001-08-04_07:00:00;19;78947 +2001-08-04_09:00:00;20.2;78947 +2001-08-04_10:00:00;19.4;78947 +2001-08-04_11:00:00;20;78947 +2001-08-04_12:00:00;20.3;78947 +2001-08-04_13:00:00;21.3;78947 +2001-08-04_14:00:00;22.2;78947 +2001-08-04_15:00:00;22.1;78947 +2001-08-04_16:00:00;22.5;78947 +2001-08-04_17:00:00;22;78947 +2001-08-04_18:00:00;18.6;78947 +2001-08-04_19:00:00;17.1;78947 +2001-08-04_20:00:00;16.5;78947 +2001-08-04_21:00:00;16.2;78947 +2001-08-04_22:00:00;16.2;78947 +2001-08-04_23:00:00;16.7;78947 +2001-08-05_00:00:00;16.4;78947 +2001-08-05_01:00:00;16.7;78947 +2001-08-05_02:00:00;16.5;78947 +2001-08-05_03:00:00;16.4;78947 +2001-08-05_04:00:00;16.6;78947 +2001-08-05_05:00:00;16.5;78947 +2001-08-05_06:00:00;16.1;78947 +2001-08-05_07:00:00;16.3;78947 +2001-08-05_09:00:00;16.5;78947 +2001-08-05_10:00:00;16.7;78947 +2001-08-05_11:00:00;16.8;78947 +2001-08-05_12:00:00;17;78947 +2001-08-05_13:00:00;16.9;78947 +2001-08-05_14:00:00;16.6;78947 +2001-08-05_15:00:00;16.3;78947 +2001-08-05_16:00:00;16.1;78947 +2001-08-05_17:00:00;15.9;78947 +2001-08-05_18:00:00;16;78947 +2001-08-05_19:00:00;15.8;78947 +2001-08-05_20:00:00;15.6;78947 +2001-08-05_21:00:00;15.6;78947 +2001-08-05_22:00:00;15.5;78947 +2001-08-05_23:00:00;15.4;78947 +2001-08-06_00:00:00;15;78947 +2001-08-06_01:00:00;14.8;78947 +2001-08-06_02:00:00;14.6;78947 +2001-08-06_03:00:00;14.5;78947 +2001-08-06_04:00:00;14.5;78947 +2001-08-06_05:00:00;14.5;78947 +2001-08-06_06:00:00;14.7;78947 +2001-08-06_08:00:00;14.8;78947 +2001-08-06_09:00:00;15.7;78947 +2001-08-06_10:00:00;16.1;78947 +2001-08-06_11:00:00;15.8;78947 +2001-08-06_12:00:00;16.5;78947 +2001-08-06_13:00:00;15.8;78947 +2001-08-06_14:00:00;16.5;78947 +2001-08-06_15:00:00;16.2;78947 +2001-08-06_16:00:00;16.1;78947 +2001-08-06_17:00:00;15.7;78947 +2001-08-06_18:00:00;14.8;78947 +2001-08-06_19:00:00;14.2;78947 +2001-08-06_20:00:00;13.1;78947 +2001-08-06_21:00:00;12.3;78947 +2001-08-06_22:00:00;11.7;78947 +2001-08-06_23:00:00;11.3;78947 +2001-08-07_00:00:00;11.2;78947 +2001-08-07_01:00:00;11.5;78947 +2001-08-07_02:00:00;11.6;78947 +2001-08-07_03:00:00;11.8;78947 +2001-08-07_04:00:00;11.7;78947 +2001-08-07_05:00:00;12.3;78947 +2001-08-07_06:00:00;13.1;78947 +2001-08-07_07:00:00;15;78947 +2001-08-07_08:00:00;14.6;78947 +2001-08-07_09:00:00;13.8;78947 +2001-08-07_10:00:00;14.3;78947 +2001-08-07_11:00:00;15.5;78947 +2001-08-07_12:00:00;15.6;78947 +2001-08-07_13:00:00;15.5;78947 +2001-08-07_14:00:00;16.2;78947 +2001-08-07_15:00:00;16.5;78947 +2001-08-07_16:00:00;16;78947 +2001-08-07_17:00:00;15.9;78947 +2001-08-07_18:00:00;14.8;78947 +2001-08-07_19:00:00;14.5;78947 +2001-08-07_20:00:00;14.2;78947 +2001-08-07_21:00:00;14.5;78947 +2001-08-07_22:00:00;14.6;78947 +2001-08-07_23:00:00;13.5;78947 +2001-08-08_00:00:00;13.2;78947 +2001-08-08_01:00:00;13.1;78947 +2001-08-08_02:00:00;13.4;78947 +2001-08-08_03:00:00;13.2;78947 +2001-08-08_04:00:00;12.9;78947 +2001-08-08_05:00:00;12.7;78947 +2001-08-08_06:00:00;14.1;78947 +2001-08-08_07:00:00;14.9;78947 +2001-08-08_08:00:00;14.7;78947 +2001-08-08_09:00:00;15.5;78947 +2001-08-08_10:00:00;15;78947 +2001-08-08_11:00:00;16.7;78947 +2001-08-08_12:00:00;14.6;70000 +2001-08-08_13:00:00;14;70000 +2001-08-08_14:00:00;13.8;58927 +2001-08-08_15:00:00;13.6;70000 +2001-08-08_16:00:00;14.1;58927 +2001-08-08_17:00:00;14.5;70000 +2001-08-08_18:00:00;14.7;58927 +2001-08-08_19:00:00;14.9;70000 +2001-08-08_20:00:00;13.7;78947 +2001-08-08_21:00:00;13.8;78947 +2001-08-08_22:00:00;15.8;70000 +2001-08-08_23:00:00;15.9;70000 +2001-08-09_00:00:00;12.5;78947 +2001-08-09_01:00:00;13.1;78947 +2001-08-09_02:00:00;13.1;78947 +2001-08-09_03:00:00;13.2;78947 +2001-08-09_04:00:00;13.5;78947 +2001-08-09_05:00:00;13.7;78947 +2001-08-09_06:00:00;13.8;78947 +2001-08-09_07:00:00;14.7;70000 +2001-08-09_08:00:00;14.7;58927 +2001-08-09_09:00:00;14.7;70000 +2001-08-09_10:00:00;14.6;70000 +2001-08-09_11:00:00;14.6;70000 +2001-08-09_12:00:00;14.4;70000 +2001-08-09_13:00:00;14.2;70000 +2001-08-09_14:00:00;14.4;70000 +2001-08-09_15:00:00;14.3;70000 +2001-08-09_16:00:00;14.2;70000 +2001-08-09_17:00:00;14;70000 +2001-08-09_18:00:00;13.9;70000 +2001-08-09_19:00:00;12.9;70000 +2001-08-09_20:00:00;13.2;70000 +2001-08-09_21:00:00;13.1;70000 +2001-08-09_22:00:00;13.1;70000 +2001-08-09_23:00:00;13.1;70000 +2001-08-10_00:00:00;12.8;70000 +2001-08-10_01:00:00;11.6;70000 +2001-08-10_02:00:00;12.2;70000 +2001-08-10_03:00:00;12.7;70000 +2001-08-10_04:00:00;12.5;70000 +2001-08-10_05:00:00;12.4;70000 +2001-08-10_06:00:00;12.5;70000 +2001-08-10_07:00:00;12.6;70000 +2001-08-10_08:00:00;12.6;70000 +2001-08-10_09:00:00;11.8;70000 +2001-08-10_10:00:00;12.1;70000 +2001-08-10_11:00:00;12.1;70000 +2001-08-10_12:00:00;12.7;70000 +2001-08-10_13:00:00;13;70000 +2001-08-10_14:00:00;13.1;70000 +2001-08-10_15:00:00;13.3;70000 +2001-08-10_16:00:00;13.4;70000 +2001-08-10_17:00:00;13.5;70000 +2001-08-10_18:00:00;13.5;70000 +2001-08-10_19:00:00;13.6;70000 +2001-08-10_20:00:00;13.8;70000 +2001-08-10_21:00:00;13.9;70000 +2001-08-10_22:00:00;13.9;70000 +2001-08-10_23:00:00;13.7;58927 +2001-08-11_00:00:00;13.5;70000 +2001-08-11_01:00:00;13.4;70000 +2001-08-11_02:00:00;13.1;70000 +2001-08-11_03:00:00;12.5;70000 +2001-08-11_04:00:00;12.7;70000 +2001-08-11_05:00:00;13.2;70000 +2001-08-11_06:00:00;12.9;70000 +2001-08-11_07:00:00;14.3;70000 +2001-08-11_08:00:00;15;70000 +2001-08-11_09:00:00;15.8;70000 +2001-08-11_10:00:00;15.8;58927 +2001-08-11_11:00:00;15.7;70000 +2001-08-11_12:00:00;15.9;70000 +2001-08-11_13:00:00;16;70000 +2001-08-11_14:00:00;16.1;70000 +2001-08-11_15:00:00;16.2;70000 +2001-08-11_16:00:00;16;70000 +2001-08-11_17:00:00;16.1;70000 +2001-08-11_18:00:00;16.1;70000 +2001-08-11_19:00:00;16;70000 +2001-08-11_20:00:00;15.9;70000 +2001-08-11_21:00:00;15.7;70000 +2001-08-11_22:00:00;15.6;70000 +2001-08-11_23:00:00;15.6;70000 +2001-08-12_00:00:00;15.6;70000 +2001-08-12_01:00:00;15.3;70000 +2001-08-12_02:00:00;15.2;70000 +2001-08-12_03:00:00;15.1;70000 +2001-08-12_04:00:00;15;70000 +2001-08-12_05:00:00;14.7;70000 +2001-08-12_06:00:00;14.7;70000 +2001-08-12_07:00:00;14.9;70000 +2001-08-12_08:00:00;14.9;70000 +2001-08-12_09:00:00;14.9;70000 +2001-08-12_10:00:00;15.3;70000 +2001-08-12_11:00:00;15.6;70000 +2001-08-12_12:00:00;15.8;70000 +2001-08-12_13:00:00;15.6;70000 +2001-08-12_14:00:00;15.9;70000 +2001-08-12_15:00:00;16.2;70000 +2001-08-12_16:00:00;16.2;70000 +2001-08-12_17:00:00;16.3;70000 +2001-08-12_18:00:00;16.9;70000 +2001-08-12_19:00:00;16.6;70000 +2001-08-12_20:00:00;15.7;70000 +2001-08-12_21:00:00;16.2;70000 +2001-08-12_22:00:00;16.5;70000 +2001-08-12_23:00:00;16.3;70000 +2001-08-13_00:00:00;16.1;70000 +2001-08-13_01:00:00;16;70000 +2001-08-13_02:00:00;15.9;70000 +2001-08-13_03:00:00;15.7;70000 +2001-08-13_04:00:00;15.8;70000 +2001-08-13_05:00:00;15.5;70000 +2001-08-13_06:00:00;15.3;70000 +2001-08-13_07:00:00;15.6;70000 +2001-08-13_08:00:00;16.2;70000 +2001-08-13_09:00:00;16.6;70000 +2001-08-13_10:00:00;16.9;70000 +2001-08-13_11:00:00;17.2;70000 +2001-08-13_12:00:00;17.5;70000 +2001-08-13_13:00:00;17.4;70000 +2001-08-13_14:00:00;17.6;70000 +2001-08-13_15:00:00;17.2;70000 +2001-08-13_16:00:00;17.1;70000 +2001-08-13_17:00:00;17.2;70000 +2001-08-13_18:00:00;17.2;70000 +2001-08-13_19:00:00;17.9;70000 +2001-08-13_20:00:00;17.5;70000 +2001-08-13_21:00:00;17.3;70000 +2001-08-13_22:00:00;17.2;70000 +2001-08-13_23:00:00;17.1;70000 +2001-08-14_00:00:00;17.1;70000 +2001-08-14_01:00:00;17.2;70000 +2001-08-14_02:00:00;16.9;70000 +2001-08-14_03:00:00;17.2;70000 +2001-08-14_04:00:00;17;70000 +2001-08-14_05:00:00;16.9;70000 +2001-08-14_06:00:00;16.8;70000 +2001-08-14_07:00:00;17;70000 +2001-08-14_08:00:00;17.3;70000 +2001-08-14_09:00:00;17;70000 +2001-08-14_10:00:00;17.1;70000 +2001-08-14_11:00:00;17.3;70000 +2001-08-14_12:00:00;17.1;70000 +2001-08-14_13:00:00;17;70000 +2001-08-14_14:00:00;17.1;70000 +2001-08-14_15:00:00;16.9;70000 +2001-08-14_16:00:00;16.7;70000 +2001-08-14_17:00:00;16.5;70000 +2001-08-14_18:00:00;16.3;70000 +2001-08-14_19:00:00;16.2;70000 +2001-08-14_20:00:00;16.2;70000 +2001-08-14_21:00:00;16.2;70000 +2001-08-14_22:00:00;15.9;70000 +2001-08-14_23:00:00;16;70000 +2001-08-15_00:00:00;15.4;70000 +2001-08-15_01:00:00;15.3;70000 +2001-08-15_02:00:00;15.3;70000 +2001-08-15_03:00:00;15.3;70000 +2001-08-15_04:00:00;14.9;70000 +2001-08-15_05:00:00;15;70000 +2001-08-15_06:00:00;15;70000 +2001-08-15_07:00:00;15.2;70000 +2001-08-15_08:00:00;15.1;70000 +2001-08-15_09:00:00;15.4;70000 +2001-08-15_10:00:00;15.3;70000 +2001-08-15_11:00:00;15.3;70000 +2001-08-15_12:00:00;15.6;70000 +2001-08-15_13:00:00;15.6;70000 +2001-08-15_14:00:00;15.5;70000 +2001-08-15_15:00:00;15.5;70000 +2001-08-15_16:00:00;15.5;70000 +2001-08-15_17:00:00;15.5;70000 +2001-08-15_18:00:00;15.3;70000 +2001-08-15_19:00:00;15.3;70000 +2001-08-15_20:00:00;15.2;70000 +2001-08-15_21:00:00;15.2;70000 +2001-08-15_22:00:00;15.1;70000 +2001-08-15_23:00:00;15;70000 +2001-08-16_00:00:00;15;70000 +2001-08-16_01:00:00;14.8;70000 +2001-08-16_02:00:00;14.8;70000 +2001-08-16_03:00:00;14.8;70000 +2001-08-16_04:00:00;14.6;70000 +2001-08-16_05:00:00;14.6;70000 +2001-08-16_06:00:00;14.5;70000 +2001-08-16_07:00:00;14.6;70000 +2001-08-16_08:00:00;14.5;70000 +2001-08-16_09:00:00;14.5;70000 +2001-08-16_10:00:00;14.8;70000 +2001-08-16_11:00:00;14.9;70000 +2001-08-16_12:00:00;15;70000 +2001-08-16_13:00:00;15.1;70000 +2001-08-16_14:00:00;15.2;70000 +2001-08-16_15:00:00;15.3;70000 +2001-08-16_16:00:00;15.4;70000 +2001-08-16_17:00:00;15.3;70000 +2001-08-16_18:00:00;15.4;70000 +2001-08-16_19:00:00;15.4;70000 +2001-08-16_20:00:00;15.5;70000 +2001-08-16_21:00:00;15.6;70000 +2001-08-16_22:00:00;15.7;70000 +2001-08-16_23:00:00;15.3;70000 +2001-08-17_00:00:00;15.2;70000 +2001-08-17_01:00:00;14.9;70000 +2001-08-17_02:00:00;14.6;70000 +2001-08-17_03:00:00;14.6;70000 +2001-08-17_04:00:00;14.3;70000 +2001-08-17_05:00:00;14.1;70000 +2001-08-17_06:00:00;14.3;70000 +2001-08-17_07:00:00;14.5;70000 +2001-08-17_08:00:00;14.8;70000 +2001-08-17_09:00:00;15.1;70000 +2001-08-17_10:00:00;15.4;70000 +2001-08-17_11:00:00;15.8;70000 +2001-08-17_12:00:00;16;70000 +2001-08-17_13:00:00;16.1;70000 +2001-08-17_14:00:00;16.3;70000 +2001-08-17_15:00:00;16.5;70000 +2001-08-17_16:00:00;16.7;70000 +2001-08-17_17:00:00;16.5;70000 +2001-08-17_18:00:00;16.5;70000 +2001-08-17_19:00:00;16.3;70000 +2001-08-17_20:00:00;15.9;70000 +2001-08-17_21:00:00;15.9;70000 +2001-08-17_22:00:00;15.8;70000 +2001-08-17_23:00:00;15.6;70000 +2001-08-18_00:00:00;15.5;70000 +2001-08-18_01:00:00;15.4;70000 +2001-08-18_02:00:00;15.3;70000 +2001-08-18_03:00:00;15.2;70000 +2001-08-18_04:00:00;15.2;70000 +2001-08-18_05:00:00;15;70000 +2001-08-18_06:00:00;15;70000 +2001-08-18_07:00:00;15;70000 +2001-08-18_08:00:00;15.2;70000 +2001-08-18_09:00:00;15.2;70000 +2001-08-18_10:00:00;15.2;70000 +2001-08-18_11:00:00;15.2;70000 +2001-08-18_12:00:00;15.2;70000 +2001-08-18_13:00:00;15.2;70203 +2001-08-18_14:00:00;15.3;70000 +2001-08-18_15:00:00;15.3;70000 +2001-08-18_16:00:00;15.3;70000 +2001-08-18_17:00:00;15.3;70000 +2001-08-18_18:00:00;15.2;70000 +2001-08-18_19:00:00;15.2;70000 +2001-08-18_20:00:00;15.2;70000 +2001-08-18_21:00:00;15.2;70000 +2001-08-18_22:00:00;15.2;70000 +2001-08-18_23:00:00;15.2;70203 +2001-08-19_00:00:00;15.2;70203 +2001-08-19_01:00:00;15.2;70203 +2001-08-19_02:00:00;15.2;70203 +2001-08-19_03:00:00;15;70000 +2001-08-19_04:00:00;15;70000 +2001-08-19_05:00:00;15;70000 +2001-08-19_06:00:00;14.9;70000 +2001-08-19_07:00:00;15.1;70000 +2001-08-19_08:00:00;15;70000 +2001-08-19_09:00:00;15;70000 +2001-08-19_10:00:00;15.1;70000 +2001-08-19_11:00:00;15;70000 +2001-08-19_12:00:00;15.1;70000 +2001-08-19_13:00:00;15.1;70000 +2001-08-19_14:00:00;15.1;70000 +2001-08-19_15:00:00;15;70000 +2001-08-19_16:00:00;15;70000 +2001-08-19_17:00:00;15;70000 +2001-08-19_18:00:00;14.9;70000 +2001-08-19_19:00:00;14.8;70000 +2001-08-19_20:00:00;14.8;70000 +2001-08-19_21:00:00;14.6;70000 +2001-08-19_22:00:00;14.6;70000 +2001-08-19_23:00:00;14.5;70000 +2001-08-20_00:00:00;14.8;70000 +2001-08-20_01:00:00;14.6;70000 +2001-08-20_02:00:00;14.8;70000 +2001-08-20_03:00:00;14.1;70000 +2001-08-20_04:00:00;14.9;70000 +2001-08-20_05:00:00;15.2;70000 +2001-08-20_06:00:00;15.2;70000 +2001-08-20_07:00:00;13.9;70000 +2001-08-20_08:00:00;13.5;70000 +2001-08-20_09:00:00;14.6;70000 +2001-08-20_10:00:00;15.8;70000 +2001-08-20_11:00:00;15.8;70000 +2001-08-20_12:00:00;15.9;70000 +2001-08-20_13:00:00;15.8;70000 +2001-08-20_14:00:00;15.8;70000 +2001-08-20_15:00:00;15.9;70000 +2001-08-20_16:00:00;16;70000 +2001-08-20_17:00:00;16;70000 +2001-08-20_18:00:00;15.8;70000 +2001-08-20_19:00:00;15.6;70000 +2001-08-20_20:00:00;15.5;70000 +2001-08-20_21:00:00;15.5;70000 +2001-08-20_22:00:00;15.4;70000 +2001-08-20_23:00:00;15.3;70000 +2001-08-21_00:00:00;15.4;70000 +2001-08-21_01:00:00;15.4;70000 +2001-08-21_02:00:00;15.5;70000 +2001-08-21_03:00:00;15.6;70000 +2001-08-21_04:00:00;15.8;70000 +2001-08-21_05:00:00;15.4;70000 +2001-08-21_06:00:00;14.7;70000 +2001-08-21_07:00:00;14.9;70000 +2001-08-21_08:00:00;14.9;70000 +2001-08-21_09:00:00;14.4;70000 +2001-08-21_10:00:00;14.9;70000 +2001-08-21_11:00:00;15.9;70000 +2001-08-21_12:00:00;15.8;70000 +2001-08-21_13:00:00;16.1;70000 +2001-08-21_14:00:00;16.1;70000 +2001-08-21_15:00:00;15.9;70000 +2001-08-21_16:00:00;15.8;70000 +2001-08-21_17:00:00;15.8;70000 +2001-08-21_18:00:00;15.7;70000 +2001-08-21_19:00:00;15.7;70000 +2001-08-21_20:00:00;15.6;70000 +2001-08-21_21:00:00;15.5;70000 +2001-08-21_22:00:00;15.3;70000 +2001-08-21_23:00:00;15.2;70000 +2001-08-22_00:00:00;15.2;70000 +2001-08-22_01:00:00;15.3;70000 +2001-08-22_02:00:00;15.4;70000 +2001-08-22_03:00:00;15.4;70000 +2001-08-22_04:00:00;15.2;70000 +2001-08-22_05:00:00;15.2;70000 +2001-08-22_06:00:00;15.5;70000 +2001-08-22_07:00:00;15.5;70000 +2001-08-22_08:00:00;15.5;70000 +2001-08-22_09:00:00;15.5;70000 +2001-08-22_10:00:00;15.4;70000 +2001-08-22_11:00:00;15.3;70000 +2001-08-22_12:00:00;15.2;70000 +2001-08-22_13:00:00;15.2;70000 +2001-08-22_14:00:00;15.2;70000 +2001-08-22_15:00:00;15.2;70000 +2001-08-22_16:00:00;15;70000 +2001-08-22_17:00:00;15.3;70000 +2001-08-22_18:00:00;15.2;70000 +2001-08-22_19:00:00;15.2;70000 +2001-08-22_20:00:00;15.3;70000 +2001-08-22_21:00:00;15.3;70000 +2001-08-22_22:00:00;15;70000 +2001-08-22_23:00:00;15.2;70000 +2001-08-23_00:00:00;15.2;70000 +2001-08-23_01:00:00;15;70000 +2001-08-23_02:00:00;14.9;70000 +2001-08-23_03:00:00;14.9;70000 +2001-08-23_04:00:00;14.8;70000 +2001-08-23_05:00:00;14.9;70000 +2001-08-23_06:00:00;15;70000 +2001-08-23_07:00:00;15.1;70000 +2001-08-23_08:00:00;14.8;70000 +2001-08-23_09:00:00;15.1;70000 +2001-08-23_10:00:00;15.2;70000 +2001-08-23_11:00:00;15.2;70000 +2001-08-23_12:00:00;15.5;70000 +2001-08-23_13:00:00;15.6;70000 +2001-08-23_14:00:00;15.7;70000 +2001-08-23_15:00:00;15.9;70000 +2001-08-23_16:00:00;16;70000 +2001-08-23_17:00:00;16.2;70000 +2001-08-23_18:00:00;16.3;70000 +2001-08-23_19:00:00;16.3;70000 +2001-08-23_20:00:00;16.5;70000 +2001-08-23_21:00:00;16.5;70000 +2001-08-23_22:00:00;16.9;70000 +2001-08-23_23:00:00;16.6;70000 +2001-08-24_00:00:00;16.5;70000 +2001-08-24_01:00:00;16.2;70000 +2001-08-24_02:00:00;16.6;70000 +2001-08-24_03:00:00;16;70000 +2001-08-24_04:00:00;16.6;70000 +2001-08-24_05:00:00;16.7;70000 +2001-08-24_06:00:00;15.8;70000 +2001-08-24_07:00:00;14.9;70000 +2001-08-24_08:00:00;14.6;70000 +2001-08-24_09:00:00;15.2;70000 +2001-08-24_10:00:00;15.7;70000 +2001-08-24_11:00:00;15.7;70000 +2001-08-24_12:00:00;15.8;70000 +2001-08-24_13:00:00;16.3;70000 +2001-08-24_14:00:00;17.2;70000 +2001-08-24_15:00:00;17.3;70000 +2001-08-24_16:00:00;17.4;70000 +2001-08-24_17:00:00;17.4;70000 +2001-08-24_18:00:00;15.8;70000 +2001-08-24_19:00:00;15.4;70000 +2001-08-24_20:00:00;15.3;70000 +2001-08-24_21:00:00;15;70000 +2001-08-24_22:00:00;15.5;70000 +2001-08-24_23:00:00;15.5;70000 +2001-08-25_00:00:00;15.5;70000 +2001-08-25_01:00:00;15.3;70000 +2001-08-25_02:00:00;15.3;70000 +2001-08-25_03:00:00;15.3;70000 +2001-08-25_04:00:00;15.3;70000 +2001-08-25_05:00:00;15.3;70000 +2001-08-25_06:00:00;15.3;70203 +2001-08-25_07:00:00;15.3;70203 +2001-08-25_08:00:00;15.6;70000 +2001-08-25_09:00:00;15.7;70000 +2001-08-25_10:00:00;15.8;70000 +2001-08-25_11:00:00;15.8;70000 +2001-08-25_12:00:00;15.8;70000 +2001-08-25_13:00:00;15.9;70000 +2001-08-25_14:00:00;16.1;70000 +2001-08-25_15:00:00;16.2;70000 +2001-08-25_16:00:00;16.3;70000 +2001-08-25_17:00:00;16.2;70000 +2001-08-25_18:00:00;16.2;70000 +2001-08-25_19:00:00;16.3;70000 +2001-08-25_20:00:00;16.3;70000 +2001-08-25_21:00:00;16.3;70000 +2001-08-25_22:00:00;16.3;70000 +2001-08-25_23:00:00;16.1;70000 +2001-08-26_00:00:00;15.8;70000 +2001-08-26_01:00:00;16.4;70000 +2001-08-26_02:00:00;16.4;70000 +2001-08-26_03:00:00;16.5;70000 +2001-08-26_04:00:00;16.5;70000 +2001-08-26_05:00:00;16.2;70000 +2001-08-26_06:00:00;16.3;70000 +2001-08-26_07:00:00;16.5;70000 +2001-08-26_08:00:00;16.4;70000 +2001-08-26_09:00:00;16.6;70000 +2001-08-26_10:00:00;16.9;70000 +2001-08-26_11:00:00;17.1;70000 +2001-08-26_12:00:00;16.6;70000 +2001-08-26_13:00:00;16.2;70000 +2001-08-26_14:00:00;16.7;70000 +2001-08-26_15:00:00;17.3;70000 +2001-08-26_16:00:00;17.7;70000 +2001-08-26_17:00:00;16.4;70000 +2001-08-26_18:00:00;17.9;70000 +2001-08-26_19:00:00;18.1;70000 +2001-08-26_20:00:00;18.5;70000 +2001-08-26_21:00:00;19;70000 +2001-08-26_22:00:00;18.4;70000 +2001-08-26_23:00:00;18.4;70000 +2001-08-27_00:00:00;18.1;70000 +2001-08-27_01:00:00;17.8;70000 +2001-08-27_02:00:00;17.7;70000 +2001-08-27_03:00:00;17.6;70000 +2001-08-27_04:00:00;17.5;70000 +2001-08-27_05:00:00;17.5;70000 +2001-08-27_06:00:00;17.6;70000 +2001-08-27_07:00:00;17.1;70000 +2001-08-27_08:00:00;16.9;70000 +2001-08-27_09:00:00;16.7;70000 +2001-08-27_10:00:00;15;78947 +2001-08-27_11:00:00;15;78947 +2001-08-27_12:00:00;18.1;78947 +2001-08-27_13:00:00;18.1;78947 +2001-08-27_14:00:00;16.5;78947 +2001-08-27_15:00:00;16.9;78947 +2001-08-27_16:00:00;15.1;78947 +2001-08-27_17:00:00;15.2;78947 +2001-08-27_18:00:00;15.4;78947 +2001-08-27_19:00:00;14.9;78947 +2001-08-27_20:00:00;14.7;78947 +2001-08-27_21:00:00;14.3;78947 +2001-08-27_22:00:00;14.4;78947 +2001-08-27_23:00:00;14.1;78947 +2001-08-28_00:00:00;12.8;78947 +2001-08-28_01:00:00;12.7;78947 +2001-08-28_02:00:00;12.6;78947 +2001-08-28_03:00:00;12.4;78947 +2001-08-28_04:00:00;12.6;78947 +2001-08-28_05:00:00;11.9;78947 +2001-08-28_06:00:00;12.6;78947 +2001-08-28_07:00:00;12.7;78947 +2001-08-28_08:00:00;13.5;78947 +2001-08-28_09:00:00;12.8;78947 +2001-08-28_10:00:00;12.9;78947 +2001-08-28_11:00:00;13.2;78947 +2001-08-28_12:00:00;14.4;78947 +2001-08-28_13:00:00;13;78947 +2001-08-28_14:00:00;14.1;78947 +2001-08-28_15:00:00;13.2;78947 +2001-08-28_16:00:00;13;78947 +2001-08-28_17:00:00;12.7;78947 +2001-08-28_18:00:00;12.8;78947 +2001-08-28_19:00:00;13;78947 +2001-08-28_20:00:00;13.1;78947 +2001-08-28_21:00:00;12.9;78947 +2001-08-28_22:00:00;13;78947 +2001-08-28_23:00:00;13.4;78947 +2001-08-29_00:00:00;14.1;78947 +2001-08-29_01:00:00;13.3;78947 +2001-08-29_02:00:00;13.6;78947 +2001-08-29_03:00:00;13.8;78947 +2001-08-29_04:00:00;13.8;78947 +2001-08-29_05:00:00;13.9;78947 +2001-08-29_06:00:00;15.7;70000 +2001-08-29_07:00:00;15.9;70000 +2001-08-29_08:00:00;16.1;70000 +2001-08-29_09:00:00;16.1;70000 +2001-08-29_10:00:00;15.8;70000 +2001-08-29_11:00:00;15.4;70000 +2001-08-29_12:00:00;15.3;70000 +2001-08-29_13:00:00;15.7;70000 +2001-08-29_14:00:00;15.8;70000 +2001-08-29_15:00:00;15;70000 +2001-08-29_16:00:00;14.8;70000 +2001-08-29_17:00:00;14.5;70000 +2001-08-29_18:00:00;13.2;70000 +2001-08-29_19:00:00;13.6;70000 +2001-08-29_20:00:00;13.4;70000 +2001-08-29_21:00:00;13.3;70000 +2001-08-29_22:00:00;13.5;70000 +2001-08-29_23:00:00;13.2;70000 +2001-08-30_00:00:00;13.5;70000 +2001-08-30_01:00:00;13.3;70000 +2001-08-30_02:00:00;13.2;70000 +2001-08-30_03:00:00;13.6;70000 +2001-08-30_04:00:00;13.5;70000 +2001-08-30_05:00:00;13.5;70000 +2001-08-30_06:00:00;13.5;70000 +2001-08-30_07:00:00;13.6;70000 +2001-08-30_08:00:00;13.5;70000 +2001-08-30_09:00:00;13.5;70000 +2001-08-30_10:00:00;13.1;70000 +2001-08-30_11:00:00;13.2;70000 +2001-08-30_12:00:00;13.2;70000 +2001-08-30_13:00:00;13.4;70000 +2001-08-30_14:00:00;13.2;70000 +2001-08-30_15:00:00;13;70000 +2001-08-30_16:00:00;12.6;70000 +2001-08-30_17:00:00;12.8;70000 +2001-08-30_18:00:00;13.3;70000 +2001-08-30_19:00:00;14;70000 +2001-08-30_20:00:00;14.7;70000 +2001-08-30_21:00:00;14.8;70000 +2001-08-30_22:00:00;14.7;70000 +2001-08-30_23:00:00;14.6;70000 +2001-08-31_00:00:00;14.7;70000 +2001-08-31_01:00:00;14.6;70000 +2001-08-31_02:00:00;14.3;70000 +2001-08-31_03:00:00;14.4;70000 +2001-08-31_04:00:00;14;70000 +2001-08-31_05:00:00;14.2;70000 +2001-08-31_06:00:00;13.6;70000 +2001-08-31_07:00:00;14.5;70000 +2001-08-31_08:00:00;14.6;70000 +2001-08-31_09:00:00;14.3;70000 +2001-08-31_10:00:00;14.2;70000 +2001-08-31_11:00:00;14.5;70000 +2001-08-31_12:00:00;14.3;70000 +2001-08-31_13:00:00;14.2;70000 +2001-08-31_14:00:00;14.3;70000 +2001-08-31_15:00:00;14.3;70000 +2001-08-31_16:00:00;14.2;70000 +2001-08-31_17:00:00;14.3;70000 +2001-08-31_18:00:00;14.2;70000 +2001-08-31_19:00:00;14.1;70000 +2001-08-31_20:00:00;14.2;70000 +2001-08-31_21:00:00;14.3;70000 +2001-08-31_22:00:00;14.2;70000 +2001-08-31_23:00:00;14.3;70000 +2001-09-01_00:00:00;14.6;70000 +2001-09-01_01:00:00;14.8;70000 +2001-09-01_02:00:00;10.5;78947 +2001-09-01_03:00:00;9.7;78947 +2001-09-01_04:00:00;9.2;78947 +2001-09-01_05:00:00;8.9;78947 +2001-09-01_06:00:00;14.9;70000 +2001-09-01_07:00:00;14.8;70000 +2001-09-01_08:00:00;14.8;70000 +2001-09-01_09:00:00;14.6;70000 +2001-09-01_10:00:00;14.7;70000 +2001-09-01_11:00:00;14.9;70000 +2001-09-01_12:00:00;14.8;70000 +2001-09-01_13:00:00;14.4;70000 +2001-09-01_14:00:00;14.1;70000 +2001-09-01_15:00:00;14.1;70000 +2001-09-01_16:00:00;14.3;70000 +2001-09-01_17:00:00;14.3;70000 +2001-09-01_18:00:00;14.2;70000 +2001-09-01_19:00:00;14.1;70000 +2001-09-01_20:00:00;13.9;70000 +2001-09-01_21:00:00;13.9;70000 +2001-09-01_22:00:00;13.8;70000 +2001-09-01_23:00:00;13.6;70000 +2001-09-02_00:00:00;13.6;70000 +2001-09-02_01:00:00;13.3;70000 +2001-09-02_02:00:00;13.2;70000 +2001-09-02_03:00:00;13.2;70000 +2001-09-02_04:00:00;13.1;70000 +2001-09-02_05:00:00;13.1;70000 +2001-09-02_06:00:00;13.2;70000 +2001-09-02_07:00:00;13;70000 +2001-09-02_08:00:00;13.1;70000 +2001-09-02_09:00:00;13.4;70000 +2001-09-02_10:00:00;13.6;70000 +2001-09-02_11:00:00;13.7;70000 +2001-09-02_12:00:00;13.8;70000 +2001-09-02_13:00:00;13.9;70000 +2001-09-02_14:00:00;13.5;70000 +2001-09-02_15:00:00;13.6;70000 +2001-09-02_16:00:00;14;70000 +2001-09-02_17:00:00;13.6;70000 +2001-09-02_18:00:00;13.6;70000 +2001-09-02_19:00:00;13.9;70000 +2001-09-02_20:00:00;14.3;70000 +2001-09-02_21:00:00;14.6;70000 +2001-09-02_22:00:00;14.6;70000 +2001-09-02_23:00:00;14.9;70000 +2001-09-03_00:00:00;14.5;70000 +2001-09-03_01:00:00;14.6;70000 +2001-09-03_02:00:00;14.8;70000 +2001-09-03_03:00:00;14.7;70000 +2001-09-03_04:00:00;14.8;70000 +2001-09-03_05:00:00;14.9;70000 +2001-09-03_06:00:00;15.1;70000 +2001-09-03_07:00:00;15.1;70000 +2001-09-03_08:00:00;15.2;70000 +2001-09-03_09:00:00;15.5;70000 +2001-09-03_10:00:00;15.8;70000 +2001-09-03_11:00:00;15.8;70000 +2001-09-03_12:00:00;15.9;70000 +2001-09-03_13:00:00;16.2;70000 +2001-09-03_14:00:00;16.2;70000 +2001-09-03_15:00:00;16.2;70000 +2001-09-03_16:00:00;16.1;70000 +2001-09-03_17:00:00;16.1;70000 +2001-09-03_18:00:00;16.2;70000 +2001-09-03_19:00:00;16.3;70000 +2001-09-03_20:00:00;16.4;70000 +2001-09-03_21:00:00;16.4;70000 +2001-09-03_22:00:00;16.1;70000 +2001-09-03_23:00:00;16.2;70000 +2001-09-04_00:00:00;15.9;70000 +2001-09-04_01:00:00;15.6;70000 +2001-09-04_02:00:00;15.3;70000 +2001-09-04_03:00:00;15.3;70000 +2001-09-04_04:00:00;15.6;70000 +2001-09-04_05:00:00;15.6;70000 +2001-09-04_06:00:00;15.4;70000 +2001-09-04_07:00:00;15.4;70000 +2001-09-04_08:00:00;15.3;70000 +2001-09-04_09:00:00;15.4;70000 +2001-09-04_10:00:00;15.5;70000 +2001-09-04_11:00:00;15.8;70000 +2001-09-04_12:00:00;16;70000 +2001-09-04_13:00:00;15.3;70000 +2001-09-04_14:00:00;15.3;70000 +2001-09-04_15:00:00;15.3;70000 +2001-09-04_16:00:00;15.3;70000 +2001-09-04_17:00:00;15.3;70000 +2001-09-04_18:00:00;15.3;70203 +2001-09-04_19:00:00;15.3;70203 +2001-09-04_20:00:00;15;70000 +2001-09-04_21:00:00;15.4;70000 +2001-09-04_22:00:00;15.4;70000 +2001-09-04_23:00:00;15.6;70000 +2001-09-05_00:00:00;15.6;70000 +2001-09-05_01:00:00;15.4;70000 +2001-09-05_02:00:00;15.7;70000 +2001-09-05_03:00:00;15.8;70000 +2001-09-05_04:00:00;15.9;70000 +2001-09-05_05:00:00;15.9;70000 +2001-09-05_06:00:00;16;70000 +2001-09-05_07:00:00;16;70000 +2001-09-05_08:00:00;16.1;70000 +2001-09-05_09:00:00;16.2;70000 +2001-09-05_10:00:00;15.9;70000 +2001-09-05_11:00:00;15.9;70000 +2001-09-05_12:00:00;15.7;70000 +2001-09-05_13:00:00;15.5;70000 +2001-09-05_14:00:00;15;70000 +2001-09-05_15:00:00;14.7;70000 +2001-09-05_16:00:00;14.9;70000 +2001-09-05_17:00:00;14.8;70000 +2001-09-05_18:00:00;14.6;70000 +2001-09-05_19:00:00;14.5;70000 +2001-09-05_20:00:00;14.6;70000 +2001-09-05_21:00:00;14.8;70000 +2001-09-05_22:00:00;14.3;70000 +2001-09-05_23:00:00;14.2;70000 +2001-09-06_00:00:00;14.2;70000 +2001-09-06_01:00:00;14.2;70000 +2001-09-06_02:00:00;14.1;70000 +2001-09-06_03:00:00;14.1;70000 +2001-09-06_04:00:00;14.1;70000 +2001-09-06_05:00:00;14;70000 +2001-09-06_06:00:00;14.1;70000 +2001-09-06_07:00:00;14.3;70000 +2001-09-06_08:00:00;14.6;70000 +2001-09-06_09:00:00;14.9;70000 +2001-09-06_10:00:00;13.8;70000 +2001-09-06_11:00:00;14.3;70000 +2001-09-06_12:00:00;14.8;70000 +2001-09-06_13:00:00;14.9;70000 +2001-09-06_14:00:00;14.9;70000 +2001-09-06_15:00:00;14.9;70000 +2001-09-06_16:00:00;14.9;70000 +2001-09-06_17:00:00;14.8;70000 +2001-09-06_18:00:00;14.7;70000 +2001-09-06_19:00:00;14.6;70000 +2001-09-06_20:00:00;14.6;70000 +2001-09-06_21:00:00;14.5;70000 +2001-09-06_22:00:00;14.7;70000 +2001-09-06_23:00:00;14.7;70000 +2001-09-07_00:00:00;14.6;70000 +2001-09-07_01:00:00;14.6;70000 +2001-09-07_02:00:00;14.5;70000 +2001-09-07_03:00:00;14.3;70000 +2001-09-07_04:00:00;14.6;70000 +2001-09-07_05:00:00;14;70000 +2001-09-07_06:00:00;13.6;70000 +2001-09-07_07:00:00;13.7;70000 +2001-09-07_08:00:00;12.9;70000 +2001-09-07_09:00:00;13.6;70000 +2001-09-07_10:00:00;13.4;70000 +2001-09-07_11:00:00;13.2;70000 +2001-09-07_12:00:00;12.3;70000 +2001-09-07_13:00:00;13.6;70000 +2001-09-07_14:00:00;13.6;70000 +2001-09-07_15:00:00;13.3;70000 +2001-09-07_16:00:00;13.5;70000 +2001-09-07_17:00:00;13.6;70000 +2001-09-07_18:00:00;13.5;70000 +2001-09-07_19:00:00;13.4;70000 +2001-09-07_20:00:00;13.5;70000 +2001-09-07_21:00:00;13.1;70000 +2001-09-07_22:00:00;12.5;70000 +2001-09-07_23:00:00;12.5;70000 +2001-09-08_00:00:00;12.5;70000 +2001-09-08_01:00:00;12.8;70000 +2001-09-08_02:00:00;11.8;70000 +2001-09-08_03:00:00;12.5;70000 +2001-09-08_04:00:00;13.1;70000 +2001-09-08_05:00:00;13.2;70000 +2001-09-08_06:00:00;12.6;70000 +2001-09-08_07:00:00;12.9;70000 +2001-09-08_08:00:00;13.5;70000 +2001-09-08_09:00:00;13.5;70000 +2001-09-08_10:00:00;13.4;70000 +2001-09-08_11:00:00;13.6;70000 +2001-09-08_12:00:00;12.5;70000 +2001-09-08_13:00:00;13.2;70000 +2001-09-08_14:00:00;13.5;70000 +2001-09-08_15:00:00;13.6;70000 +2001-09-08_16:00:00;13.6;70000 +2001-09-08_17:00:00;13.6;70000 +2001-09-08_18:00:00;13.6;70000 +2001-09-08_19:00:00;13.6;70000 +2001-09-08_20:00:00;13.6;70203 +2001-09-08_21:00:00;13.7;70000 +2001-09-08_22:00:00;13.6;70000 +2001-09-08_23:00:00;13.6;70000 +2001-09-09_00:00:00;13.8;70000 +2001-09-09_01:00:00;13.6;70000 +2001-09-09_02:00:00;13.5;70000 +2001-09-09_03:00:00;13.6;70000 +2001-09-09_04:00:00;13.5;70000 +2001-09-09_05:00:00;12.3;70000 +2001-09-09_06:00:00;13.3;70000 +2001-09-09_07:00:00;13.2;70000 +2001-09-09_08:00:00;13.2;70000 +2001-09-09_09:00:00;13.3;70000 +2001-09-09_10:00:00;13.4;58927 +2001-09-09_11:00:00;13.5;70000 +2001-09-09_12:00:00;13.6;70000 +2001-09-09_13:00:00;13.6;70000 +2001-09-09_14:00:00;13.6;70000 +2001-09-09_15:00:00;13.6;70000 +2001-09-09_16:00:00;13.7;70000 +2001-09-09_17:00:00;13.8;70000 +2001-09-09_18:00:00;12.1;70000 +2001-09-09_19:00:00;13.3;70000 +2001-09-09_20:00:00;13.1;70000 +2001-09-09_21:00:00;12.9;70000 +2001-09-09_22:00:00;11.8;70000 +2001-09-09_23:00:00;11.6;70000 +2001-09-10_00:00:00;11.8;70000 +2001-09-10_01:00:00;11.5;70000 +2001-09-10_02:00:00;11.6;70000 +2001-09-10_03:00:00;11.9;70000 +2001-09-10_04:00:00;12.4;70000 +2001-09-10_05:00:00;12.5;70000 +2001-09-10_06:00:00;12.9;70000 +2001-09-10_07:00:00;12.8;70000 +2001-09-10_08:00:00;13;70000 +2001-09-10_09:00:00;14.6;70000 +2001-09-10_10:00:00;14.8;70000 +2001-09-10_11:00:00;15;70000 +2001-09-10_12:00:00;15.1;70000 +2001-09-10_13:00:00;15.3;70000 +2001-09-10_14:00:00;15.3;70000 +2001-09-10_15:00:00;15.2;70000 +2001-09-10_16:00:00;15.6;70000 +2001-09-10_17:00:00;15.4;70000 +2001-09-10_18:00:00;15.4;70000 +2001-09-10_19:00:00;15;70000 +2001-09-10_20:00:00;15.6;70000 +2001-09-10_21:00:00;16.4;70000 +2001-09-10_22:00:00;16.6;70000 +2001-09-10_23:00:00;16.4;70000 +2001-09-11_00:00:00;16.3;70000 +2001-09-11_01:00:00;16.1;70000 +2001-09-11_02:00:00;15.9;70000 +2001-09-11_03:00:00;15.9;70000 +2001-09-11_04:00:00;15.9;70000 +2001-09-11_05:00:00;15.6;70000 +2001-09-11_06:00:00;15.6;70000 +2001-09-11_07:00:00;15.4;70000 +2001-09-11_08:00:00;15.7;70000 +2001-09-11_09:00:00;15.7;70000 +2001-09-11_10:00:00;15.6;70000 +2001-09-11_11:00:00;15.3;70000 +2001-09-11_12:00:00;15.2;70000 +2001-09-11_13:00:00;15.2;70000 +2001-09-11_14:00:00;15.2;70000 +2001-09-11_15:00:00;15.3;70000 +2001-09-11_16:00:00;15.4;70000 +2001-09-11_17:00:00;15.5;70000 +2001-09-11_18:00:00;15.4;70000 +2001-09-11_19:00:00;15.1;70000 +2001-09-11_20:00:00;15;70000 +2001-09-11_21:00:00;14.9;70000 +2001-09-11_22:00:00;14.9;70000 +2001-09-11_23:00:00;15;70000 +2001-09-12_00:00:00;15.1;70000 +2001-09-12_01:00:00;14.9;70000 +2001-09-12_02:00:00;14.3;70000 +2001-09-12_03:00:00;13.6;70000 +2001-09-12_04:00:00;14.2;70000 +2001-09-12_05:00:00;14.3;70000 +2001-09-12_06:00:00;15;70000 +2001-09-12_07:00:00;14.9;70000 +2001-09-12_08:00:00;15.1;70000 +2001-09-12_09:00:00;14.9;70000 +2001-09-12_10:00:00;14.8;70000 +2001-09-12_11:00:00;14.8;70000 +2001-09-12_12:00:00;13.6;70000 +2001-09-12_13:00:00;13.3;70000 +2001-09-12_14:00:00;13.6;70000 +2001-09-12_15:00:00;14.1;70000 +2001-09-12_16:00:00;15;70000 +2001-09-12_17:00:00;15.2;70000 +2001-09-12_18:00:00;14.2;70000 +2001-09-12_19:00:00;14.2;70000 +2001-09-12_20:00:00;14.1;70000 +2001-09-12_21:00:00;14.2;70000 +2001-09-12_22:00:00;14.2;70000 +2001-09-12_23:00:00;14.2;70000 +2001-09-13_00:00:00;14.3;70000 +2001-09-13_01:00:00;14.4;70000 +2001-09-13_02:00:00;14.2;70000 +2001-09-13_03:00:00;14.3;70000 +2001-09-13_04:00:00;14.2;70000 +2001-09-13_05:00:00;13.9;70000 +2001-09-13_06:00:00;13.9;70000 +2001-09-13_07:00:00;13.9;70000 +2001-09-13_08:00:00;14.2;70000 +2001-09-13_09:00:00;14.2;70000 +2001-09-13_10:00:00;14.1;70000 +2001-09-13_11:00:00;13.8;70000 +2001-09-13_12:00:00;13.5;70000 +2001-09-13_13:00:00;14.2;70000 +2001-09-13_14:00:00;14;70000 +2001-09-13_15:00:00;14.1;70000 +2001-09-13_16:00:00;13.6;70000 +2001-09-13_17:00:00;13.9;70000 +2001-09-13_18:00:00;13.5;70000 +2001-09-13_19:00:00;13.9;70000 +2001-09-13_20:00:00;14.1;70000 +2001-09-13_21:00:00;14.1;70000 +2001-09-13_22:00:00;13.9;70000 +2001-09-13_23:00:00;13.9;70000 +2001-09-14_00:00:00;13.9;70000 +2001-09-14_01:00:00;14;70000 +2001-09-14_02:00:00;14;70000 +2001-09-14_03:00:00;13.9;70000 +2001-09-14_04:00:00;14.1;70000 +2001-09-14_05:00:00;14.1;70000 +2001-09-14_06:00:00;14.1;70000 +2001-09-14_07:00:00;13.9;70000 +2001-09-14_08:00:00;13.8;70000 +2001-09-14_09:00:00;13.3;70000 +2001-09-14_10:00:00;13.2;70000 +2001-09-14_11:00:00;13.2;70000 +2001-09-14_12:00:00;12.9;70000 +2001-09-14_13:00:00;12.9;70000 +2001-09-14_14:00:00;12.9;70000 +2001-09-14_15:00:00;12.9;70000 +2001-09-14_16:00:00;12.7;70000 +2001-09-14_17:00:00;12.8;70000 +2001-09-14_18:00:00;12.5;70000 +2001-09-14_19:00:00;12.6;70000 +2001-09-14_20:00:00;12.6;70000 +2001-09-14_21:00:00;12;70000 +2001-09-14_22:00:00;11.8;70000 +2001-09-14_23:00:00;12.9;70000 +2001-09-15_00:00:00;13.3;70000 +2001-09-15_01:00:00;12.8;70000 +2001-09-15_02:00:00;11.9;70000 +2001-09-15_03:00:00;12.6;70000 +2001-09-15_04:00:00;13.2;70000 +2001-09-15_05:00:00;13.2;70000 +2001-09-15_06:00:00;12.9;70000 +2001-09-15_07:00:00;12.8;70000 +2001-09-15_08:00:00;12.5;70000 +2001-09-15_09:00:00;11.4;70000 +2001-09-15_10:00:00;12.7;70000 +2001-09-15_11:00:00;11.1;70000 +2001-09-15_12:00:00;12.2;70000 +2001-09-15_13:00:00;12.1;70000 +2001-09-15_14:00:00;12.1;70000 +2001-09-15_15:00:00;12.2;70000 +2001-09-15_16:00:00;12.2;70000 +2001-09-15_17:00:00;12.2;70000 +2001-09-15_18:00:00;12.2;70000 +2001-09-15_19:00:00;12.2;70000 +2001-09-15_20:00:00;12.1;70000 +2001-09-15_21:00:00;12.1;70000 +2001-09-15_22:00:00;12.1;70000 +2001-09-15_23:00:00;12.2;70000 +2001-09-16_00:00:00;11.2;70000 +2001-09-16_01:00:00;11.6;70000 +2001-09-16_02:00:00;11.1;70000 +2001-09-16_03:00:00;11.5;70000 +2001-09-16_04:00:00;11.3;70000 +2001-09-16_05:00:00;11.5;70000 +2001-09-16_06:00:00;11.2;70000 +2001-09-16_07:00:00;11.5;70000 +2001-09-16_08:00:00;11.2;70000 +2001-09-16_09:00:00;11.4;70000 +2001-09-16_10:00:00;11.3;70000 +2001-09-16_11:00:00;11.5;70000 +2001-09-16_12:00:00;11.7;70000 +2001-09-16_13:00:00;11.8;70000 +2001-09-16_14:00:00;11.8;70000 +2001-09-16_15:00:00;11.9;70000 +2001-09-16_16:00:00;12.1;70000 +2001-09-16_17:00:00;12.1;70000 +2001-09-16_18:00:00;11.5;70000 +2001-09-16_19:00:00;11.5;70000 +2001-09-16_20:00:00;11.7;70000 +2001-09-16_21:00:00;11.8;70000 +2001-09-16_22:00:00;11.7;70000 +2001-09-16_23:00:00;11.9;70000 +2001-09-17_00:00:00;12;70000 +2001-09-17_01:00:00;11.6;70000 +2001-09-17_02:00:00;11.8;70000 +2001-09-17_03:00:00;11.9;70000 +2001-09-17_04:00:00;12;70000 +2001-09-17_05:00:00;12;70000 +2001-09-17_06:00:00;12.3;70000 +2001-09-17_07:00:00;12.5;70000 +2001-09-17_08:00:00;12.4;70000 +2001-09-17_09:00:00;12.6;70000 +2001-09-17_10:00:00;13.2;70000 +2001-09-17_11:00:00;13.1;70000 +2001-09-17_12:00:00;13.3;70000 +2001-09-17_13:00:00;12.8;70000 +2001-09-17_14:00:00;12.6;70000 +2001-09-17_15:00:00;12.9;70000 +2001-09-17_16:00:00;12.9;70000 +2001-09-17_17:00:00;14;70000 +2001-09-17_18:00:00;14.1;70000 +2001-09-17_19:00:00;14.1;70000 +2001-09-17_20:00:00;14.2;70000 +2001-09-17_21:00:00;14.2;70000 +2001-09-17_22:00:00;14.5;70000 +2001-09-17_23:00:00;14.2;70000 +2001-09-18_00:00:00;14.3;70000 +2001-09-18_01:00:00;14.2;70000 +2001-09-18_02:00:00;14.5;70000 +2001-09-18_03:00:00;14.3;70000 +2001-09-18_04:00:00;12.8;70000 +2001-09-18_05:00:00;12.2;70000 +2001-09-18_06:00:00;12.7;70000 +2001-09-18_07:00:00;12.8;70000 +2001-09-18_08:00:00;12.8;70000 +2001-09-18_09:00:00;13.2;70000 +2001-09-18_10:00:00;13.7;70000 +2001-09-18_11:00:00;14.3;70000 +2001-09-18_12:00:00;14.3;70000 +2001-09-18_13:00:00;14.5;70000 +2001-09-18_14:00:00;14.4;70000 +2001-09-18_15:00:00;14.5;70000 +2001-09-18_16:00:00;14.5;70000 +2001-09-18_17:00:00;14.6;70000 +2001-09-18_18:00:00;14.5;70000 +2001-09-18_19:00:00;14.2;70000 +2001-09-18_20:00:00;13.3;70000 +2001-09-18_21:00:00;13.1;70000 +2001-09-18_22:00:00;12.8;70000 +2001-09-18_23:00:00;12.7;70000 +2001-09-19_00:00:00;12.6;70000 +2001-09-19_01:00:00;12.4;70000 +2001-09-19_02:00:00;12.1;70000 +2001-09-19_03:00:00;11.9;70000 +2001-09-19_04:00:00;12.1;70000 +2001-09-19_05:00:00;11;78947 +2001-09-19_06:00:00;11.6;78947 +2001-09-19_07:00:00;11.7;78947 +2001-09-19_09:00:00;13.1;78947 +2001-09-19_10:00:00;13.5;78947 +2001-09-19_11:00:00;13.7;78947 +2001-09-19_12:00:00;12.2;78947 +2001-09-19_13:00:00;12.5;78947 +2001-09-19_14:00:00;12.2;78947 +2001-09-19_15:00:00;11.9;78947 +2001-09-19_16:00:00;11.8;78947 +2001-09-19_17:00:00;11.8;78947 +2001-09-19_18:00:00;11.6;78947 +2001-09-19_19:00:00;11.7;78947 +2001-09-19_20:00:00;11.8;78947 +2001-09-19_21:00:00;11.7;78947 +2001-09-19_22:00:00;11.7;78947 +2001-09-19_23:00:00;11.5;78947 +2001-09-20_00:00:00;11.4;78947 +2001-09-20_01:00:00;11.7;78947 +2001-09-20_02:00:00;12.2;78947 +2001-09-20_03:00:00;12.7;78947 +2001-09-20_04:00:00;12.7;78947 +2001-09-20_06:00:00;12.4;78947 +2001-09-20_07:00:00;12.5;78947 +2001-09-20_08:00:00;12.9;78947 +2001-09-20_09:00:00;13.2;78947 +2001-09-20_10:00:00;13.2;78947 +2001-09-20_11:00:00;13.4;78947 +2001-09-20_12:00:00;13.9;78947 +2001-09-20_13:00:00;14.4;78947 +2001-09-20_14:00:00;14.4;78947 +2001-09-20_15:00:00;14;78947 +2001-09-20_16:00:00;13.3;78947 +2001-09-20_17:00:00;12.7;78947 +2001-09-20_18:00:00;11.7;78947 +2001-09-20_19:00:00;11.6;78947 +2001-09-20_20:00:00;11.3;78947 +2001-09-20_21:00:00;11.3;78947 +2001-09-20_22:00:00;11.2;78947 +2001-09-20_23:00:00;11.3;78947 +2001-09-21_00:00:00;10.7;78947 +2001-09-21_01:00:00;10.8;78947 +2001-09-21_02:00:00;10.8;78947 +2001-09-21_03:00:00;11.3;78947 +2001-09-21_04:00:00;11.4;78947 +2001-09-21_05:00:00;11.3;78947 +2001-09-21_06:00:00;11.5;78947 +2001-09-21_07:00:00;11.9;78947 +2001-09-21_09:00:00;13.2;78947 +2001-09-21_10:00:00;13.3;78947 +2001-09-21_11:00:00;12.9;78947 +2001-09-21_12:00:00;13.3;78947 +2001-09-21_13:00:00;13.6;78947 +2001-09-21_14:00:00;12.7;78947 +2001-09-21_15:00:00;11.7;78947 +2001-09-21_16:00:00;12;78947 +2001-09-21_17:00:00;11.7;78947 +2001-09-21_18:00:00;11.1;78947 +2001-09-21_19:00:00;11.3;78947 +2001-09-21_20:00:00;11.4;78947 +2001-09-21_21:00:00;11.4;78947 +2001-09-21_22:00:00;10.8;78947 +2001-09-21_23:00:00;10.9;78947 +2001-09-22_00:00:00;10.9;78947 +2001-09-22_01:00:00;11.2;78947 +2001-09-22_02:00:00;11.2;78947 +2001-09-22_03:00:00;11.1;78947 +2001-09-22_04:00:00;11.2;78947 +2001-09-22_05:00:00;11.1;78947 +2001-09-22_06:00:00;10.6;78947 +2001-09-22_07:00:00;10.7;78947 +2001-09-22_09:00:00;11.3;78947 +2001-09-22_10:00:00;12.3;78947 +2001-09-22_11:00:00;12.9;78947 +2001-09-22_12:00:00;11.7;78947 +2001-09-22_13:00:00;11.8;78947 +2001-09-22_14:00:00;11.6;78947 +2001-09-22_15:00:00;11.7;78947 +2001-09-22_16:00:00;11.6;78947 +2001-09-22_17:00:00;11.6;78947 +2001-09-22_18:00:00;11.2;78947 +2001-09-22_19:00:00;11.1;78947 +2001-09-22_20:00:00;11;78947 +2001-09-22_21:00:00;10.8;78947 +2001-09-22_22:00:00;10.7;78947 +2001-09-22_23:00:00;10.8;78947 +2001-09-23_00:00:00;10.7;78947 +2001-09-23_01:00:00;10.8;78947 +2001-09-23_02:00:00;10.9;78947 +2001-09-23_03:00:00;10.8;78947 +2001-09-23_04:00:00;10.7;78947 +2001-09-23_05:00:00;10.9;78947 +2001-09-23_06:00:00;10.8;78947 +2001-09-23_07:00:00;11;78947 +2001-09-23_08:00:00;11.1;78947 +2001-09-23_09:00:00;11.7;78947 +2001-09-23_10:00:00;12.2;78947 +2001-09-23_11:00:00;12.6;78947 +2001-09-23_12:00:00;11.6;78947 +2001-09-23_13:00:00;12.6;78947 +2001-09-23_14:00:00;12.8;78947 +2001-09-23_15:00:00;12.7;78947 +2001-09-23_16:00:00;12.1;78947 +2001-09-23_17:00:00;11.6;78947 +2001-09-23_18:00:00;10.5;78947 +2001-09-23_19:00:00;10.6;78947 +2001-09-23_20:00:00;11.2;78947 +2001-09-23_21:00:00;11.6;78947 +2001-09-23_22:00:00;11.8;78947 +2001-09-23_23:00:00;12;78947 +2001-09-24_00:00:00;11.8;78947 +2001-09-24_01:00:00;12.1;78947 +2001-09-24_02:00:00;12.2;78947 +2001-09-24_03:00:00;12.4;78947 +2001-09-24_04:00:00;12.5;78947 +2001-09-24_05:00:00;12.7;78947 +2001-09-24_06:00:00;12.5;78947 +2001-09-24_07:00:00;12.8;78947 +2001-09-24_08:00:00;13.4;78947 +2001-09-24_09:00:00;13.6;78947 +2001-09-24_10:00:00;13.9;78947 +2001-09-24_11:00:00;15;78947 +2001-09-24_12:00:00;14.7;78947 +2001-09-24_13:00:00;14.6;78947 +2001-09-24_14:00:00;14.2;78947 +2001-09-24_15:00:00;13.8;78947 +2001-09-24_16:00:00;13.7;78947 +2001-09-24_17:00:00;13.5;78947 +2001-09-24_18:00:00;13.3;78947 +2001-09-24_19:00:00;13.1;78947 +2001-09-24_20:00:00;13;78947 +2001-09-24_21:00:00;12.9;78947 +2001-09-24_22:00:00;12.9;78947 +2001-09-24_23:00:00;12.8;78947 +2001-09-25_00:00:00;12.4;78947 +2001-09-25_01:00:00;12.4;78947 +2001-09-25_02:00:00;12.4;78947 +2001-09-25_03:00:00;12.4;78947 +2001-09-25_04:00:00;12.4;78947 +2001-09-25_05:00:00;12.5;78947 +2001-09-25_06:00:00;13.1;78947 +2001-09-25_07:00:00;13.3;78947 +2001-09-25_08:00:00;13.7;78947 +2001-09-25_09:00:00;13.8;78947 +2001-09-25_10:00:00;13.9;78947 +2001-09-25_12:00:00;15.2;78947 +2001-09-25_13:00:00;15.5;78947 +2001-09-25_14:00:00;15.8;78947 +2001-09-25_15:00:00;15.2;78947 +2001-09-25_16:00:00;14.5;78947 +2001-09-25_17:00:00;14;78947 +2001-09-25_18:00:00;14.9;78947 +2001-09-25_19:00:00;14.7;78947 +2001-09-25_20:00:00;14.3;78947 +2001-09-25_21:00:00;13.3;78947 +2001-09-25_22:00:00;13.3;78947 +2001-09-25_23:00:00;13.5;78947 +2001-09-26_00:00:00;13.1;78947 +2001-09-26_01:00:00;13.2;78947 +2001-09-26_02:00:00;13.3;78947 +2001-09-26_03:00:00;13.3;78947 +2001-09-26_04:00:00;13.4;78947 +2001-09-26_05:00:00;13.5;78947 +2001-09-26_06:00:00;13.7;78947 +2001-09-26_07:00:00;13.4;78947 +2001-09-26_08:00:00;13.2;78947 +2001-09-26_09:00:00;12.9;78947 +2001-09-26_10:00:00;12.5;78947 +2001-09-26_11:00:00;13.4;78947 +2001-09-26_12:00:00;13.3;78947 +2001-09-26_13:00:00;13.7;78947 +2001-09-26_14:00:00;13.5;78947 +2001-09-26_15:00:00;13;78947 +2001-09-26_16:00:00;12.5;78947 +2001-09-26_17:00:00;12.4;78947 +2001-09-26_18:00:00;12;78947 +2001-09-26_19:00:00;11.7;78947 +2001-09-26_20:00:00;11.5;78947 +2001-09-26_21:00:00;11.4;78947 +2001-09-26_22:00:00;11.3;78947 +2001-09-26_23:00:00;11.1;78947 +2001-09-27_00:00:00;10.8;78947 +2001-09-27_01:00:00;10.5;78947 +2001-09-27_02:00:00;10.1;78947 +2001-09-27_03:00:00;9.8;78947 +2001-09-27_04:00:00;9.7;78947 +2001-09-27_05:00:00;10.1;78947 +2001-09-27_06:00:00;9.5;78947 +2001-09-27_07:00:00;10.9;78947 +2001-09-27_09:00:00;12.5;78947 +2001-09-27_10:00:00;13.2;78947 +2001-09-27_11:00:00;13.9;78947 +2001-09-27_12:00:00;13.8;78947 +2001-09-27_13:00:00;13.7;78947 +2001-09-27_14:00:00;13.5;78947 +2001-09-27_15:00:00;12.9;78947 +2001-09-27_16:00:00;12.6;78947 +2001-09-27_17:00:00;12.4;78947 +2001-09-27_18:00:00;12.2;78947 +2001-09-27_19:00:00;12.3;78947 +2001-09-27_20:00:00;12.5;78947 +2001-09-27_21:00:00;12.7;78947 +2001-09-27_22:00:00;12.8;78947 +2001-09-27_23:00:00;12.9;78947 +2001-09-28_00:00:00;13;78947 +2001-09-28_01:00:00;13.1;78947 +2001-09-28_02:00:00;13.2;78947 +2001-09-28_03:00:00;13.3;78947 +2001-09-28_04:00:00;13.4;78947 +2001-09-28_05:00:00;13.4;78947 +2001-09-28_06:00:00;13.3;78947 +2001-09-28_07:00:00;13.3;78947 +2001-09-28_08:00:00;13.5;78947 +2001-09-28_09:00:00;14.1;78947 +2001-09-28_10:00:00;14.5;78947 +2001-09-28_11:00:00;14.2;78947 +2001-09-28_12:00:00;15.2;78947 +2001-09-28_13:00:00;14.6;78947 +2001-09-28_14:00:00;14.2;78947 +2001-09-28_15:00:00;14;78947 +2001-09-28_16:00:00;14.1;78947 +2001-09-28_17:00:00;14.1;78947 +2001-09-28_18:00:00;14.1;78947 +2001-09-28_19:00:00;14.1;78947 +2001-09-28_20:00:00;14;78947 +2001-09-28_21:00:00;13.9;78947 +2001-09-28_22:00:00;13.9;78947 +2001-09-28_23:00:00;13.8;78947 +2001-09-29_00:00:00;14.1;78947 +2001-09-29_01:00:00;14.1;78947 +2001-09-29_02:00:00;14;78947 +2001-09-29_03:00:00;13.7;78947 +2001-09-29_04:00:00;13.3;78947 +2001-09-29_05:00:00;13;78947 +2001-09-29_06:00:00;12.9;78947 +2001-09-29_07:00:00;13.1;78947 +2001-09-29_09:00:00;14.1;78947 +2001-09-29_10:00:00;14.7;78947 +2001-09-29_11:00:00;15.3;78947 +2001-09-29_12:00:00;14.1;78947 +2001-09-29_13:00:00;14.2;78947 +2001-09-29_14:00:00;14.6;78947 +2001-09-29_15:00:00;14.7;78947 +2001-09-29_16:00:00;14.5;78947 +2001-09-29_17:00:00;14.3;78947 +2001-09-29_18:00:00;14.1;78947 +2001-09-29_19:00:00;14.1;78947 +2001-09-29_20:00:00;14.1;78947 +2001-09-29_21:00:00;14.1;78947 +2001-09-29_22:00:00;14;78947 +2001-09-29_23:00:00;13.5;78947 +2001-09-30_00:00:00;13.2;78947 +2001-09-30_01:00:00;13;78947 +2001-09-30_02:00:00;12.9;78947 +2001-09-30_03:00:00;12.8;78947 +2001-09-30_04:00:00;12.7;78947 +2001-09-30_05:00:00;12.7;78947 +2001-09-30_06:00:00;12.9;78947 +2001-09-30_07:00:00;13;78947 +2001-09-30_09:00:00;13.9;78947 +2001-09-30_10:00:00;14.7;78947 +2001-09-30_11:00:00;15.4;78947 +2001-09-30_12:00:00;16.7;78947 +2001-09-30_13:00:00;16.7;78947 +2001-09-30_14:00:00;16.6;78947 +2001-09-30_15:00:00;16.4;78947 +2001-09-30_16:00:00;16.1;78947 +2001-09-30_17:00:00;15.3;78947 +2001-09-30_18:00:00;14.9;78947 +2001-09-30_19:00:00;14.8;78947 +2001-09-30_20:00:00;14.9;78947 +2001-09-30_21:00:00;14.8;78947 +2001-09-30_22:00:00;14.7;78947 +2001-09-30_23:00:00;14.7;78947 +2001-10-01_00:00:00;14.1;78947 +2001-10-01_01:00:00;14.2;78947 +2001-10-01_02:00:00;14.2;78947 +2001-10-01_03:00:00;14.1;78947 +2001-10-01_04:00:00;13.8;78947 +2001-10-01_05:00:00;13.6;78947 +2001-10-01_06:00:00;13.4;78947 +2001-10-01_07:00:00;13.8;78947 +2001-10-01_08:00:00;14.6;78947 +2001-10-01_09:00:00;15.5;78947 +2001-10-01_10:00:00;16.2;78947 +2001-10-01_11:00:00;16.7;78947 +2001-10-01_12:00:00;17.7;78947 +2001-10-01_13:00:00;18;78947 +2001-10-01_14:00:00;17.8;78947 +2001-10-01_15:00:00;17.2;78947 +2001-10-01_16:00:00;16.2;78947 +2001-10-01_17:00:00;15.3;78947 +2001-10-01_18:00:00;14.9;78947 +2001-10-01_19:00:00;14.8;78947 +2001-10-01_20:00:00;14.5;78947 +2001-10-01_21:00:00;14.2;78947 +2001-10-01_22:00:00;14.1;78947 +2001-10-01_23:00:00;14;78947 +2001-10-02_00:00:00;14.1;78947 +2001-10-02_01:00:00;13.9;78947 +2001-10-02_02:00:00;13.6;78947 +2001-10-02_03:00:00;13.6;78947 +2001-10-02_04:00:00;13.5;78947 +2001-10-02_05:00:00;13.5;78947 +2001-10-02_06:00:00;13.8;78947 +2001-10-02_07:00:00;13.9;78947 +2001-10-02_08:00:00;14;78947 +2001-10-02_09:00:00;14.4;78947 +2001-10-02_10:00:00;14.4;78947 +2001-10-02_11:00:00;14.1;78947 +2001-10-02_12:00:00;15;78947 +2001-10-02_13:00:00;14.7;78947 +2001-10-02_14:00:00;13.9;78947 +2001-10-02_15:00:00;13.4;78947 +2001-10-02_16:00:00;13;78947 +2001-10-02_17:00:00;12.7;78947 +2001-10-02_18:00:00;12.3;78947 +2001-10-02_19:00:00;12;78947 +2001-10-02_20:00:00;11.3;78947 +2001-10-02_21:00:00;10.6;78947 +2001-10-02_22:00:00;11.4;78947 +2001-10-02_23:00:00;11.1;78947 +2001-10-03_00:00:00;10.9;78947 +2001-10-03_01:00:00;11.3;78947 +2001-10-03_02:00:00;11.6;78947 +2001-10-03_03:00:00;11.8;78947 +2001-10-03_04:00:00;11.3;78947 +2001-10-03_05:00:00;11.7;78947 +2001-10-03_06:00:00;12.2;78947 +2001-10-03_07:00:00;12.8;78947 +2001-10-03_08:00:00;13.3;78947 +2001-10-03_09:00:00;13.1;78947 +2001-10-03_10:00:00;13.4;78947 +2001-10-03_11:00:00;13.7;78947 +2001-10-03_12:00:00;13.8;78947 +2001-10-03_13:00:00;13.9;78947 +2001-10-03_14:00:00;13.6;78947 +2001-10-03_15:00:00;13.9;78947 +2001-10-03_16:00:00;14.2;78947 +2001-10-03_17:00:00;14.3;78947 +2001-10-03_18:00:00;13.9;78947 +2001-10-03_19:00:00;14.1;78947 +2001-10-03_20:00:00;14.4;78947 +2001-10-03_21:00:00;12.9;78947 +2001-10-03_22:00:00;12.8;78947 +2001-10-03_23:00:00;13.1;78947 +2001-10-04_00:00:00;12.1;78947 +2001-10-04_01:00:00;12.3;78947 +2001-10-04_02:00:00;11.9;78947 +2001-10-04_03:00:00;11.5;78947 +2001-10-04_04:00:00;11.7;78947 +2001-10-04_05:00:00;12.3;78947 +2001-10-04_06:00:00;11;78947 +2001-10-04_07:00:00;11.1;78947 +2001-10-04_09:00:00;11.5;78947 +2001-10-04_10:00:00;11.6;78947 +2001-10-04_11:00:00;12;78947 +2001-10-04_12:00:00;11.6;78947 +2001-10-04_13:00:00;12.2;78947 +2001-10-04_14:00:00;13;78947 +2001-10-04_15:00:00;12;78947 +2001-10-04_16:00:00;12.4;78947 +2001-10-04_17:00:00;11.5;78947 +2001-10-04_18:00:00;11.1;78947 +2001-10-04_19:00:00;11.2;78947 +2001-10-04_20:00:00;11.8;78947 +2001-10-04_21:00:00;11.5;78947 +2001-10-04_22:00:00;11.4;78947 +2001-10-04_23:00:00;11.2;78947 +2001-10-05_00:00:00;10.5;78947 +2001-10-05_01:00:00;10.8;78947 +2001-10-05_02:00:00;10.7;78947 +2001-10-05_03:00:00;10.9;78947 +2001-10-05_04:00:00;11.1;78947 +2001-10-05_05:00:00;10.9;78947 +2001-10-05_06:00:00;10.8;78947 +2001-10-05_07:00:00;10.9;78947 +2001-10-05_09:00:00;12.2;78947 +2001-10-05_10:00:00;12.8;78947 +2001-10-05_11:00:00;12.6;78947 +2001-10-05_12:00:00;11.9;78947 +2001-10-05_13:00:00;12.3;78947 +2001-10-05_14:00:00;13;78947 +2001-10-05_15:00:00;13.4;78947 +2001-10-05_16:00:00;13.5;78947 +2001-10-05_17:00:00;13.6;78947 +2001-10-05_18:00:00;13.6;78947 +2001-10-05_19:00:00;13.7;78947 +2001-10-05_20:00:00;13.7;78947 +2001-10-05_21:00:00;13.6;78947 +2001-10-05_22:00:00;13.4;78947 +2001-10-05_23:00:00;13.3;78947 +2001-10-06_00:00:00;12.4;78947 +2001-10-06_01:00:00;12;78947 +2001-10-06_02:00:00;11.9;78947 +2001-10-06_03:00:00;11.9;78947 +2001-10-06_04:00:00;11.2;78947 +2001-10-06_05:00:00;11;78947 +2001-10-06_06:00:00;9.5;78947 +2001-10-06_07:00:00;9.7;78947 +2001-10-06_08:00:00;10.2;78947 +2001-10-06_09:00:00;10.9;78947 +2001-10-06_10:00:00;11.1;78947 +2001-10-06_11:00:00;11.3;78947 +2001-10-06_12:00:00;10.5;78947 +2001-10-06_13:00:00;9.8;78947 +2001-10-06_14:00:00;10;78947 +2001-10-06_15:00:00;10.1;78947 +2001-10-06_16:00:00;10.1;78947 +2001-10-06_17:00:00;9.9;78947 +2001-10-06_18:00:00;9.1;78947 +2001-10-06_19:00:00;8.8;78947 +2001-10-06_20:00:00;8.5;78947 +2001-10-06_21:00:00;9;78947 +2001-10-06_22:00:00;9;78947 +2001-10-06_23:00:00;9.1;78947 +2001-10-07_00:00:00;9.2;78947 +2001-10-07_01:00:00;9.6;78947 +2001-10-07_02:00:00;9.5;78947 +2001-10-07_03:00:00;9.9;78947 +2001-10-07_04:00:00;10.6;78947 +2001-10-07_05:00:00;10.1;78947 +2001-10-07_06:00:00;9.5;78947 +2001-10-07_07:00:00;9.4;78947 +2001-10-07_09:00:00;10.7;78947 +2001-10-07_10:00:00;10.7;78947 +2001-10-07_11:00:00;9.5;78947 +2001-10-07_12:00:00;9.9;78947 +2001-10-07_13:00:00;9.8;78947 +2001-10-07_14:00:00;10.5;78947 +2001-10-07_15:00:00;9.1;78947 +2001-10-07_16:00:00;9.5;78947 +2001-10-07_17:00:00;9.5;78947 +2001-10-07_18:00:00;8.1;78947 +2001-10-07_19:00:00;9.1;78947 +2001-10-07_20:00:00;9.3;78947 +2001-10-07_21:00:00;9.3;78947 +2001-10-07_22:00:00;9.2;78947 +2001-10-07_23:00:00;9.2;78947 +2001-10-08_00:00:00;7.8;78947 +2001-10-08_01:00:00;7.7;78947 +2001-10-08_02:00:00;7.9;78947 +2001-10-08_03:00:00;7.6;78947 +2001-10-08_04:00:00;7.8;78947 +2001-10-08_05:00:00;7.9;78947 +2001-10-08_06:00:00;7.1;78947 +2001-10-08_07:00:00;6.8;78947 +2001-10-08_08:00:00;7.1;78947 +2001-10-08_09:00:00;8.2;78947 +2001-10-08_10:00:00;9.3;78947 +2001-10-08_11:00:00;10;78947 +2001-10-08_12:00:00;9.8;78947 +2001-10-08_13:00:00;10.2;78947 +2001-10-08_14:00:00;10.2;78947 +2001-10-08_15:00:00;10;78947 +2001-10-08_16:00:00;9.6;78947 +2001-10-08_17:00:00;8.5;78947 +2001-10-08_18:00:00;5.9;78947 +2001-10-08_19:00:00;4.4;78947 +2001-10-08_20:00:00;3.5;78947 +2001-10-08_21:00:00;3.6;78947 +2001-10-08_22:00:00;4.6;78947 +2001-10-08_23:00:00;6.2;78947 +2001-10-09_00:00:00;8;78947 +2001-10-09_01:00:00;8.8;78947 +2001-10-09_02:00:00;8.9;78947 +2001-10-09_03:00:00;8.8;78947 +2001-10-09_04:00:00;8.9;78947 +2001-10-09_05:00:00;8.9;78947 +2001-10-09_06:00:00;9.6;78947 +2001-10-09_07:00:00;9.5;78947 +2001-10-09_08:00:00;9.1;78947 +2001-10-09_09:00:00;9.2;78947 +2001-10-09_10:00:00;9.4;78947 +2001-10-09_11:00:00;9.7;78947 +2001-10-09_12:00:00;9.7;78947 +2001-10-09_13:00:00;10.5;78947 +2001-10-09_14:00:00;11.2;78947 +2001-10-09_15:00:00;10.2;78947 +2001-10-09_16:00:00;9.9;78947 +2001-10-09_17:00:00;10.2;78947 +2001-10-09_18:00:00;10.8;78947 +2001-10-09_19:00:00;11.1;78947 +2001-10-09_20:00:00;10.9;78947 +2001-10-09_21:00:00;10.6;78947 +2001-10-09_22:00:00;10.4;78947 +2001-10-09_23:00:00;10.2;78947 +2001-10-10_00:00:00;9.8;78947 +2001-10-10_01:00:00;9.9;78947 +2001-10-10_02:00:00;9.9;78947 +2001-10-10_03:00:00;9.9;78947 +2001-10-10_04:00:00;10;78947 +2001-10-10_05:00:00;10.2;78947 +2001-10-10_06:00:00;10.2;78947 +2001-10-10_07:00:00;10.3;78947 +2001-10-10_08:00:00;10;78947 +2001-10-10_09:00:00;9.7;78947 +2001-10-10_10:00:00;10.7;78947 +2001-10-10_11:00:00;10.2;78947 +2001-10-10_12:00:00;10.5;78947 +2001-10-10_13:00:00;10.4;78947 +2001-10-10_14:00:00;10.9;78947 +2001-10-10_15:00:00;11;78947 +2001-10-10_16:00:00;10.4;78947 +2001-10-10_17:00:00;10.1;78947 +2001-10-10_18:00:00;9.5;78947 +2001-10-10_19:00:00;9.5;78947 +2001-10-10_20:00:00;9.7;78947 +2001-10-10_21:00:00;9.7;78947 +2001-10-10_22:00:00;9.8;78947 +2001-10-10_23:00:00;9.8;78947 +2001-10-11_00:00:00;9.6;78947 +2001-10-11_01:00:00;9.3;78947 +2001-10-11_02:00:00;9.7;78947 +2001-10-11_03:00:00;10;78947 +2001-10-11_04:00:00;9.8;78947 +2001-10-11_05:00:00;9.6;78947 +2001-10-11_06:00:00;9.2;78947 +2001-10-11_07:00:00;9.2;78947 +2001-10-11_08:00:00;9.4;78947 +2001-10-11_09:00:00;10.3;78947 +2001-10-11_10:00:00;9.6;78947 +2001-10-11_11:00:00;10.3;78947 +2001-10-11_12:00:00;10.9;78947 +2001-10-11_13:00:00;11;78947 +2001-10-11_14:00:00;10.7;78947 +2001-10-11_15:00:00;8.7;78947 +2001-10-11_16:00:00;8.7;78947 +2001-10-11_17:00:00;8.8;78947 +2001-10-11_18:00:00;8.2;78947 +2001-10-11_19:00:00;8.6;78947 +2001-10-11_20:00:00;8.7;78947 +2001-10-11_21:00:00;9.2;78947 +2001-10-11_22:00:00;9.3;78947 +2001-10-11_23:00:00;9.1;78947 +2001-10-12_00:00:00;9.3;78947 +2001-10-12_01:00:00;9.4;78947 +2001-10-12_02:00:00;9.6;78947 +2001-10-12_03:00:00;9.7;78947 +2001-10-12_04:00:00;9.6;78947 +2001-10-12_05:00:00;9.3;78947 +2001-10-12_06:00:00;9.1;78947 +2001-10-12_07:00:00;9.2;78947 +2001-10-12_08:00:00;9.3;78947 +2001-10-12_11:00:00;9.5;78947 +2001-10-12_12:00:00;10.6;78947 +2001-10-12_13:00:00;10.5;78947 +2001-10-12_14:00:00;11;78947 +2001-10-12_15:00:00;10.6;78947 +2001-10-12_16:00:00;10.1;78947 +2001-10-12_17:00:00;9.8;78947 +2001-10-12_18:00:00;8.7;78947 +2001-10-12_19:00:00;8.4;78947 +2001-10-12_20:00:00;8;78947 +2001-10-12_21:00:00;7.8;78947 +2001-10-12_22:00:00;7.3;78947 +2001-10-12_23:00:00;6.8;78947 +2001-10-13_00:00:00;6.4;78947 +2001-10-13_01:00:00;6.1;78947 +2001-10-13_02:00:00;5.5;78947 +2001-10-13_03:00:00;4.6;78947 +2001-10-13_04:00:00;3.5;78947 +2001-10-13_05:00:00;2.7;78947 +2001-10-13_06:00:00;2.2;78947 +2001-10-13_07:00:00;2;78947 +2001-10-13_09:00:00;6.4;78947 +2001-10-13_10:00:00;9.4;78947 +2001-10-13_11:00:00;10.5;78947 +2001-10-13_12:00:00;10.3;78947 +2001-10-13_13:00:00;10.4;78947 +2001-10-13_14:00:00;10.3;78947 +2001-10-13_15:00:00;9.9;78947 +2001-10-13_16:00:00;8.7;78947 +2001-10-13_17:00:00;6.7;78947 +2001-10-13_18:00:00;5.2;78947 +2001-10-13_19:00:00;4.6;78947 +2001-10-13_20:00:00;4.7;78947 +2001-10-13_21:00:00;4.7;78947 +2001-10-13_22:00:00;4.7;78947 +2001-10-13_23:00:00;5.4;78947 +2001-10-14_00:00:00;6.1;78947 +2001-10-14_01:00:00;6.7;78947 +2001-10-14_02:00:00;7.6;78947 +2001-10-14_03:00:00;8.1;78947 +2001-10-14_04:00:00;8.3;78947 +2001-10-14_06:00:00;9.5;78947 +2001-10-14_07:00:00;9.6;78947 +2001-10-14_09:00:00;9.9;78947 +2001-10-14_10:00:00;9.9;78947 +2001-10-14_11:00:00;10.3;78947 +2001-10-14_12:00:00;11.2;78947 +2001-10-14_13:00:00;11.1;78947 +2001-10-14_14:00:00;11.2;78947 +2001-10-14_15:00:00;11.3;78947 +2001-10-14_16:00:00;11.1;78947 +2001-10-14_17:00:00;10.4;78947 +2001-10-14_18:00:00;10.2;78947 +2001-10-14_19:00:00;10.3;78947 +2001-10-14_20:00:00;9.8;78947 +2001-10-14_21:00:00;9.6;78947 +2001-10-14_22:00:00;9.6;78947 +2001-10-14_23:00:00;9.8;78947 +2001-10-15_00:00:00;9.9;78947 +2001-10-15_01:00:00;10.3;78947 +2001-10-15_02:00:00;10.4;78947 +2001-10-15_03:00:00;10.5;78947 +2001-10-15_04:00:00;10.6;78947 +2001-10-15_05:00:00;10.8;78947 +2001-10-15_06:00:00;9.7;78947 +2001-10-15_07:00:00;9.9;78947 +2001-10-15_08:00:00;10.4;78947 +2001-10-15_09:00:00;10.6;78947 +2001-10-15_10:00:00;11;78947 +2001-10-15_11:00:00;11.1;78947 +2001-10-15_12:00:00;12.1;78947 +2001-10-15_13:00:00;11.1;78947 +2001-10-15_14:00:00;11.3;78947 +2001-10-15_15:00:00;11;78947 +2001-10-15_16:00:00;10.5;78947 +2001-10-15_17:00:00;10.3;78947 +2001-10-15_18:00:00;10.4;78947 +2001-10-15_19:00:00;10.1;78947 +2001-10-15_20:00:00;10.2;78947 +2001-10-15_21:00:00;10.1;78947 +2001-10-15_22:00:00;9.9;78947 +2001-10-15_23:00:00;10.1;78947 +2001-10-16_00:00:00;9.8;78947 +2001-10-16_01:00:00;9.8;78947 +2001-10-16_02:00:00;9.9;78947 +2001-10-16_03:00:00;9.9;78947 +2001-10-16_04:00:00;10.1;78947 +2001-10-16_05:00:00;10.3;78947 +2001-10-16_06:00:00;10.4;78947 +2001-10-16_07:00:00;10.4;78947 +2001-10-16_08:00:00;10.6;78947 +2001-10-16_09:00:00;11;78947 +2001-10-16_10:00:00;11.5;78947 +2001-10-16_11:00:00;11.7;78947 +2001-10-16_12:00:00;11.6;78947 +2001-10-16_13:00:00;11.5;78947 +2001-10-16_14:00:00;11.3;78947 +2001-10-16_15:00:00;11.1;78947 +2001-10-16_16:00:00;11.1;78947 +2001-10-16_17:00:00;11.1;78947 +2001-10-16_18:00:00;10.8;78947 +2001-10-16_19:00:00;11;78947 +2001-10-16_20:00:00;11.2;78947 +2001-10-16_21:00:00;11.3;78947 +2001-10-16_22:00:00;11.4;78947 +2001-10-16_23:00:00;11.4;78947 +2001-10-17_00:00:00;10.8;78947 +2001-10-17_01:00:00;10.5;78947 +2001-10-17_02:00:00;10.9;78947 +2001-10-17_03:00:00;11.1;78947 +2001-10-17_04:00:00;11.1;78947 +2001-10-17_05:00:00;10.7;78947 +2001-10-17_06:00:00;10.7;78947 +2001-10-17_07:00:00;10.5;78947 +2001-10-17_09:00:00;11.8;78947 +2001-10-17_10:00:00;11.5;78947 +2001-10-17_11:00:00;11.5;78947 +2001-10-17_12:00:00;11.4;78947 +2001-10-17_13:00:00;11.3;78947 +2001-10-17_14:00:00;11.1;78947 +2001-10-17_15:00:00;11;78947 +2001-10-17_16:00:00;10.8;78947 +2001-10-17_17:00:00;10.8;78947 +2001-10-17_18:00:00;10.5;78947 +2001-10-17_19:00:00;10.6;78947 +2001-10-17_20:00:00;10.7;78947 +2001-10-17_21:00:00;10.5;78947 +2001-10-17_22:00:00;10.6;78947 +2001-10-17_23:00:00;11.1;78947 +2001-10-18_00:00:00;8.5;78947 +2001-10-18_01:00:00;6.8;78947 +2001-10-18_02:00:00;7.5;78947 +2001-10-18_03:00:00;8.7;78947 +2001-10-18_04:00:00;8.6;78947 +2001-10-18_05:00:00;8.7;78947 +2001-10-18_06:00:00;8.5;78947 +2001-10-18_07:00:00;8.5;78947 +2001-10-18_09:00:00;9.5;78947 +2001-10-18_10:00:00;9.5;78947 +2001-10-18_11:00:00;9.9;78947 +2001-10-18_12:00:00;9.7;78947 +2001-10-18_13:00:00;9.1;78947 +2001-10-18_14:00:00;9.3;78947 +2001-10-18_15:00:00;9;78947 +2001-10-18_16:00:00;8.5;78947 +2001-10-18_17:00:00;8.3;78947 +2001-10-18_18:00:00;8.2;78947 +2001-10-18_19:00:00;8.3;78947 +2001-10-18_20:00:00;8.1;78947 +2001-10-18_21:00:00;7.6;78947 +2001-10-18_22:00:00;6.2;78947 +2001-10-18_23:00:00;6.7;78947 +2001-10-19_00:00:00;7.4;78947 +2001-10-19_01:00:00;7.1;78947 +2001-10-19_02:00:00;7.8;78947 +2001-10-19_03:00:00;8;78947 +2001-10-19_04:00:00;7.6;78947 +2001-10-19_05:00:00;6.3;78947 +2001-10-19_06:00:00;7.2;78947 +2001-10-19_07:00:00;8;78947 +2001-10-19_09:00:00;7.6;78947 +2001-10-19_10:00:00;7.6;78947 +2001-10-19_11:00:00;7.6;78947 +2001-10-19_12:00:00;6.9;78947 +2001-10-19_13:00:00;6.8;78947 +2001-10-19_14:00:00;6.6;78947 +2001-10-19_15:00:00;6.4;78947 +2001-10-19_16:00:00;6.3;78947 +2001-10-19_17:00:00;6.2;78947 +2001-10-19_18:00:00;6.9;78947 +2001-10-19_19:00:00;6.6;78947 +2001-10-19_20:00:00;6.4;78947 +2001-10-19_21:00:00;6;78947 +2001-10-19_22:00:00;5.9;78947 +2001-10-19_23:00:00;5.7;78947 +2001-10-20_00:00:00;6.1;78947 +2001-10-20_01:00:00;6.2;78947 +2001-10-20_02:00:00;7.1;78947 +2001-10-20_03:00:00;6.4;78947 +2001-10-20_04:00:00;6.3;78947 +2001-10-20_05:00:00;6.6;78947 +2001-10-20_06:00:00;7;78947 +2001-10-20_07:00:00;6.1;78947 +2001-10-20_09:00:00;5.6;78947 +2001-10-20_10:00:00;6.1;78947 +2001-10-20_11:00:00;6.6;78947 +2001-10-20_12:00:00;6.8;78947 +2001-10-20_13:00:00;6.4;78947 +2001-10-20_14:00:00;6.7;78947 +2001-10-20_15:00:00;6.2;78947 +2001-10-20_16:00:00;5.7;78947 +2001-10-20_17:00:00;5.2;78947 +2001-10-20_18:00:00;4;78947 +2001-10-20_19:00:00;3.1;78947 +2001-10-20_20:00:00;3.1;78947 +2001-10-20_21:00:00;3.1;78947 +2001-10-20_22:00:00;2.1;78947 +2001-10-20_23:00:00;2;78947 +2001-10-21_00:00:00;2.2;78947 +2001-10-21_01:00:00;3.7;78947 +2001-10-21_02:00:00;5.1;78947 +2001-10-21_03:00:00;6;78947 +2001-10-21_04:00:00;6.4;78947 +2001-10-21_05:00:00;6.7;78947 +2001-10-21_06:00:00;6.9;78947 +2001-10-21_07:00:00;7.3;78947 diff --git a/migrations/utils/email.go b/migrations/utils/email.go new file mode 100644 index 0000000..17ba488 --- /dev/null +++ b/migrations/utils/email.go @@ -0,0 +1,60 @@ +package utils + +import ( + "encoding/base64" + "fmt" + "log/slog" + "net/smtp" + "os" + "runtime/debug" + "strings" +) + +func sendEmail(subject, body string, to []string) { + // server and from/to + host := "aspmx.l.google.com" + port := "25" + from := "oda-noreply@met.no" + + // add stuff to headers and make the message body + header := make(map[string]string) + header["From"] = from + header["To"] = strings.Join(to, ",") + header["Subject"] = subject + header["MIME-Version"] = "1.0" + header["Content-Type"] = "text/plain; charset=\"utf-8\"" + header["Content-Transfer-Encoding"] = "base64" + message := "" + for k, v := range header { + message += fmt.Sprintf("%s: %s\r\n", k, v) + } + + body = body + "\n\n" + fmt.Sprintf("Ran with the following command:\n%s", strings.Join(os.Args, " ")) + message += "\r\n" + base64.StdEncoding.EncodeToString([]byte(body)) + + // send the email + err := smtp.SendMail(host+":"+port, nil, from, to, []byte(message)) + if err != nil { + slog.Error(err.Error()) + return + } + slog.Info("Email sent successfully!") +} + +// TODO: modify this to be more flexible +// send an email and resume the panic +func SendEmailOnPanic(function string, recipients []string) { + if r := recover(); r != nil { + if recipients != nil { + body := "KDVH importer was unable to finish successfully, and the error was not handled." + + " This email is sent from a recover function triggered in " + + function + + ".\n\nError message:" + + fmt.Sprint(r) + + "\n\nStack trace:\n\n" + + string(debug.Stack()) + sendEmail("LARD – KDVH importer panicked", body, recipients) + } + panic(r) + } +} diff --git a/migrations/utils/utils.go b/migrations/utils/utils.go new file mode 100644 index 0000000..d8c7085 --- /dev/null +++ b/migrations/utils/utils.go @@ -0,0 +1,62 @@ +package utils + +import ( + "fmt" + "log" + "log/slog" + "os" + "slices" + + "github.com/schollz/progressbar/v3" +) + +func NewBar(size int, description string) *progressbar.ProgressBar { + return progressbar.NewOptions(size, + progressbar.OptionOnCompletion(func() { fmt.Println() }), + progressbar.OptionSetDescription(description), + progressbar.OptionShowCount(), + progressbar.OptionSetPredictTime(false), + progressbar.OptionShowElapsedTimeOnFinish(), + progressbar.OptionSetTheme(progressbar.Theme{ + Saucer: "=", + SaucerHead: ">", + SaucerPadding: " ", + BarStart: "[", + BarEnd: "]", + }), + ) +} + +// Filters elements of a slice by comparing them to the elements of a reference slice. +// formatMsg is an optional format string with a single format argument that can be used +// to add context on why the element may be missing from the reference slice +func FilterSlice[T comparable](slice, reference []T, formatMsg string) []T { + if slice == nil { + return reference + } + + if formatMsg == "" { + formatMsg = "User input '%s' not present in reference, skipping" + } + + // I hate this so much + out := slice[:0] + for _, s := range slice { + if !slices.Contains(reference, s) { + slog.Warn(fmt.Sprintf(formatMsg, s)) + continue + } + out = append(out, s) + } + return out +} + +func SetLogFile(tableName, procedure string) { + filename := fmt.Sprintf("%s_%s_log.txt", tableName, procedure) + fh, err := os.Create(filename) + if err != nil { + slog.Error(fmt.Sprintf("Could not create log '%s': %s", filename, err)) + return + } + log.SetOutput(fh) +}