From 681b9d13a24f7127366b7d5e1d5c5d550b0339de Mon Sep 17 00:00:00 2001 From: Manuel Carrer Date: Thu, 14 Nov 2024 13:44:23 +0100 Subject: [PATCH] Change name of ConvertFunction and add some docs --- migrations/kdvh/import_functions.go | 24 ++++++++++++++---------- migrations/kdvh/main.go | 16 ++++++++-------- migrations/kdvh/table.go | 3 ++- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/migrations/kdvh/import_functions.go b/migrations/kdvh/import_functions.go index 0ab06a7..cafebe7 100644 --- a/migrations/kdvh/import_functions.go +++ b/migrations/kdvh/import_functions.go @@ -250,7 +250,8 @@ func (obs *KdvhObs) Useinfo() string { // and `useinfo` generated by Kvalobs for the observation, based on `Obs.Flags` and `Obs.Data` // Different KDVH tables need different ways to perform this conversion. -func makeDataPage(obs KdvhObs) (LardObs, error) { +// Default ConvertFunction +func Convert(obs KdvhObs) (LardObs, error) { var valPtr *float32 controlinfo := VALUE_PASSED_QC @@ -283,9 +284,10 @@ func makeDataPage(obs KdvhObs) (LardObs, error) { }, nil } -// modify obstimes to always use totime -func makeDataPageProduct(obs KdvhObs) (LardObs, error) { - obsLard, err := makeDataPage(obs) +// This function modifies obstimes to always use totime +// This is needed because KDVH used incorrect and incosistent timestamps +func ConvertProduct(obs KdvhObs) (LardObs, error) { + obsLard, err := Convert(obs) if !obs.offset.IsZero() { if temp, ok := obs.offset.AddTo(obsLard.Obstime); ok { obsLard.Obstime = temp @@ -294,7 +296,7 @@ func makeDataPageProduct(obs KdvhObs) (LardObs, error) { return obsLard, err } -func makeDataPageEdata(obs KdvhObs) (LardObs, error) { +func ConvertEdata(obs KdvhObs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -320,7 +322,7 @@ func makeDataPageEdata(obs KdvhObs) (LardObs, error) { }, nil } -func makeDataPagePdata(obs KdvhObs) (LardObs, error) { +func ConvertPdata(obs KdvhObs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -361,7 +363,7 @@ func makeDataPagePdata(obs KdvhObs) (LardObs, error) { }, nil } -func makeDataPageNdata(obs KdvhObs) (LardObs, error) { +func ConvertNdata(obs KdvhObs) (LardObs, error) { var controlinfo string var valPtr *float32 @@ -402,7 +404,7 @@ func makeDataPageNdata(obs KdvhObs) (LardObs, error) { }, nil } -func makeDataPageVdata(obs KdvhObs) (LardObs, error) { +func ConvertVdata(obs KdvhObs) (LardObs, error) { var useinfo, controlinfo string var valPtr *float32 @@ -447,13 +449,15 @@ func makeDataPageVdata(obs KdvhObs) (LardObs, error) { }, nil } -func makeDataPageDiurnalInterpolated(obs KdvhObs) (LardObs, error) { +// Specific conversionfunction for diurnal interpolated, +// with hardcoded useinfo and controlinfo +func ConvertDiurnalInterpolated(obs KdvhObs) (LardObs, error) { val, err := strconv.ParseFloat(obs.Data, 32) if err != nil { return LardObs{}, err } - f32 := float32(val) + f32 := float32(val) return LardObs{ Obstime: obs.Obstime, Data: &f32, diff --git a/migrations/kdvh/main.go b/migrations/kdvh/main.go index 2cb4b33..ed41472 100644 --- a/migrations/kdvh/main.go +++ b/migrations/kdvh/main.go @@ -10,16 +10,16 @@ type Cmd struct { var KDVH map[string]*Table = map[string]*Table{ // Section 1: tables that need to be migrated entirely // TODO: figure out if we need to use the elem_code_paramid_level_sensor_t_edata table? - "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetConvFunc(makeDataPageEdata).SetImport(3000), + "T_EDATA": NewTable("T_EDATA", "T_EFLAG", "T_ELEM_EDATA").SetConvFunc(ConvertEdata).SetImport(3000), "T_METARDATA": NewTable("T_METARDATA", "", "T_ELEM_METARDATA").SetDumpFunc(dumpDataOnly).SetImport(3000), // Section 2: tables with some data in kvalobs, import only up to 2005-12-31 "T_ADATA": NewTable("T_ADATA", "T_AFLAG", "T_ELEM_OBS").SetImport(2006), "T_MDATA": NewTable("T_MDATA", "T_MFLAG", "T_ELEM_OBS").SetImport(2006), "T_TJ_DATA": NewTable("T_TJ_DATA", "T_TJ_FLAG", "T_ELEM_OBS").SetImport(2006), - "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPagePdata).SetImport(2006), - "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageNdata).SetImport(2006), - "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetConvFunc(makeDataPageVdata).SetImport(2006), + "T_PDATA": NewTable("T_PDATA", "T_PFLAG", "T_ELEM_OBS").SetConvFunc(ConvertPdata).SetImport(2006), + "T_NDATA": NewTable("T_NDATA", "T_NFLAG", "T_ELEM_OBS").SetConvFunc(ConvertNdata).SetImport(2006), + "T_VDATA": NewTable("T_VDATA", "T_VFLAG", "T_ELEM_OBS").SetConvFunc(ConvertVdata).SetImport(2006), "T_UTLANDDATA": NewTable("T_UTLANDDATA", "T_UTLANDFLAG", "T_ELEM_OBS").SetImport(2006), // Section 3: tables that should only be dumped @@ -32,10 +32,10 @@ var KDVH map[string]*Table = map[string]*Table{ "T_SVVDATA": NewTable("T_SVVDATA", "T_SVVFLAG", "T_ELEM_OBS"), // Section 4: special cases, namely digitized historical data - "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetConvFunc(makeDataPageProduct).SetImport(1957), - "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetConvFunc(makeDataPageProduct).SetImport(2006), - "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH").SetDumpFunc(dumpDataOnly).SetConvFunc(makeDataPageProduct), - "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", "").SetDumpFunc(dumpHomogenMonth).SetConvFunc(makeDataPageProduct), + "T_MONTH": NewTable("T_MONTH", "T_MONTH_FLAG", "T_ELEM_MONTH").SetConvFunc(ConvertProduct).SetImport(1957), + "T_DIURNAL": NewTable("T_DIURNAL", "T_DIURNAL_FLAG", "T_ELEM_DIURNAL").SetConvFunc(ConvertProduct).SetImport(2006), + "T_HOMOGEN_DIURNAL": NewTable("T_HOMOGEN_DIURNAL", "", "T_ELEM_HOMOGEN_MONTH").SetDumpFunc(dumpDataOnly).SetConvFunc(ConvertProduct), + "T_HOMOGEN_MONTH": NewTable("T_HOMOGEN_MONTH", "T_ELEM_HOMOGEN_MONTH", "").SetDumpFunc(dumpHomogenMonth).SetConvFunc(ConvertProduct), // Tables missing in the KDVH proxy: // 1. these exist in a separate database diff --git a/migrations/kdvh/table.go b/migrations/kdvh/table.go index bb2ffc8..1d0147c 100644 --- a/migrations/kdvh/table.go +++ b/migrations/kdvh/table.go @@ -47,6 +47,7 @@ type DumpMeta struct { logStr string } +// Implementation of these functions can be found in `import_functions.go` type ConvertFunction func(KdvhObs) (LardObs, error) type KdvhObs struct { *TimeseriesInfo @@ -100,7 +101,7 @@ func NewTable(data, flag, elem string) *Table { ElemTableName: elem, Path: data + "_combined", // NOTE: '_combined' kept for backward compatibility with original scripts dumpFunc: dumpDataAndFlags, - convFunc: makeDataPage, + convFunc: Convert, } }