diff --git a/internal/merger/merger.go b/internal/merger/merger.go index d0a12b29d..2433b691c 100644 --- a/internal/merger/merger.go +++ b/internal/merger/merger.go @@ -257,6 +257,7 @@ func MergeEvent(e1, e2 *pbv2.EventResponse) *pbv2.EventResponse { } // MergeObservation merges two V2 observation responses. +// Note the facets in o1 is preferred over that of o2. func MergeObservation(o1, o2 *pbv2.ObservationResponse) *pbv2.ObservationResponse { if o1 == nil { return o2 diff --git a/internal/server/v2/facet/golden/series_facet/series_facet.json b/internal/server/v2/facet/golden/series_facet/series_facet.json index 437f32be4..40f206c7d 100644 --- a/internal/server/v2/facet/golden/series_facet/series_facet.json +++ b/internal/server/v2/facet/golden/series_facet/series_facet.json @@ -114,6 +114,14 @@ }, "geoId/06": { "ordered_facets": [ + { + "facet_id": "local", + "observations": [ + { + "value": 1 + } + ] + }, { "facet_id": "2176550201", "obs_count": 123, diff --git a/internal/server/v2/facet/series.go b/internal/server/v2/facet/series.go index ec28e54aa..a75e5b5a3 100644 --- a/internal/server/v2/facet/series.go +++ b/internal/server/v2/facet/series.go @@ -143,15 +143,18 @@ func SeriesFacet( } } varEntityData := result.ByVariable[v].ByEntity[e] - varEntityData.OrderedFacets = append(varEntityData.OrderedFacets, - &pbv2.FacetObservation{ - FacetId: "local", - Observations: []*pb.PointStat{ - { - Value: proto.Float64(float64(count)), - }, + sqlFacet := &pbv2.FacetObservation{ + FacetId: "local", + Observations: []*pb.PointStat{ + { + Value: proto.Float64(float64(count)), }, }, + } + varEntityData.OrderedFacets = append( + // Order sql facet to the top + []*pbv2.FacetObservation{sqlFacet}, + varEntityData.OrderedFacets..., ) } } diff --git a/internal/server/v2/observation/contained_in.go b/internal/server/v2/observation/contained_in.go index a78abd26d..1a4ae3a63 100644 --- a/internal/server/v2/observation/contained_in.go +++ b/internal/server/v2/observation/contained_in.go @@ -277,7 +277,8 @@ func FetchContainedIn( } sqlResult = trimDirectResp(directResp) } - result = merger.MergeObservation(result, sqlResult) + // Prefer SQL data over BT data, so put sqlResult first. + result = merger.MergeObservation(sqlResult, result) } return result, nil } diff --git a/internal/server/v2/observation/direct.go b/internal/server/v2/observation/direct.go index 2a8ffee4e..2de2c0f9a 100644 --- a/internal/server/v2/observation/direct.go +++ b/internal/server/v2/observation/direct.go @@ -80,7 +80,7 @@ func FetchDirect( queryDate string, filter *pbv2.FacetFilter, ) (*pbv2.ObservationResponse, error) { - o1, err := FetchDirectBT( + btObservation, err := FetchDirectBT( ctx, store.BtGroup, variables, @@ -91,7 +91,7 @@ func FetchDirect( if err != nil { return nil, err } - o2, err := FetchDirectSQL( + sqlObservation, err := FetchDirectSQL( ctx, store.SQLClient, sqlProvenances, @@ -103,7 +103,8 @@ func FetchDirect( if err != nil { return nil, err } - return merger.MergeObservation(o1, o2), nil + // Prefer SQL data over BT data, so put sqlObservation first. + return merger.MergeObservation(sqlObservation, btObservation), nil } // FetchDirectBT fetches data from Bigtable cache. diff --git a/internal/server/v2/observation/golden/direct/2010.json b/internal/server/v2/observation/golden/direct/2010.json index 43d6b3368..876f3ef43 100644 --- a/internal/server/v2/observation/golden/direct/2010.json +++ b/internal/server/v2/observation/golden/direct/2010.json @@ -325,6 +325,18 @@ "dummy": {}, "geoId/06": { "ordered_facets": [ + { + "facet_id": "custom", + "observations": [ + { + "date": "2010", + "value": 999999 + } + ], + "obs_count": 1, + "earliest_date": "2010", + "latest_date": "2010" + }, { "facet_id": "2176550201", "observations": [ @@ -644,6 +656,10 @@ "import_name": "EurostatData", "provenance_url": "https://ec.europa.eu/eurostat/data/database", "measurement_method": "EurostatRegionalPopulationData" + }, + "custom": { + "import_name": "Custom Prov", + "provenance_url": "custom.datacommons.org" } } } \ No newline at end of file diff --git a/internal/server/v2/observation/golden/direct/all.json b/internal/server/v2/observation/golden/direct/all.json index 9dd012716..12835e8af 100644 --- a/internal/server/v2/observation/golden/direct/all.json +++ b/internal/server/v2/observation/golden/direct/all.json @@ -3729,6 +3729,18 @@ "dummy": {}, "geoId/06": { "ordered_facets": [ + { + "facet_id": "custom", + "observations": [ + { + "date": "2010", + "value": 999999 + } + ], + "obs_count": 1, + "earliest_date": "2010", + "latest_date": "2010" + }, { "facet_id": "2176550201", "observations": [ diff --git a/internal/server/v2/observation/golden/direct/latest.json b/internal/server/v2/observation/golden/direct/latest.json index df02b43a0..0c26706b7 100644 --- a/internal/server/v2/observation/golden/direct/latest.json +++ b/internal/server/v2/observation/golden/direct/latest.json @@ -409,6 +409,18 @@ "dummy": {}, "geoId/06": { "ordered_facets": [ + { + "facet_id": "custom", + "observations": [ + { + "date": "2010", + "value": 999999 + } + ], + "obs_count": 1, + "earliest_date": "2010", + "latest_date": "2010" + }, { "facet_id": "2176550201", "observations": [ diff --git a/test/datacommons.db b/test/datacommons.db index ecec71d88..8cc52f52f 100644 Binary files a/test/datacommons.db and b/test/datacommons.db differ diff --git a/test/observations.csv b/test/observations.csv index ea376d073..fe1475246 100644 --- a/test/observations.csv +++ b/test/observations.csv @@ -67,6 +67,7 @@ geoId/06,test_var_2,2010,1000,custom geoId/06,test_var_2,2020,2000,custom geoId/05,test_var_2,2010,5000,custom geoId/05,test_var_2,2020,6000,custom +geoId/06,Count_Person,2010,999999,custom test_entity,Count_Person,2022,100,custom test_entity,test_var_1,2022,200,custom wikidataId/Q506877,test_var_1,2023,50,custom