Rank observations from SQL database (custom DC) over that from Bigtab…

…le (base DC) (#1348)
datacommonsorg · Apr 11, 2024 · 6e695ef · 6e695ef
1 parent ebd5f2f
commit 6e695ef
Show file tree

Hide file tree

Showing 10 changed files with 66 additions and 11 deletions.
diff --git a/internal/merger/merger.go b/internal/merger/merger.go
@@ -257,6 +257,7 @@ func MergeEvent(e1, e2 *pbv2.EventResponse) *pbv2.EventResponse {
 }
 
 // MergeObservation merges two V2 observation responses.
+// Note the facets in o1 is preferred over that of o2.
 func MergeObservation(o1, o2 *pbv2.ObservationResponse) *pbv2.ObservationResponse {
 	if o1 == nil {
 		return o2

diff --git a/internal/server/v2/facet/golden/series_facet/series_facet.json b/internal/server/v2/facet/golden/series_facet/series_facet.json
@@ -114,6 +114,14 @@
         },
         "geoId/06": {
           "ordered_facets": [
+            {
+              "facet_id": "local",
+              "observations": [
+                {
+                  "value": 1
+                }
+              ]
+            },
             {
               "facet_id": "2176550201",
               "obs_count": 123,

diff --git a/internal/server/v2/facet/series.go b/internal/server/v2/facet/series.go
@@ -143,15 +143,18 @@ func SeriesFacet(
 					}
 				}
 				varEntityData := result.ByVariable[v].ByEntity[e]
-				varEntityData.OrderedFacets = append(varEntityData.OrderedFacets,
-					&pbv2.FacetObservation{
-						FacetId: "local",
-						Observations: []*pb.PointStat{
-							{
-								Value: proto.Float64(float64(count)),
-							},
+				sqlFacet := &pbv2.FacetObservation{
+					FacetId: "local",
+					Observations: []*pb.PointStat{
+						{
+							Value: proto.Float64(float64(count)),
 						},
 					},
+				}
+				varEntityData.OrderedFacets = append(
+					// Order sql facet to the top
+					[]*pbv2.FacetObservation{sqlFacet},
+					varEntityData.OrderedFacets...,
 				)
 			}
 		}

diff --git a/internal/server/v2/observation/contained_in.go b/internal/server/v2/observation/contained_in.go
@@ -277,7 +277,8 @@ func FetchContainedIn(
 			}
 			sqlResult = trimDirectResp(directResp)
 		}
-		result = merger.MergeObservation(result, sqlResult)
+		// Prefer SQL data over BT data, so put sqlResult first.
+		result = merger.MergeObservation(sqlResult, result)
 	}
 	return result, nil
 }
diff --git a/internal/server/v2/observation/direct.go b/internal/server/v2/observation/direct.go
@@ -80,7 +80,7 @@ func FetchDirect(
 	queryDate string,
 	filter *pbv2.FacetFilter,
 ) (*pbv2.ObservationResponse, error) {
-	o1, err := FetchDirectBT(
+	btObservation, err := FetchDirectBT(
 		ctx,
 		store.BtGroup,
 		variables,
@@ -91,7 +91,7 @@ func FetchDirect(
 	if err != nil {
 		return nil, err
 	}
-	o2, err := FetchDirectSQL(
+	sqlObservation, err := FetchDirectSQL(
 		ctx,
 		store.SQLClient,
 		sqlProvenances,
@@ -103,7 +103,8 @@ func FetchDirect(
 	if err != nil {
 		return nil, err
 	}
-	return merger.MergeObservation(o1, o2), nil
+	// Prefer SQL data over BT data, so put sqlObservation first.
+	return merger.MergeObservation(sqlObservation, btObservation), nil
 }
 
 // FetchDirectBT fetches data from Bigtable cache.

diff --git a/internal/server/v2/observation/golden/direct/2010.json b/internal/server/v2/observation/golden/direct/2010.json
@@ -325,6 +325,18 @@
         "dummy": {},
         "geoId/06": {
           "ordered_facets": [
+            {
+              "facet_id": "custom",
+              "observations": [
+                {
+                  "date": "2010",
+                  "value": 999999
+                }
+              ],
+              "obs_count": 1,
+              "earliest_date": "2010",
+              "latest_date": "2010"
+            },
             {
               "facet_id": "2176550201",
               "observations": [
@@ -644,6 +656,10 @@
       "import_name": "EurostatData",
       "provenance_url": "https://ec.europa.eu/eurostat/data/database",
       "measurement_method": "EurostatRegionalPopulationData"
+    },
+    "custom": {
+      "import_name": "Custom Prov",
+      "provenance_url": "custom.datacommons.org"
     }
   }
 }
diff --git a/internal/server/v2/observation/golden/direct/all.json b/internal/server/v2/observation/golden/direct/all.json
@@ -3729,6 +3729,18 @@
         "dummy": {},
         "geoId/06": {
           "ordered_facets": [
+            {
+              "facet_id": "custom",
+              "observations": [
+                {
+                  "date": "2010",
+                  "value": 999999
+                }
+              ],
+              "obs_count": 1,
+              "earliest_date": "2010",
+              "latest_date": "2010"
+            },
             {
               "facet_id": "2176550201",
               "observations": [

diff --git a/internal/server/v2/observation/golden/direct/latest.json b/internal/server/v2/observation/golden/direct/latest.json
@@ -409,6 +409,18 @@
         "dummy": {},
         "geoId/06": {
           "ordered_facets": [
+            {
+              "facet_id": "custom",
+              "observations": [
+                {
+                  "date": "2010",
+                  "value": 999999
+                }
+              ],
+              "obs_count": 1,
+              "earliest_date": "2010",
+              "latest_date": "2010"
+            },
             {
               "facet_id": "2176550201",
               "observations": [

diff --git a/test/datacommons.db b/test/datacommons.db
diff --git a/test/observations.csv b/test/observations.csv
@@ -67,6 +67,7 @@ geoId/06,test_var_2,2010,1000,custom
 geoId/06,test_var_2,2020,2000,custom
 geoId/05,test_var_2,2010,5000,custom
 geoId/05,test_var_2,2020,6000,custom
+geoId/06,Count_Person,2010,999999,custom
 test_entity,Count_Person,2022,100,custom
 test_entity,test_var_1,2022,200,custom
 wikidataId/Q506877,test_var_1,2023,50,custom