From 5768a7f91470d96d5e67bcbcf8e97317f2e53768 Mon Sep 17 00:00:00 2001 From: Bo Xu Date: Fri, 7 Aug 2020 12:28:37 -0700 Subject: [PATCH] Remove GetChart api which is used by old GNI tool (#267) * Remove GetChart api which is used by old GNI tool * Remove chart data key building function --- e2etest/get_chart_data_test.go | 81 ----- golden_response/staging/get_chart_data.json | 379 -------------------- proto/mixer.proto | 21 -- server/chart_data.go | 77 ---- server/key.go | 8 - 5 files changed, 566 deletions(-) delete mode 100644 e2etest/get_chart_data_test.go delete mode 100644 golden_response/staging/get_chart_data.json delete mode 100644 server/chart_data.go diff --git a/e2etest/get_chart_data_test.go b/e2etest/get_chart_data_test.go deleted file mode 100644 index 9e21f51ba..000000000 --- a/e2etest/get_chart_data_test.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2etest - -import ( - "context" - "encoding/json" - "io/ioutil" - "path" - "runtime" - "testing" - - pb "github.com/datacommonsorg/mixer/proto" - "github.com/datacommonsorg/mixer/server" - "github.com/google/go-cmp/cmp" - "google.golang.org/protobuf/testing/protocmp" -) - -func TestGetChartData(t *testing.T) { - ctx := context.Background() - - memcacheData, err := loadMemcache() - if err != nil { - t.Fatalf("Failed to load memcache %v", err) - } - - client, err := setup(server.NewMemcache(memcacheData)) - if err != nil { - t.Fatalf("Failed to set up mixer and client") - } - _, filename, _, _ := runtime.Caller(0) - goldenFile := path.Join( - path.Dir(filename), "../golden_response/staging/get_chart_data.json") - - keys := []string{ - "country/ITA^count^measured^^^^Person", - "geoId/06^count^measured^^^^Person", - "nces/062631003930^count^measured^^^^Student", - "nces/062631003930^count^measured^^^^Student^gender^Female", - "geoId/06085^cumulativeCount^measured^^^^MedicalConditionIncident^incidentType^COVID_19^medicalStatus^ConfirmedOrProbableCase", - } - req := &pb.GetChartDataRequest{ - Keys: keys, - } - resp, err := client.GetChartData(ctx, req) - if err != nil { - t.Fatalf("could not GetChartData: %s", err) - } - var result map[string]*pb.ObsTimeSeries - err = json.Unmarshal([]byte(resp.GetPayload()), &result) - if err != nil { - t.Fatalf("Can not Unmarshal payload: %v", err) - } - - if generateGolden { - updateGolden(result, goldenFile) - return - } - - var expected map[string]*pb.ObsTimeSeries - file, _ := ioutil.ReadFile(goldenFile) - err = json.Unmarshal(file, &expected) - if err != nil { - t.Fatalf("Can not Unmarshal golden file %s: %v", goldenFile, err) - } - if diff := cmp.Diff(result, expected, protocmp.Transform()); diff != "" { - t.Errorf("payload got diff: %v", diff) - } -} diff --git a/golden_response/staging/get_chart_data.json b/golden_response/staging/get_chart_data.json deleted file mode 100644 index c2abf529c..000000000 --- a/golden_response/staging/get_chart_data.json +++ /dev/null @@ -1,379 +0,0 @@ -{ - "country/ITA^count^measured^^^^Person": { - "data": { - "1960": 50025500, - "1961": 50373900, - "1962": 50698800, - "1963": 51060100, - "1964": 51443900, - "1965": 51906800, - "1966": 52317900, - "1967": 52720100, - "1968": 53080900, - "1969": 53390600, - "1970": 53685300, - "1971": 53958400, - "1972": 54188579, - "1973": 54574111, - "1974": 54928700, - "1975": 55293036, - "1976": 55588966, - "1977": 55847553, - "1978": 56063269, - "1979": 56247017, - "1980": 56388480, - "1981": 56479285, - "1982": 56524064, - "1983": 56563031, - "1984": 56565117, - "1985": 56588319, - "1986": 56597823, - "1987": 56594487, - "1988": 56609375, - "1989": 56649201, - "1990": 56694360, - "1991": 56744119, - "1992": 56772923, - "1993": 56821250, - "1994": 56842392, - "1995": 56844408, - "1996": 56844197, - "1997": 56876364, - "1998": 56904379, - "1999": 56909109, - "2000": 56923524, - "2001": 56960692, - "2002": 56987507, - "2003": 57130506, - "2004": 57495900, - "2005": 57874753, - "2006": 58064214, - "2007": 58223744, - "2008": 58652875, - "2009": 59000586, - "2010": 59190143, - "2011": 59364690, - "2012": 59394207, - "2013": 59685227, - "2014": 60782668, - "2015": 60795612, - "2016": 60665551, - "2017": 60589445, - "2018": 60483973, - "2019": 60359546 - }, - "place_name": "Italy", - "place_dcid": "country/ITA", - "provenance_domain": "europa.eu" - }, - "geoId/06085^cumulativeCount^measured^^^^MedicalConditionIncident^incidentType^COVID_19^medicalStatus^ConfirmedOrProbableCase": { - "data": { - "2020-01-31": 1, - "2020-02-01": 1, - "2020-02-02": 2, - "2020-02-03": 2, - "2020-02-04": 2, - "2020-02-05": 2, - "2020-02-06": 2, - "2020-02-07": 2, - "2020-02-08": 2, - "2020-02-09": 2, - "2020-02-10": 2, - "2020-02-11": 2, - "2020-02-12": 2, - "2020-02-13": 2, - "2020-02-14": 2, - "2020-02-15": 2, - "2020-02-16": 2, - "2020-02-17": 2, - "2020-02-18": 2, - "2020-02-19": 2, - "2020-02-20": 2, - "2020-02-21": 2, - "2020-02-22": 2, - "2020-02-23": 2, - "2020-02-24": 2, - "2020-02-25": 2, - "2020-02-26": 2, - "2020-02-27": 2, - "2020-02-28": 3, - "2020-02-29": 4, - "2020-03-01": 7, - "2020-03-02": 9, - "2020-03-03": 11, - "2020-03-04": 14, - "2020-03-05": 20, - "2020-03-06": 24, - "2020-03-07": 32, - "2020-03-08": 37, - "2020-03-09": 43, - "2020-03-10": 45, - "2020-03-11": 48, - "2020-03-12": 66, - "2020-03-13": 79, - "2020-03-14": 91, - "2020-03-15": 114, - "2020-03-16": 138, - "2020-03-17": 155, - "2020-03-18": 175, - "2020-03-19": 189, - "2020-03-20": 196, - "2020-03-21": 263, - "2020-03-22": 303, - "2020-03-23": 321, - "2020-03-24": 375, - "2020-03-25": 459, - "2020-03-26": 542, - "2020-03-27": 574, - "2020-03-28": 591, - "2020-03-29": 646, - "2020-03-30": 848, - "2020-03-31": 890, - "2020-04-01": 956, - "2020-04-02": 1019, - "2020-04-03": 1094, - "2020-04-04": 1148, - "2020-04-05": 1207, - "2020-04-06": 1224, - "2020-04-07": 1285, - "2020-04-08": 1380, - "2020-04-09": 1442, - "2020-04-10": 1484, - "2020-04-11": 1566, - "2020-04-12": 1621, - "2020-04-13": 1666, - "2020-04-14": 1666, - "2020-04-15": 1793, - "2020-04-16": 1833, - "2020-04-17": 1870, - "2020-04-18": 1870, - "2020-04-19": 1870, - "2020-04-20": 1922, - "2020-04-21": 1946, - "2020-04-22": 1962, - "2020-04-23": 1987, - "2020-04-24": 2018, - "2020-04-25": 2040, - "2020-04-26": 2084, - "2020-04-27": 2105, - "2020-04-28": 2122, - "2020-04-29": 2134, - "2020-04-30": 2163, - "2020-05-01": 2179, - "2020-05-02": 2204, - "2020-05-03": 2231, - "2020-05-04": 2244, - "2020-05-05": 2255, - "2020-05-06": 2268, - "2020-05-07": 2281, - "2020-05-08": 2290, - "2020-05-09": 2307, - "2020-05-10": 2339, - "2020-05-11": 2341, - "2020-05-12": 2364, - "2020-05-13": 2364, - "2020-05-14": 2391, - "2020-05-15": 2403, - "2020-05-16": 2418, - "2020-05-17": 2453, - "2020-05-18": 2453, - "2020-05-19": 2483, - "2020-05-20": 2492, - "2020-05-21": 2508, - "2020-05-22": 2546, - "2020-05-23": 2571, - "2020-05-24": 2617, - "2020-05-25": 2652, - "2020-05-26": 2675, - "2020-05-27": 2688, - "2020-05-28": 2701, - "2020-05-29": 2707, - "2020-05-30": 2731, - "2020-05-31": 2776, - "2020-06-01": 2805, - "2020-06-02": 2819, - "2020-06-03": 2832, - "2020-06-04": 2850, - "2020-06-05": 2892, - "2020-06-06": 2920, - "2020-06-07": 2942, - "2020-06-08": 2973, - "2020-06-09": 3017, - "2020-06-10": 3032, - "2020-06-11": 3063, - "2020-06-12": 3117, - "2020-06-13": 3157, - "2020-06-14": 3197, - "2020-06-15": 3230, - "2020-06-16": 3254, - "2020-06-17": 3288, - "2020-06-18": 3363, - "2020-06-19": 3398, - "2020-06-20": 3458, - "2020-06-21": 3547, - "2020-06-22": 3606, - "2020-06-23": 3727, - "2020-06-24": 3832, - "2020-06-25": 3888, - "2020-06-26": 3984, - "2020-06-27": 3984, - "2020-06-28": 4162, - "2020-06-29": 4162, - "2020-06-30": 4231, - "2020-07-01": 4572, - "2020-07-02": 4750, - "2020-07-03": 4849, - "2020-07-04": 5077 - }, - "place_name": "Santa Clara County", - "place_dcid": "geoId/06085" - }, - "geoId/06^count^measured^^^^Person": { - "data": { - "1900": 1490000, - "1901": 1550000, - "1902": 1623000, - "1903": 1702000, - "1904": 1792000, - "1905": 1893000, - "1906": 1976000, - "1907": 2054000, - "1908": 2161000, - "1909": 2282000, - "1910": 2406000, - "1911": 2534000, - "1912": 2668000, - "1913": 2811000, - "1914": 2934000, - "1915": 3008000, - "1916": 3071000, - "1917": 3171000, - "1918": 3262000, - "1919": 3339000, - "1920": 3554000, - "1921": 3795000, - "1922": 3991000, - "1923": 4270000, - "1924": 4541000, - "1925": 4730000, - "1926": 4929000, - "1927": 5147000, - "1928": 5344000, - "1929": 5531000, - "1930": 5711000, - "1931": 5824000, - "1932": 5894000, - "1933": 5963000, - "1934": 6060000, - "1935": 6175000, - "1936": 6341000, - "1937": 6528000, - "1938": 6656000, - "1939": 6785000, - "1940": 6950000, - "1941": 7237000, - "1942": 7735000, - "1943": 8506000, - "1944": 8945000, - "1945": 9344000, - "1946": 9559000, - "1947": 9832000, - "1948": 10064000, - "1949": 10337000, - "1950": 10677000, - "1951": 11134000, - "1952": 11635000, - "1953": 12251000, - "1954": 12746000, - "1955": 13133000, - "1956": 13713000, - "1957": 14264000, - "1958": 14880000, - "1959": 15467000, - "1960": 15870000, - "1961": 16497000, - "1962": 17072000, - "1963": 17668000, - "1964": 18151000, - "1965": 18585000, - "1966": 18858000, - "1967": 19176000, - "1968": 19394000, - "1969": 19711000, - "1970": 19971069, - "1971": 20345939, - "1972": 20585469, - "1973": 20868728, - "1974": 21173865, - "1975": 21537849, - "1976": 21935909, - "1977": 22352396, - "1978": 22835958, - "1979": 23256880, - "1980": 24285937, - "1981": 24820007, - "1982": 25360023, - "1983": 25844389, - "1984": 26441112, - "1985": 27102241, - "1986": 27777153, - "1987": 28464253, - "1988": 29218161, - "1989": 29760021, - "1990": 29950111, - "1991": 30414114, - "1992": 30875920, - "1993": 31147208, - "1994": 31317179, - "1995": 31493525, - "1996": 31780829, - "1997": 32217708, - "1998": 32682794, - "1999": 33145121, - "2000": 33634154, - "2001": 34481753, - "2002": 34867773, - "2003": 35236589, - "2004": 35538256, - "2005": 35770688, - "2006": 35947461, - "2007": 36185908, - "2008": 36538008, - "2009": 36887615, - "2010": 37266600, - "2011": 37638369, - "2012": 37948800, - "2013": 38260787, - "2014": 38596972, - "2015": 38918045, - "2016": 39167117, - "2017": 39358497, - "2018": 39461588, - "2019": 39512223 - }, - "place_name": "California", - "place_dcid": "geoId/06", - "provenance_domain": "census.gov" - }, - "nces/062631003930^count^measured^^^^Student": { - "data": { - "2011": 1836 - }, - "place_name": "Mountain View High", - "place_dcid": "nces/062631003930", - "provenance_domain": "ed.gov" - }, - "nces/062631003930^count^measured^^^^Student^gender^Female": { - "data": { - "2011": 860, - "2012": 884, - "2013": 899, - "2014": 904, - "2015": 949, - "2016": 935 - }, - "place_name": "Mountain View High", - "place_dcid": "nces/062631003930", - "provenance_domain": "ed.gov" - } -} \ No newline at end of file diff --git a/proto/mixer.proto b/proto/mixer.proto index 1b5746343..257d0dc90 100644 --- a/proto/mixer.proto +++ b/proto/mixer.proto @@ -337,18 +337,6 @@ message GetInterestingPlaceAspectsResponse { string payload = 1; } -// Request to get chart data. -message GetChartDataRequest{ - // Keys. - repeated string keys = 1; -} - -// Response to get chart data. -message GetChartDataResponse{ - // The JSON payload. - string payload = 1; -} - // Request to get landing page info for places. message GetLandingPageRequest { // Place DCIDs. @@ -723,15 +711,6 @@ service Mixer { option (google.api.http) = { get: "/node/interesting-place-aspects"}; } - // Get chart data. - rpc GetChartData(GetChartDataRequest) returns ( - GetChartDataResponse) { - option (google.api.http) = { - post: "/node/chart-data" - body: "*" - }; - } - // Get landing page info for a place. rpc GetLandingPage(GetLandingPageRequest) returns (GetLandingPageResponse) { option (google.api.http) = { diff --git a/server/chart_data.go b/server/chart_data.go deleted file mode 100644 index d314381f2..000000000 --- a/server/chart_data.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2020 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package server - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - pb "github.com/datacommonsorg/mixer/proto" - "github.com/datacommonsorg/mixer/util" -) - -// GetChartData implements API for Mixer.GetChartData. -func (s *Server) GetChartData(ctx context.Context, - in *pb.GetChartDataRequest) (*pb.GetChartDataResponse, error) { - keys := in.GetKeys() - if len(keys) == 0 { - return nil, fmt.Errorf("missing required arguments") - } - result := map[string]*pb.ObsTimeSeries{} - rowList := buildChartDataKey(keys) - - // Read from branch cache first - memData := s.memcache.ReadParallel(rowList, convertToObsSeries, util.RemoveKeyPrefix) - for key, data := range memData { - if data == nil { - result[key] = nil - } else { - result[key] = data.(*pb.ObsTimeSeries) - } - } - // Read data from Bigtable if not all data is obtained from memcache. - if len(memData) < len(keys) { - dataMap, err := bigTableReadRowsParallel( - ctx, s.btTable, rowList, - convertToObsSeries, util.RemoveKeyPrefix) - if err != nil { - return nil, err - } - for key, data := range dataMap { - if _, ok := result[key]; !ok { - if data == nil { - result[key] = nil - } else { - result[key] = data.(*pb.ObsTimeSeries) - } - } - } - } - for key := range result { - result[key].PlaceDcid = strings.Split(result[key].PlaceDcid, "^")[0] - } - - for dcid := range result { - result[dcid] = filterAndRank(result[dcid], "", "", "") - } - - jsonRaw, err := json.Marshal(result) - if err != nil { - return nil, err - } - return &pb.GetChartDataResponse{Payload: string(jsonRaw)}, nil -} diff --git a/server/key.go b/server/key.go index 9e586d03e..ab1052db9 100644 --- a/server/key.go +++ b/server/key.go @@ -102,14 +102,6 @@ func buildPopPVKey(dcids []string) bigtable.RowList { return rowList } -func buildChartDataKey(keys []string) bigtable.RowList { - rowList := bigtable.RowList{} - for _, key := range keys { - rowList = append(rowList, fmt.Sprintf("%s%s", util.BtChartDataPrefix, key)) - } - return rowList -} - func buildPlaceInKey(dcids []string, placeType string) bigtable.RowList { rowList := bigtable.RowList{} for _, dcid := range dcids {