From 40c64e24c4fb8a2b4232b06df2fcac4cc84394cc Mon Sep 17 00:00:00 2001 From: Hil Liao Date: Wed, 13 Feb 2019 10:27:18 -0800 Subject: [PATCH] added calories get and insert endpoints to call Google Fitness API for daily calories expended data: get -> get calories expended from Google fitness API post -> insert the calories data to Google BigQuery table --- README.md | 20 ++++ app.config | 3 +- app.yaml | 2 +- backend.py | 110 +++++++++++++++++-- fit.py | 72 ++++++++++++- fit_api.postman_collection.json | 181 ++++++++++++++++++++++++++++++++ 6 files changed, 378 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 794a2a2..62d4cc6 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ A web interface to gain consent from a user, to obtain their Google Fit data fro ### Create tables in Database and BigQuery `sudo mysql < structure.sql` to create the database and tables. Create the tables in Google Cloud BigQuery -> + **activities table** |Field name|Type|Mode| @@ -56,6 +57,25 @@ A web interface to gain consent from a user, to obtain their Google Fit data fro |recordedLocalDate|DATE|NULLABLE| |bpm|INTEGER|REQUIRED| +**steps** + +|Field name|Type|Mode| +|----------|----|----| +|username|STRING|REQUIRED| +|recordedLocalDate|DATE|REQUIRED| +|steps|INTEGER|REQUIRED| +|originDataSourceId|STRING|NULLABLE| + +**calories** + +|Field name|Type|Mode| +|----------|----|----| +|username|STRING|REQUIRED| +|recordedLocalDate|DATE|REQUIRED| +|calories|FLOAT|REQUIRED| +|originDataSourceId|STRING|NULLABLE| + + insert acticity types into activity_types table by running the section of structure.sql: ```INSERT INTO `activity_types` (name, id) VALUES```. Modify the syntax to fit [https://cloud.google.com/bigquery/docs/reference/standard-sql/query-syntax] diff --git a/app.config b/app.config index d6e7034..8d89608 100644 --- a/app.config +++ b/app.config @@ -17,4 +17,5 @@ dataset = Your_google_bigquery_dataset table_heartrate = Your_google_bigquery_table_heartrate table_activities = Your_google_bigquery_table_activities table_segments = Your_google_bigquery_table_activity_segments -table_steps = Your_google_bigquery_table_steps \ No newline at end of file +table_steps = Your_google_bigquery_table_steps +table_calories = calories \ No newline at end of file diff --git a/app.yaml b/app.yaml index cc4ff52..ebf51ea 100644 --- a/app.yaml +++ b/app.yaml @@ -1,7 +1,7 @@ runtime: python env: flex # --threads 48 fails with ERROR: (gcloud.app.deploy) Error Response: [13] An internal error occurred during deployment. You may need to delete this version manually. -entrypoint: gunicorn --timeout 3600 --workers 8 --threads 10 -b :$PORT fit:app +entrypoint: gunicorn --timeout 3600 --workers 8 --threads 24 -b :$PORT fit:app runtime_config: python_version: 2 diff --git a/backend.py b/backend.py index b37159f..71d7b25 100644 --- a/backend.py +++ b/backend.py @@ -12,6 +12,7 @@ DATE_FORMAT = '%Y-%m-%d' ONE_DAY_MS = 86400000 STEPS_DATASOURCE = "derived:com.google.step_count.delta:com.google.android.gms:estimated_steps" +CALORIES_DATASOURCE = 'derived:com.google.calories.expended:com.google.android.gms:merge_calories_expended' ACTIVITY_DATASOURCE = "derived:com.google.activity.segment:com.google.android.gms:merge_activity_segments" HEART_RATE_DATASOURCE = 'derived:com.google.heart_rate.bpm:com.google.android.gms:merge_heart_rate_bpm' epoch0 = datetime(1970, 1, 1, tzinfo=pytz.utc) @@ -38,12 +39,51 @@ GCP_table_activities = config.get('bigquery_config', 'table_activities') GCP_table_segments = config.get('bigquery_config', 'table_segments') GCP_table_steps = config.get('bigquery_config', 'table_steps') +GCP_table_calories = config.get('bigquery_config', 'table_calories') def current_milli_time(): return int(round(time.time() * 1000)) +def list_datasources(http_auth): + fit_service = build('fitness', 'v1', http=http_auth) + return fit_service.users().dataSources().list(userId="me").execute() + + +def get_daily_calories(http_auth, start_year, start_month, start_day, end_time_millis, local_timezone=DEFAULT_TIMEZONE): + """ + Get user's daily calory related data + :param http_auth: username authenticated HTTP client to call Google API + :param start_year: start getting calory data from local date's year + :param start_month: start getting calory data from local date's month + :param start_day: start getting calory data from local date's day + :param end_time_millis: getting calory data up to the end datetime in milliseconds Unix Epoch time + :param local_timezone: timezone such as US/Pacific, one of the pytz.all_timezones + :return: dict of daily calory and data source ID + """ + # calculate the timestamp in local time to query Google fitness API + local_0_hour = pytz.timezone(local_timezone).localize(datetime(start_year, start_month, start_day)) + start_time_millis = int((local_0_hour - epoch0).total_seconds() * 1000) + fit_service = build('fitness', 'v1', http=http_auth) + daily_calories = {} + + calory_data = get_aggregate(fit_service, start_time_millis, end_time_millis, CALORIES_DATASOURCE) + for daily_calory_data in calory_data['bucket']: + # use local date as the key + local_date = datetime.fromtimestamp(int(daily_calory_data['startTimeMillis']) / 1000, + tz=pytz.timezone(local_timezone)) + local_date_str = local_date.strftime(DATE_FORMAT) + + data_point = daily_calory_data['dataset'][0]['point'] + if data_point: + calories = data_point[0]['value'][0]['fpVal'] + data_source_id = data_point[0]['originDataSourceId'] + daily_calories[local_date_str] = {'calories': calories, 'originDataSourceId': data_source_id} + + return daily_calories + + def get_daily_steps(http_auth, start_year, start_month, start_day, end_time_millis, local_timezone=DEFAULT_TIMEZONE): """ Get user's daily step related data @@ -257,14 +297,14 @@ def insert_steps(username, steps, local_timezone=DEFAULT_TIMEZONE): now_local = now_utc.astimezone(pytz.timezone(local_timezone)) for localDate, value in steps.iteritems(): - incoming_activity_date = datetime.strptime(localDate, DATE_FORMAT).date() + incoming_steps_date = datetime.strptime(localDate, DATE_FORMAT).date() - # Do not insert today's activities because error occurs updating or deleting them - if incoming_activity_date == now_local.date(): + # Do not insert today's steps because error occurs updating or deleting them + if incoming_steps_date == now_local.date(): continue - # if incoming step's date not found in the existing steps table, insert incoming step count - if incoming_activity_date not in existing_step_dates: + # if incoming step's date not found in the existing table, insert incoming step count + if incoming_steps_date not in existing_step_dates: rows_to_insert.append( (username, localDate, value['steps'], value['originDataSourceId']) ) @@ -278,6 +318,50 @@ def insert_steps(username, steps, local_timezone=DEFAULT_TIMEZONE): return len(rows_to_insert) +def insert_calories(username, calories, local_timezone=DEFAULT_TIMEZONE): + """ + insert calories to BigQuery except local date of today's calories per local_timezone + :param username: user's Gmail + :param calories: dictionary of local date as key, value is another dict of calories, originDataSourceId + :param local_timezone: timezone such as US/Pacific, one of the pytz.all_timezones + :return: inserted row count + """ + bigquery_client = bigquery.Client() + dataset_ref = bigquery_client.dataset(GCP_dataset) + table_calories_ref = dataset_ref.table(GCP_table_calories) + table_calories = bigquery_client.get_table(table_calories_ref) + + # check existing rows by local date + query = "SELECT DISTINCT recordedLocalDate FROM `{}.{}.{}` WHERE username = '{}' ORDER BY recordedLocalDate DESC ".format( + GCP_project, GCP_dataset, GCP_table_calories, username) + query_job = bigquery_client.query(query) + existing_calories_dates = [row['recordedLocalDate'] for row in query_job.result()] + rows_to_insert = [] + now_utc = datetime.now(pytz.timezone('UTC')) + now_local = now_utc.astimezone(pytz.timezone(local_timezone)) + + for localDate, value in calories.iteritems(): + incoming_calories_date = datetime.strptime(localDate, DATE_FORMAT).date() + + # Do not insert today's calories because error occurs updating or deleting them + if incoming_calories_date == now_local.date(): + continue + + # if incoming calory's date not found in the existing table, insert incoming calories + if incoming_calories_date not in existing_calories_dates: + rows_to_insert.append( + (username, localDate, value['calories'], value['originDataSourceId']) + ) + + if rows_to_insert: + # BigQuery API request + errors = bigquery_client.insert_rows(table_calories, rows_to_insert) + if errors: + raise Exception(str(errors)) + + return len(rows_to_insert) + + def insert_activities(username, activities, local_timezone=DEFAULT_TIMEZONE): """ insert activities to BigQuery except local date of today's activities per local_timezone @@ -359,7 +443,19 @@ def post_steps(self): self.insert_steps_result = insert_steps(self.username, self.steps, self.local_timezone) return self.insert_steps_result else: - raise RuntimeError('no .steps to insert to BigQuery') + raise RuntimeError('no self.steps to insert to BigQuery') + + def get_calories(self): + self.calories = get_daily_calories(self.http_auth, self.start_year, self.start_month, self.start_day, + self.end_time_millis, self.local_timezone) + return self.calories + + def post_calories(self): + if self.calories is not None: + self.insert_calories_result = insert_calories(self.username, self.calories, self.local_timezone) + return self.insert_calories_result + else: + raise RuntimeError('no self.calories to insert to BigQuery') def get_and_post_heart_rate(self): self.insert_heart_rate_result = get_and_insert_heart_rate(self.http_auth, self.username, self.start_year, @@ -377,4 +473,4 @@ def post_activities(self): self.insert_activities_result = insert_activities(self.username, self.activities, self.local_timezone) return self.insert_activities_result else: - raise RuntimeError('no .activities to insert to BigQuery') + raise RuntimeError('no self.activities to insert to BigQuery') diff --git a/fit.py b/fit.py index fe283c3..7e0f4e4 100644 --- a/fit.py +++ b/fit.py @@ -121,6 +121,55 @@ def get_steps(username): return HTTPError(err.resp.status, "Google API HttpError: " + str(err)) +@app.get('/v1/users//calories') +def get_calories(username): + error = check_headers_apikey() + if error: + return error + http_auth, timezone = get_google_http_auth_n_user_timezone(username) + end_time_millis, start_date, error = extract_header_dates() + + if error: + if isinstance(error, HTTPError): + return error + else: + return HTTPResponse({ + 'code': httplib.BAD_REQUEST, + 'error': str(error)}, httplib.BAD_REQUEST) + else: + try: + # end_time_millis in headers data is optional + if end_time_millis is None: + end_time_millis = backend.current_milli_time() + + calories = backend.get_daily_calories(http_auth, start_date['year'], start_date['month'], start_date['day'], + end_time_millis, local_timezone=timezone) + response.content_type = 'application/json' + return calories + except client.HttpAccessTokenRefreshError as err: + return HTTPError(httplib.UNAUTHORIZED, "Refresh token invalid: " + str(err)) + except googleapiclient.errors.HttpError as err: + return HTTPError(err.resp.status, "Google API HttpError: " + str(err)) + + +@app.post('/v1/users//calories') +def insert_calories(username): + error = check_headers_apikey() + if error: + return error + calories = get_calories(username) + if isinstance(calories, HTTPError) or isinstance(calories, HTTPResponse): + return calories + + insert_result = { + 'inserted_count': backend.insert_calories(username, calories), + 'calories': calories + } + + response.content_type = 'application/json' + return insert_result + + @app.get('/v1') def main(): return static_file("post.html", ".") @@ -231,6 +280,23 @@ def extract_header_dates(): return end_time_millis, start_date, None +@app.get('/v1/users//datasources') +def list_all_datasources(username): + error = check_headers_apikey() + if error: + return error + http_auth, timezone = get_google_http_auth_n_user_timezone(username) + + try: + datasources = backend.list_datasources(http_auth) + except client.HttpAccessTokenRefreshError as err: + return HTTPError(httplib.UNAUTHORIZED, "Refresh token invalid: " + str(err)) + except googleapiclient.errors.HttpError as err: + return HTTPError(err.resp.status, "Google API HttpError: " + str(err)) + + return json.dumps(datasources) + + @app.post('/v1/users//heart') def insert_heart_rate(username): error = check_headers_apikey() @@ -365,7 +431,7 @@ def insert_daily_fitness_data_thread(bucket_name, retry, username): yesterday_local.month, yesterday_local.day, backend.current_milli_time(), timezone) retry[username] = {} - categories = {'heartrate', 'activities', 'steps'} + categories = {'heartrate', 'activities', 'steps', 'calories'} for category in categories: retry[username][category] = {} # countdown is the number of retries @@ -390,6 +456,10 @@ def insert_daily_fitness_data_thread(bucket_name, retry, username): # get and insert step counts get_result = df.get_steps() insert_result = df.post_steps() + elif category == 'calories': + # get and insert calories + get_result = df.get_calories() + insert_result = df.post_calories() # set to None upon success of getting API data and inserting to BigQuery retry[username][category]['countdown'] = None except client.HttpAccessTokenRefreshError as err: diff --git a/fit_api.postman_collection.json b/fit_api.postman_collection.json index 1f3aec0..76df656 100644 --- a/fit_api.postman_collection.json +++ b/fit_api.postman_collection.json @@ -602,6 +602,187 @@ "description": "Call the on demand endpoint to similate cron job runninng. Cron job does not need query string. The query string of users is for debugging." }, "response": [] + }, + { + "name": "list all datasources", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "method": "GET", + "header": [ + { + "key": "apikey", + "value": "{{apikey}}", + "type": "text" + }, + { + "key": "start_year", + "value": "2019", + "type": "text" + }, + { + "key": "start_month", + "value": "2", + "type": "text" + }, + { + "key": "start_day", + "value": "7", + "type": "text" + }, + { + "key": "end_time_millis", + "value": "1549907482000", + "type": "text", + "disabled": true + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "apikey", + "value": "secret", + "type": "text", + "disabled": true + } + ] + }, + "url": { + "raw": "{{host_url}}/v1/users/{{hil_gmail}}/datasources", + "host": [ + "{{host_url}}" + ], + "path": [ + "v1", + "users", + "{{hil_gmail}}", + "datasources" + ] + } + }, + "response": [] + }, + { + "name": "get daily calories", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "method": "GET", + "header": [ + { + "key": "apikey", + "value": "{{apikey}}", + "type": "text" + }, + { + "key": "start_year", + "value": "2019", + "type": "text" + }, + { + "key": "start_month", + "value": "2", + "type": "text" + }, + { + "key": "start_day", + "value": "7", + "type": "text" + }, + { + "key": "end_time_millis", + "value": "1549907482000", + "type": "text", + "disabled": true + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "apikey", + "value": "secret", + "type": "text", + "disabled": true + } + ] + }, + "url": { + "raw": "{{host_url}}/v1/users/{{hil_gmail}}/calories", + "host": [ + "{{host_url}}" + ], + "path": [ + "v1", + "users", + "{{hil_gmail}}", + "calories" + ] + } + }, + "response": [] + }, + { + "name": "insert calories", + "request": { + "method": "POST", + "header": [ + { + "key": "apikey", + "type": "text", + "value": "{{apikey}}" + }, + { + "key": "start_year", + "type": "text", + "value": "2019" + }, + { + "key": "start_month", + "type": "text", + "value": "2" + }, + { + "key": "start_day", + "type": "text", + "value": "7" + }, + { + "key": "end_time_millis", + "type": "text", + "value": "1549907482000", + "disabled": true + } + ], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "apikey", + "value": "secret", + "type": "text", + "disabled": true + } + ] + }, + "url": { + "raw": "{{host_url}}/v1/users/{{hil_gmail}}/calories", + "host": [ + "{{host_url}}" + ], + "path": [ + "v1", + "users", + "{{hil_gmail}}", + "calories" + ] + }, + "description": "call Google fitness API to get daily calories and insert the number of float to Google BigQuery table" + }, + "response": [] } ], "description": "new /v1 methods that depend on Google Cloud BigQuery"