From ab676a499387e8fa073a451ea6e32b8b545a627e Mon Sep 17 00:00:00 2001 From: Matsuko Friedland Date: Sat, 10 Nov 2018 13:26:21 -0800 Subject: [PATCH 1/6] Initial commit --- .editorconfig | 11 +++++++++++ README.md | 4 +++- package.json | 19 +++++++++++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .editorconfig create mode 100644 package.json diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..cf8d7fa --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +# editorconfig.org + +root = true + +[*] +indent_style = space +indent_size = 2 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true diff --git a/README.md b/README.md index 7d6512a..30c19f5 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ -# asl-tab-api \ No newline at end of file +# asl-tab-api + +Get a YouTube channel's video data to use in the [ASL Tab browser extension](https://github.com/missmatsuko/asl-tab). diff --git a/package.json b/package.json new file mode 100644 index 0000000..61f770a --- /dev/null +++ b/package.json @@ -0,0 +1,19 @@ +{ + "name": "asl-tab-api", + "version": "1.0.0", + "description": "Get a YouTube channel's video data to use in the ASL Tab browser extension.", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/missmatsuko/asl-tab-api.git" + }, + "author": "Matsuko Friedland (https://matsuko.ca)", + "license": "ISC", + "bugs": { + "url": "https://github.com/missmatsuko/asl-tab-api/issues" + }, + "homepage": "https://github.com/missmatsuko/asl-tab-api#readme" +} From 5e610cdd9f78f4cef25b6862e0a51c7aeb78f9f2 Mon Sep 17 00:00:00 2001 From: Matsuko Friedland Date: Sat, 10 Nov 2018 13:47:22 -0800 Subject: [PATCH 2/6] Log some data from the YouTube playlist API Just getting something started. --- .env.example | 2 ++ .gitignore | 5 +++++ index.js | 35 +++++++++++++++++++++++++++++++++++ package-lock.json | 23 +++++++++++++++++++++++ package.json | 8 +++++++- src/composeQueryParamUrl.js | 11 +++++++++++ 6 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 index.js create mode 100644 package-lock.json create mode 100644 src/composeQueryParamUrl.js diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..7dcdd5d --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +YOUTUBE_API_KEY="" +YOUTUBE_PLAYLIST_ID="" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..6b1478e --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# dependencies +/node_modules + +# environment +.env diff --git a/index.js b/index.js new file mode 100644 index 0000000..0d47eb4 --- /dev/null +++ b/index.js @@ -0,0 +1,35 @@ +// Get env variables +require('dotenv').config(); + +import fetch from 'node-fetch'; +import composeQueryParamUrl from './src/composeQueryParamUrl'; + +const YOUTUBE_API_KEY = process.env.YOUTUBE_API_KEY; +const YOUTUBE_PLAYLIST_ID = process.env.YOUTUBE_PLAYLIST_ID; + +// Get some data +async function getData() { + let data; + + try { + const response = await fetch( + composeQueryParamUrl( + 'https://www.googleapis.com/youtube/v3/playlists', + { + 'key': YOUTUBE_API_KEY, + 'id': YOUTUBE_PLAYLIST_ID, + 'part': 'contentDetails', + } + ) + ); + + data = await response.json(); + } catch (error) { + console.error('Error:', error); + return; + } + + console.log(data); +} + +getData(); diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..d301e7a --- /dev/null +++ b/package-lock.json @@ -0,0 +1,23 @@ +{ + "name": "asl-tab-api", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "dotenv": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.1.0.tgz", + "integrity": "sha512-/veDn2ztgRlB7gKmE3i9f6CmDIyXAy6d5nBq+whO9SLX+Zs1sXEgFLPi+aSuWqUuusMfbi84fT8j34fs1HaYUw==" + }, + "esm": { + "version": "3.0.84", + "resolved": "https://registry.npmjs.org/esm/-/esm-3.0.84.tgz", + "integrity": "sha512-SzSGoZc17S7P+12R9cg21Bdb7eybX25RnIeRZ80xZs+VZ3kdQKzqTp2k4hZJjR7p9l0186TTXSgrxzlMDBktlw==" + }, + "node-fetch": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.2.1.tgz", + "integrity": "sha512-ObXBpNCD3A/vYQiQtEWl7DuqjAXjfptYFuGHLdPl5U19/6kJuZV+8uMHLrkj3wJrJoyfg4nhgyFixZdaZoAiEQ==" + } + } +} diff --git a/package.json b/package.json index 61f770a..614e564 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "description": "Get a YouTube channel's video data to use in the ASL Tab browser extension.", "main": "index.js", "scripts": { + "start": "node -r esm index.js", "test": "echo \"Error: no test specified\" && exit 1" }, "repository": { @@ -15,5 +16,10 @@ "bugs": { "url": "https://github.com/missmatsuko/asl-tab-api/issues" }, - "homepage": "https://github.com/missmatsuko/asl-tab-api#readme" + "homepage": "https://github.com/missmatsuko/asl-tab-api#readme", + "dependencies": { + "dotenv": "^6.1.0", + "esm": "^3.0.84", + "node-fetch": "^2.2.1" + } } diff --git a/src/composeQueryParamUrl.js b/src/composeQueryParamUrl.js new file mode 100644 index 0000000..fed7f80 --- /dev/null +++ b/src/composeQueryParamUrl.js @@ -0,0 +1,11 @@ +/* + Given a base URL and an object of query parameters as key/value pairs, + returns a composed query parameter URL +*/ +const composeQueryParamUrl = function(baseUrl, queryParams) { + const queryParamsArray = Object.entries(queryParams).map(([key, value]) => { + return `${key}=${encodeURIComponent(value)}`; + }); + return `${baseUrl}?${queryParamsArray.join('&')}`; +}; + export default composeQueryParamUrl; From e909fad84bb37eccdadfc9278d883be01ec24583 Mon Sep 17 00:00:00 2001 From: Matsuko Friedland Date: Sat, 10 Nov 2018 15:51:31 -0800 Subject: [PATCH 3/6] Get all videos from a playlist and log the total number of videos --- index.js | 63 ++++++++++++++++++++++++------------- src/composeQueryParamUrl.js | 3 +- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/index.js b/index.js index 0d47eb4..e31ff80 100644 --- a/index.js +++ b/index.js @@ -8,28 +8,49 @@ const YOUTUBE_API_KEY = process.env.YOUTUBE_API_KEY; const YOUTUBE_PLAYLIST_ID = process.env.YOUTUBE_PLAYLIST_ID; // Get some data -async function getData() { - let data; - - try { - const response = await fetch( - composeQueryParamUrl( - 'https://www.googleapis.com/youtube/v3/playlists', - { - 'key': YOUTUBE_API_KEY, - 'id': YOUTUBE_PLAYLIST_ID, - 'part': 'contentDetails', - } - ) - ); - - data = await response.json(); - } catch (error) { - console.error('Error:', error); - return; +async function getData(pageToken = undefined) { + const params = { + playlistId: YOUTUBE_PLAYLIST_ID, + key: YOUTUBE_API_KEY, + maxResults: 50, // 50 is the max allowed by YouTube API: https://developers.google.com/youtube/v3/docs/playlistItems/list#maxResults + part: 'snippet', + }; + + if (pageToken) { + params.pageToken = pageToken; } - console.log(data); + const response = await fetch( + composeQueryParamUrl('https://www.googleapis.com/youtube/v3/playlistItems', params) + ); + + return await response.json(); +} + +async function getVideos() { + const videos = []; + let pageToken = undefined; + + while (true) { + const data = await getData(pageToken); + + if (data.items.length) { + videos.push(...data.items); + } + + if (data.nextPageToken) { + pageToken = data.nextPageToken; + } else { + break; + } + } + + return videos; +} + +async function main() { + const result = await getVideos(); + console.log(`Got ${result.length} videos.`); } -getData(); +main(); diff --git a/src/composeQueryParamUrl.js b/src/composeQueryParamUrl.js index fed7f80..9cac4cf 100644 --- a/src/composeQueryParamUrl.js +++ b/src/composeQueryParamUrl.js @@ -8,4 +8,5 @@ const composeQueryParamUrl = function(baseUrl, queryParams) { }); return `${baseUrl}?${queryParamsArray.join('&')}`; }; - export default composeQueryParamUrl; + +export default composeQueryParamUrl; From 1e10940df8ea1c872ea232944398acba36d8056e Mon Sep 17 00:00:00 2001 From: Matsuko Friedland Date: Mon, 19 Nov 2018 20:47:25 -0800 Subject: [PATCH 4/6] Get all YouTube channel video info and upload as JSON to S3 --- .env.example | 8 ++++ .gitignore | 3 ++ index.js | 40 ++++++++++++++--- lambda.js | 18 ++++++++ local.js | 8 ++++ package-lock.json | 111 +++++++++++++++++++++++++++++++++++++++++++++- package.json | 10 +++-- 7 files changed, 188 insertions(+), 10 deletions(-) create mode 100644 lambda.js create mode 100644 local.js diff --git a/.env.example b/.env.example index 7dcdd5d..48c3571 100644 --- a/.env.example +++ b/.env.example @@ -1,2 +1,10 @@ +# AWS +# Note: Only AWS_BUCKET_NAME needed in Lambda. Rest are for local testing. +AWS_ACCESS_KEY_ID="" +AWS_BUCKET_NAME="" +AWS_REGION="" +AWS_SECRET_ACCESS_KEY="" + +# YouTube YOUTUBE_API_KEY="" YOUTUBE_PLAYLIST_ID="" diff --git a/.gitignore b/.gitignore index 6b1478e..df0b323 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,6 @@ # environment .env + +# dist +dist.zip diff --git a/index.js b/index.js index e31ff80..f9d0c15 100644 --- a/index.js +++ b/index.js @@ -1,13 +1,22 @@ +/* +* index.js +* This file is the main js file. It gets all the video data from a YouTube channel specified in .env and uploads it to Amazon S3 as a JSON file. +*/ + // Get env variables -require('dotenv').config(); +try { + require('dotenv').config(); +} catch (error) { +} +import AWS from 'aws-sdk'; import fetch from 'node-fetch'; import composeQueryParamUrl from './src/composeQueryParamUrl'; const YOUTUBE_API_KEY = process.env.YOUTUBE_API_KEY; const YOUTUBE_PLAYLIST_ID = process.env.YOUTUBE_PLAYLIST_ID; -// Get some data +// Fetch data from YouTube async function getData(pageToken = undefined) { const params = { playlistId: YOUTUBE_PLAYLIST_ID, @@ -16,6 +25,8 @@ async function getData(pageToken = undefined) { part: 'snippet', }; + // Add params that could be empty + // YouTube API will not return results if there's a null or undefined param if (pageToken) { params.pageToken = pageToken; } @@ -27,6 +38,7 @@ async function getData(pageToken = undefined) { return await response.json(); } +// Create array of video info async function getVideos() { const videos = []; let pageToken = undefined; @@ -48,9 +60,25 @@ async function getVideos() { return videos; } -async function main() { +// Upload result as JSON file to Amazon S3 +export default async function uploadToS3() { const result = await getVideos(); - console.log(`Got ${result.length} videos.`); -} -main(); + const awsS3Params = { + apiVersion: '2006-03-01', + region: process.env.AWS_REGION, + sessionToken: process.env.AWS_SESSION_TOKEN, + } + + const s3 = new AWS.S3(awsS3Params); + + const objectParams = { + ACL: 'public-read', + Body: JSON.stringify(result), + Bucket: process.env.AWS_BUCKET_NAME, + ContentType: 'application/json', + Key: 'data.json', + } + + return s3.putObject(objectParams).promise(); +} diff --git a/lambda.js b/lambda.js new file mode 100644 index 0000000..a7d6c60 --- /dev/null +++ b/lambda.js @@ -0,0 +1,18 @@ +/* +* lambda.js +* This file is for an AWS Lamba function. +*/ + +require = require("esm")(module/*, options*/); +const uploadToS3 = require('./index.js').default; + +exports.handler = async (event) => { + await uploadToS3(); + + const response = { + statusCode: 200, + body: JSON.stringify('Upload complete.'), + }; + + return response; +}; diff --git a/local.js b/local.js new file mode 100644 index 0000000..6e01cb3 --- /dev/null +++ b/local.js @@ -0,0 +1,8 @@ +/* +* local.js +* This file is for local development. +*/ + +import uploadToS3 from './index.js'; + +uploadToS3(); diff --git a/package-lock.json b/package-lock.json index d301e7a..78d300a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,20 +4,129 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "aws-sdk": { + "version": "2.355.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.355.0.tgz", + "integrity": "sha512-/p5Oce83a58ItO8qdQ+rjXs1isGxFAWG9jse3GIHqxyLjpD1OL5/ouUE/0jgVihz1zB+hJ0CNj4yo7DdCaQ+Cg==", + "dev": true, + "requires": { + "buffer": "4.9.1", + "events": "1.1.1", + "ieee754": "1.1.8", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.1.0", + "xml2js": "0.4.19" + } + }, + "base64-js": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.0.tgz", + "integrity": "sha512-ccav/yGvoa80BQDljCxsmmQ3Xvx60/UpBIij5QN21W3wBi/hhIC9OoO+KLpu9IJTS9j4DRVJ3aDDF9cMSoa2lw==", + "dev": true + }, + "buffer": { + "version": "4.9.1", + "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz", + "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=", + "dev": true, + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, "dotenv": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.1.0.tgz", - "integrity": "sha512-/veDn2ztgRlB7gKmE3i9f6CmDIyXAy6d5nBq+whO9SLX+Zs1sXEgFLPi+aSuWqUuusMfbi84fT8j34fs1HaYUw==" + "integrity": "sha512-/veDn2ztgRlB7gKmE3i9f6CmDIyXAy6d5nBq+whO9SLX+Zs1sXEgFLPi+aSuWqUuusMfbi84fT8j34fs1HaYUw==", + "dev": true }, "esm": { "version": "3.0.84", "resolved": "https://registry.npmjs.org/esm/-/esm-3.0.84.tgz", "integrity": "sha512-SzSGoZc17S7P+12R9cg21Bdb7eybX25RnIeRZ80xZs+VZ3kdQKzqTp2k4hZJjR7p9l0186TTXSgrxzlMDBktlw==" }, + "events": { + "version": "1.1.1", + "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", + "dev": true + }, + "ieee754": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.8.tgz", + "integrity": "sha1-vjPUCsEO8ZJnAfbwii2G+/0a0+Q=", + "dev": true + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", + "dev": true + }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", + "dev": true + }, "node-fetch": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.2.1.tgz", "integrity": "sha512-ObXBpNCD3A/vYQiQtEWl7DuqjAXjfptYFuGHLdPl5U19/6kJuZV+8uMHLrkj3wJrJoyfg4nhgyFixZdaZoAiEQ==" + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", + "dev": true + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "dev": true + }, + "sax": { + "version": "1.2.1", + "resolved": "http://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=", + "dev": true + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "dev": true, + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", + "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==", + "dev": true + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "dev": true, + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "http://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", + "dev": true } } } diff --git a/package.json b/package.json index 614e564..cb1eb3f 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,9 @@ "description": "Get a YouTube channel's video data to use in the ASL Tab browser extension.", "main": "index.js", "scripts": { - "start": "node -r esm index.js", - "test": "echo \"Error: no test specified\" && exit 1" + "start": "node -r esm local.js", + "test": "echo \"Error: no test specified\" && exit 1", + "package": "rm -f dist.zip && zip -r dist.zip index.js lambda.js node_modules src --exclude node_modules/aws-sdk --exclude node_modules/aws-sdk/\\*" }, "repository": { "type": "git", @@ -18,8 +19,11 @@ }, "homepage": "https://github.com/missmatsuko/asl-tab-api#readme", "dependencies": { - "dotenv": "^6.1.0", "esm": "^3.0.84", "node-fetch": "^2.2.1" + }, + "devDependencies": { + "aws-sdk": "^2.355.0", + "dotenv": "^6.1.0" } } From 2fb30d185c2f3061209983bc505e01acada9b028 Mon Sep 17 00:00:00 2001 From: Matsuko Friedland Date: Mon, 19 Nov 2018 22:45:26 -0800 Subject: [PATCH 5/6] Add duration to resulting JSON file and remove unnecessary data --- index.js | 82 ++++++++++++++++++++++++++++++++++++++--------- package-lock.json | 5 +++ package.json | 1 + 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/index.js b/index.js index f9d0c15..d85d184 100644 --- a/index.js +++ b/index.js @@ -3,21 +3,23 @@ * This file is the main js file. It gets all the video data from a YouTube channel specified in .env and uploads it to Amazon S3 as a JSON file. */ +// Import modules +import AWS from 'aws-sdk'; +import fetch from 'node-fetch'; +import { parse, toSeconds } from 'iso8601-duration'; +import composeQueryParamUrl from './src/composeQueryParamUrl'; + // Get env variables try { require('dotenv').config(); } catch (error) { } -import AWS from 'aws-sdk'; -import fetch from 'node-fetch'; -import composeQueryParamUrl from './src/composeQueryParamUrl'; - const YOUTUBE_API_KEY = process.env.YOUTUBE_API_KEY; const YOUTUBE_PLAYLIST_ID = process.env.YOUTUBE_PLAYLIST_ID; -// Fetch data from YouTube -async function getData(pageToken = undefined) { +// Fetch playlist items data from YouTube +async function getPlaylistItemsData(pageToken = undefined) { const params = { playlistId: YOUTUBE_PLAYLIST_ID, key: YOUTUBE_API_KEY, @@ -38,31 +40,79 @@ async function getData(pageToken = undefined) { return await response.json(); } +/* +* Fetch videos data from YouTube +* +* NOTE: Number of video IDs passed to videos API must be less than 50 according to several SO posts: +* - https://stackoverflow.com/questions/36370821/does-youtube-v3-data-api-have-a-limit-to-the-number-of-ids-you-can-send-to-vide +* - https://stackoverflow.com/questions/24860601/youtube-v3-api-id-query-parameter-length +*/ +async function getVideosData(ids) { + const params = { + playlistId: YOUTUBE_PLAYLIST_ID, // max. 50 ids + key: YOUTUBE_API_KEY, + id: ids, + part: 'contentDetails', + }; + + const response = await fetch( + composeQueryParamUrl('https://www.googleapis.com/youtube/v3/videos', params) + ); + + return await response.json(); +} + // Create array of video info -async function getVideos() { - const videos = []; +async function getResult() { + const result = {}; let pageToken = undefined; while (true) { - const data = await getData(pageToken); - - if (data.items.length) { - videos.push(...data.items); + const playlistItemsData = await getPlaylistItemsData(pageToken); + + if (playlistItemsData.items.length) { + /* + * Only keep required data: + * - video ID + * - video title + * - video duration (seconds) + */ + + // Get video's title and ID from playlist items data + for (const playlistItem of playlistItemsData.items) { + const snippet = playlistItem.snippet; + const videoId = snippet.resourceId.videoId; + result[videoId] = { + id: videoId, + title: snippet.title, + } + } + + // Get video duration from videos API (this is not available from playlist items API) + const playlistItemsIds = playlistItemsData.items.map((playlistItem) => playlistItem.snippet.resourceId.videoId).join(','); + + const videosData = await getVideosData(playlistItemsIds); + + if (videosData.items.length) { + for (const video of videosData.items) { + result[video.id].duration = toSeconds(parse(video.contentDetails.duration)); + } + } } - if (data.nextPageToken) { - pageToken = data.nextPageToken; + if (playlistItemsData.nextPageToken) { + pageToken = playlistItemsData.nextPageToken; } else { break; } } - return videos; + return Object.values(result); } // Upload result as JSON file to Amazon S3 export default async function uploadToS3() { - const result = await getVideos(); + const result = await getResult(); const awsS3Params = { apiVersion: '2006-03-01', diff --git a/package-lock.json b/package-lock.json index 78d300a..26de3de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -67,6 +67,11 @@ "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", "dev": true }, + "iso8601-duration": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/iso8601-duration/-/iso8601-duration-1.1.6.tgz", + "integrity": "sha512-wyVl4tmjZyaoh9GeohL3Dh4DKwr8CgiOL8eFRyRJyk8nopHOhZ0Ql76EwOUvx3OGQeI08xGj8+4AJgyXYs3iJw==" + }, "jmespath": { "version": "0.15.0", "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", diff --git a/package.json b/package.json index cb1eb3f..ea34079 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "homepage": "https://github.com/missmatsuko/asl-tab-api#readme", "dependencies": { "esm": "^3.0.84", + "iso8601-duration": "^1.1.6", "node-fetch": "^2.2.1" }, "devDependencies": { From 516c3828b2e8fa9f13fb5c3beecb1087c917a9dd Mon Sep 17 00:00:00 2001 From: Matsuko Friedland Date: Wed, 5 Dec 2018 20:22:50 -0800 Subject: [PATCH 6/6] Add caching header to generated file Set to 2 weeks --- index.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/index.js b/index.js index d85d184..39ca22b 100644 --- a/index.js +++ b/index.js @@ -17,6 +17,7 @@ try { const YOUTUBE_API_KEY = process.env.YOUTUBE_API_KEY; const YOUTUBE_PLAYLIST_ID = process.env.YOUTUBE_PLAYLIST_ID; +const CACHE_MAX_AGE = 2 * 7 * 24 * 60 * 60; // Duration to cache data file in seconds; 2 weeks // Fetch playlist items data from YouTube async function getPlaylistItemsData(pageToken = undefined) { @@ -126,6 +127,7 @@ export default async function uploadToS3() { ACL: 'public-read', Body: JSON.stringify(result), Bucket: process.env.AWS_BUCKET_NAME, + CacheControl: `public, max-age=${ CACHE_MAX_AGE }`, ContentType: 'application/json', Key: 'data.json', }