Merge pull request #1 from missmatsuko/develop

v1.0.0

.editorconfig

@@ -0,0 +1,11 @@
+# editorconfig.org
+
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true

.env.example

@@ -0,0 +1,10 @@
+# AWS
+# Note: Only AWS_BUCKET_NAME needed in Lambda. Rest are for local testing.
+AWS_ACCESS_KEY_ID=""
+AWS_BUCKET_NAME=""
+AWS_REGION=""
+AWS_SECRET_ACCESS_KEY=""
+
+# YouTube
+YOUTUBE_API_KEY=""
+YOUTUBE_PLAYLIST_ID=""

.gitignore

@@ -0,0 +1,8 @@
+# dependencies
+/node_modules
+
+# environment
+.env
+
+# dist
+dist.zip

README.md

@@ -1 +1,3 @@
-# asl-tab-api
+# asl-tab-api
+
+Get a YouTube channel's video data to use in the [ASL Tab browser extension](https://github.com/missmatsuko/asl-tab).

index.js

@@ -0,0 +1,136 @@
+/*
+* index.js
+* This file is the main js file. It gets all the video data from a YouTube channel specified in .env and uploads it to Amazon S3 as a JSON file.
+*/
+
+// Import modules
+import AWS from 'aws-sdk';
+import fetch from 'node-fetch';
+import { parse, toSeconds } from 'iso8601-duration';
+import composeQueryParamUrl from './src/composeQueryParamUrl';
+
+// Get env variables
+try {
+  require('dotenv').config();
+} catch (error) {
+}
+
+const YOUTUBE_API_KEY = process.env.YOUTUBE_API_KEY;
+const YOUTUBE_PLAYLIST_ID = process.env.YOUTUBE_PLAYLIST_ID;
+const CACHE_MAX_AGE = 2 * 7 * 24 * 60 * 60; // Duration to cache data file in seconds; 2 weeks
+
+// Fetch playlist items data from YouTube
+async function getPlaylistItemsData(pageToken = undefined) {
+  const params = {
+    playlistId: YOUTUBE_PLAYLIST_ID,
+    key: YOUTUBE_API_KEY,
+    maxResults: 50, // 50 is the max allowed by YouTube API: https://developers.google.com/youtube/v3/docs/playlistItems/list#maxResults
+    part: 'snippet',
+  };
+
+  // Add params that could be empty
+  // YouTube API will not return results if there's a null or undefined param
+  if (pageToken) {
+    params.pageToken = pageToken;
+  }
+
+  const response = await fetch(
+    composeQueryParamUrl('https://www.googleapis.com/youtube/v3/playlistItems', params)
+  );
+
+  return await response.json();
+}
+
+/*
+* Fetch videos data from YouTube
+*
+* NOTE: Number of video IDs passed to videos API must be less than 50 according to several SO posts:
+* - https://stackoverflow.com/questions/36370821/does-youtube-v3-data-api-have-a-limit-to-the-number-of-ids-you-can-send-to-vide
+* - https://stackoverflow.com/questions/24860601/youtube-v3-api-id-query-parameter-length
+*/
+async function getVideosData(ids) {
+  const params = {
+    playlistId: YOUTUBE_PLAYLIST_ID, // max. 50 ids
+    key: YOUTUBE_API_KEY,
+    id: ids,
+    part: 'contentDetails',
+  };
+
+  const response = await fetch(
+    composeQueryParamUrl('https://www.googleapis.com/youtube/v3/videos', params)
+  );
+
+  return await response.json();
+}
+
+// Create array of video info
+async function getResult() {
+  const result = {};
+  let pageToken = undefined;
+
+  while (true) {
+    const playlistItemsData = await getPlaylistItemsData(pageToken);
+
+    if (playlistItemsData.items.length) {
+      /*
+      * Only keep required data:
+      * - video ID
+      * - video title
+      * - video duration (seconds)
+      */
+
+      // Get video's title and ID from playlist items data
+      for (const playlistItem of playlistItemsData.items) {
+        const snippet = playlistItem.snippet;
+        const videoId = snippet.resourceId.videoId;
+        result[videoId] = {
+          id: videoId,
+          title: snippet.title,
+        }
+      }
+
+      // Get video duration from videos API (this is not available from playlist items API)
+      const playlistItemsIds = playlistItemsData.items.map((playlistItem) => playlistItem.snippet.resourceId.videoId).join(',');
+
+      const videosData = await getVideosData(playlistItemsIds);
+
+      if (videosData.items.length) {
+        for (const video of videosData.items) {
+          result[video.id].duration = toSeconds(parse(video.contentDetails.duration));
+        }
+      }
+    }
+
+    if (playlistItemsData.nextPageToken) {
+      pageToken = playlistItemsData.nextPageToken;
+    } else {
+      break;
+    }
+  }
+
+  return Object.values(result);
+}
+
+// Upload result as JSON file to Amazon S3
+export default async function uploadToS3() {
+  const result = await getResult();
+
+  const awsS3Params = {
+    apiVersion: '2006-03-01',
+    region: process.env.AWS_REGION,
+    sessionToken: process.env.AWS_SESSION_TOKEN,
+  }
+
+  const s3 = new AWS.S3(awsS3Params);
+
+  const objectParams = {
+    ACL: 'public-read',
+    Body: JSON.stringify(result),
+    Bucket: process.env.AWS_BUCKET_NAME,
+    CacheControl: `public, max-age=${ CACHE_MAX_AGE }`,
+    ContentType: 'application/json',
+    Key: 'data.json',
+  }
+
+  return s3.putObject(objectParams).promise();
+}

lambda.js

@@ -0,0 +1,18 @@
+/*
+* lambda.js
+* This file is for an AWS Lamba function.
+*/
+
+require = require("esm")(module/*, options*/);
+const uploadToS3 = require('./index.js').default;
+
+exports.handler = async (event) => {
+  await uploadToS3();
+
+  const response = {
+    statusCode: 200,
+    body: JSON.stringify('Upload complete.'),
+  };
+
+  return response;
+};

local.js

@@ -0,0 +1,8 @@
+/*
+* local.js
+* This file is for local development.
+*/
+
+import uploadToS3 from './index.js';
+
+uploadToS3();

package.json

@@ -0,0 +1,30 @@
+{
+  "name": "asl-tab-api",
+  "version": "1.0.0",
+  "description": "Get a YouTube channel's video data to use in the ASL Tab browser extension.",
+  "main": "index.js",
+  "scripts": {
+    "start": "node -r esm local.js",
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "package": "rm -f dist.zip && zip -r dist.zip index.js lambda.js node_modules src --exclude node_modules/aws-sdk --exclude node_modules/aws-sdk/\\*"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/missmatsuko/asl-tab-api.git"
+  },
+  "author": "Matsuko Friedland <info@matsuko.ca> (https://matsuko.ca)",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/missmatsuko/asl-tab-api/issues"
+  },
+  "homepage": "https://github.com/missmatsuko/asl-tab-api#readme",
+  "dependencies": {
+    "esm": "^3.0.84",
+    "iso8601-duration": "^1.1.6",
+    "node-fetch": "^2.2.1"
+  },
+  "devDependencies": {
+    "aws-sdk": "^2.355.0",
+    "dotenv": "^6.1.0"
+  }
+}

src/composeQueryParamUrl.js

@@ -0,0 +1,12 @@
+/*
+  Given a base URL and an object of query parameters as key/value pairs,
+  returns a composed query parameter URL
+*/
+const composeQueryParamUrl = function(baseUrl, queryParams) {
+  const queryParamsArray = Object.entries(queryParams).map(([key, value]) => {
+    return `${key}=${encodeURIComponent(value)}`;
+  });
+  return `${baseUrl}?${queryParamsArray.join('&')}`;
+};
+
+export default composeQueryParamUrl;