diff --git a/.gitignore b/.gitignore index aaea2d9..81fa81a 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ node_modules/ /.wwwtfrc /service-account-credentials.json /secrets.tar +/schedule.json +/contents/schedule-json.txt diff --git a/.travis.yml b/.travis.yml index 6124e2b..94da9af 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,17 +7,17 @@ node_js: script: - npm run-script generate-locals - # - npm run-script generate-redirects + - npm run-script generate-redirects - npm run-script build # Because we build again, move the non-deploy build out of the way. - # - mv build test-build + - mv build test-build before_deploy: -# - openssl aes-256-cbc -K $encrypted_92a5b1b18fab_key -iv $encrypted_92a5b1b18fab_iv -in secrets.tar.enc -out secrets.tar -d -# - tar xvf secrets.tar -# - npm run-script ci:import -# - npm run-script build -# - node tests/post-build.js + - openssl aes-256-cbc -in secrets.tar.enc -out secrets.tar.out -d -pass "env:encrypted_client_secrets_password" + - tar xvf secrets.tar + - npm run-script ci:import + - npm run-script build + - node tests/post-build.js deploy: provider: pages diff --git a/plugins/nunjucks.js b/plugins/nunjucks.js index dc38b90..f3505e9 100644 --- a/plugins/nunjucks.js +++ b/plugins/nunjucks.js @@ -13,7 +13,7 @@ module.exports = function(env, callback) { extensions: {} }; Object.assign(env.config.locals, require('../locals-generated.json')); - env.config.locals.schedule = require('../schedule.json'); + // env.config.locals.schedule = require('../schedule.json'); env.config.locals.Date = Date; // Load the new nunjucks environment. @@ -47,14 +47,18 @@ module.exports = function(env, callback) { nenv.opts.autoescape = options.autoescape; class NunjucksTemplate extends env.TemplatePlugin { - constructor(template) { + constructor(template, filename) { super(); this.template = template; + this.filename = filename; } render(locals, callback) { try { let html = this.template.render(locals); + if (!html) { + throw new Error('Template render failed' + this.filename); + } html = minify(html, { removeAttributeQuotes: true, collapseWhitespace: true, @@ -69,7 +73,7 @@ module.exports = function(env, callback) { } static fromFile(filepath, callback) { - callback(null, new NunjucksTemplate(nenv.getTemplate(filepath.relative))); + callback(null, new NunjucksTemplate(nenv.getTemplate(filepath.relative), filepath.relative)); } } diff --git a/schedule.json b/schedule.json deleted file mode 100644 index 0967ef4..0000000 --- a/schedule.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/scripts/spreadsheet-import/credentials.js b/scripts/spreadsheet-import/credentials.js new file mode 100644 index 0000000..a777a6d --- /dev/null +++ b/scripts/spreadsheet-import/credentials.js @@ -0,0 +1,86 @@ +const path = require('path'); +const fs = require('fs'); +const credentialsCache = {}; + +function getCredentialsPath() { + return ( + process.env.WWWTF_CREDENTIALS_PATH || + path.resolve(process.env.HOME, '.wwwtf18') + ); +} + +/** + * Loads a private credentials-file. + * Files with sensitive information are stored in the directory ~/.wwwtf18 + * in JSON-Format. An alternative directory can be specified using the + * env-variable `WWWTF_CREDENTIALS_PATH`, but that is mostly intended + * to be used for CI/CD-servers and similar environments. + * @param {string} filename json-filename without any path. + * @param {boolean} ignoreMissing true to ignore missing files and return + * no content. + * @returns {*} the parsed content of the json-file + */ +function loadCredentials(filename, ignoreMissing = false) { + if (!credentialsCache[filename]) { + const credentialsFile = path.resolve(getCredentialsPath(), filename); + + if (ignoreMissing && !fs.existsSync(credentialsFile)) { + return null; + } + + try { + credentialsCache[filename] = require(credentialsFile); + } catch (err) { + console.error( + "šŸ” It appears that you don't have your credentials setup yet.\n" + + ' Please copy the file ' + + filename + + ' to\n ' + + getCredentialsPath() + + '\n to continue. Ask your coworkers if you never heard of that file.' + ); + + throw new Error(`failed to load ${credentialsFile}: ${err}`); + } + } + + return credentialsCache[filename]; +} + +/** + * Checks if credentials with the given filename exist. + * @param {string} filename file basename + * @return {boolean} - + */ +function hasCredentials(filename) { + const credentialsFile = path.resolve(getCredentialsPath(), filename); + + return fs.existsSync(credentialsFile); +} + +/** + * Stores credentials to a file in the credentials-store. + * @param {string} filename the file basename. + * @param {object} data the data to store. + */ +function storeCredentials(filename, data) { + credentialsCache[filename] = data; + + const credentialsFile = path.resolve(getCredentialsPath(), filename); + try { + fs.writeFileSync(credentialsFile, JSON.stringify(data)); + } catch (err) { + console.error( + `šŸ” Failed to write credentials to file ${credentialsFile}.` + ); + + throw new Error(`failed to write credentials: ${err}`); + } +} + +module.exports = { + getCredentialsPath, + loadCredentials, + hasCredentials, + storeCredentials +}; diff --git a/scripts/spreadsheet-import/image-utils/speaker-image.js b/scripts/spreadsheet-import/image-utils/speaker-image.js new file mode 100644 index 0000000..c8675c2 --- /dev/null +++ b/scripts/spreadsheet-import/image-utils/speaker-image.js @@ -0,0 +1,84 @@ +const fs = require('fs'); +const path = require('path'); +const fetch = require('node-fetch'); +const chalk = require('chalk'); +const imageType = require('image-type'); +const imageSize = require('image-size'); + +function getImageFilename(speaker, ext) { + let filename = speaker.firstname + '-' + speaker.lastname; + filename = filename.replace(/[^\w]/g, '-'); + filename = filename.replace(/--/g, '-').toLowerCase(); + + return filename + '.' + ext; +} + +function getLocalSpeakerImage(imagePath, speaker) { + if (!imagePath) { + return null; + } + + const filename = getImageFilename(speaker, 'jpg'); + const srcFilename = path.join(imagePath, filename); + const destFilename = path.join('contents/images/speaker', filename); + + if (fs.existsSync(srcFilename)) { + console.log(` --> image found in image-path:`, filename); + const buffer = fs.readFileSync(srcFilename); + const size = imageSize(buffer); + fs.writeFileSync(destFilename, buffer); + + return { + filename, + width: size.width, + height: size.height + }; + } + + return null; +} + +async function downloadSpeakerImage(speaker) { + const url = speaker.potraitImageUrl; + console.log('downloadImage', url); + if (!url) { + console.error(chalk.yellow('no image specified for ' + speaker.id)); + return {}; + } + + try { + const res = await fetch(url); + + if (!res.headers.get('content-type').startsWith('image')) { + console.error(chalk.red.bold(' !!! url is not an image', url)); + return {}; + } + + const buffer = await res.buffer(); + + const info = imageType(buffer); + if (!info) { + console.error(chalk.red.bold(' !!! no type-imformation for image', url)); + return {}; + } + + const size = imageSize(buffer); + const filename = getImageFilename(speaker, info.ext); + const fullPath = 'contents/images/speaker/' + filename; + + console.info(' --> image downloaded ', chalk.green(fullPath)); + fs.writeFileSync(fullPath, buffer); + + return { + filename, + width: size.width, + height: size.height + }; + } catch (err) { + console.error(chalk.red.bold(' !!! failed to download', url)); + console.error(err); + return {}; + } +} + +module.exports = {getLocalSpeakerImage, downloadSpeakerImage}; diff --git a/scripts/spreadsheet-import/image-utils/sponsor-image.js b/scripts/spreadsheet-import/image-utils/sponsor-image.js new file mode 100644 index 0000000..8926668 --- /dev/null +++ b/scripts/spreadsheet-import/image-utils/sponsor-image.js @@ -0,0 +1,61 @@ +const fs = require('fs'); +const path = require('path'); +const fetch = require('node-fetch'); +const chalk = require('chalk'); + +function getLocalSponsorImage(imagePath, sponsor) { + if (!imagePath) { + return null; + } + + const filename = sponsor.id + '.svg'; + const srcFilename = path.join(imagePath, filename); + const destFilename = path.join('contents/images/sponsor', filename); + + if (fs.existsSync(srcFilename)) { + console.log(` --> image found in image-path:`, filename); + const buffer = fs.readFileSync(srcFilename); + fs.writeFileSync(destFilename, buffer); + + return { + filename + }; + } + + return null; +} + +async function downloadSponsorImage(sponsor) { + const url = sponsor.logoUrl; + console.log('downloadImage', url); + if (!url) { + console.error(chalk.yellow('no image specified for ' + sponsor.id)); + return {}; + } + + try { + const res = await fetch(url); + + if (!res.headers.get('content-type').startsWith('image')) { + console.error(chalk.red.bold(' !!! url is not an image', url)); + return {}; + } + + const buffer = await res.buffer(); + const filename = sponsor.id + '.svg'; + const fullPath = `contents/images/sponsor/${filename}`; + + console.info(' --> image downloaded ', chalk.green(fullPath)); + fs.writeFileSync(fullPath, buffer); + + return { + filename + }; + } catch (err) { + console.error(chalk.red.bold(' !!! failed to download', url)); + console.error(err); + return {}; + } +} + +module.exports = {getLocalSponsorImage, downloadSponsorImage}; diff --git a/scripts/spreadsheet-import/index.js b/scripts/spreadsheet-import/index.js new file mode 100644 index 0000000..64e2dcf --- /dev/null +++ b/scripts/spreadsheet-import/index.js @@ -0,0 +1,249 @@ +const fs = require('fs'); +const path = require('path'); +const yaml = require('js-yaml'); +const wordwrap = require('wordwrap')(80); +const chalk = require('chalk'); +const program = require('commander'); +const mkdirp = require('mkdirp'); +const {promisify} = require('util'); +const {getSheetData} = require('./spreadsheet-api'); +const {processSheet, simplifySpreadsheetData} = require('./spreadsheet-utils'); +const {downloadSpeakerImage, getLocalSpeakerImage} = require('./image-utils/speaker-image'); +const {downloadSponsorImage, getLocalSponsorImage} = require('./image-utils/sponsor-image'); +const {processSchedule} = require('./process-schedule'); +const rimraf = promisify(require('rimraf')); +const timeout = promisify(setTimeout); + +// spreadsheet-format is illustrated here: +// https://docs.google.com/spreadsheets/d/14TQHTYePS0SAaXGRNF3zYXvvk8xz25CXW-uekQy4HAs/edit + +program + .description( + 'import speaker- and talk-data from the specified spreadheet and ' + + 'update the files in contents/speakers and contents/talks' + ) + .arguments('') + .action(spreadsheet => { + const rxSpreadsheetIdFromUrl = /^https:\/\/docs\.google\.com\/.*\/d\/([^/]+).*$/; + + program.spreadsheetId = spreadsheet; + + if (rxSpreadsheetIdFromUrl.test(spreadsheet)) { + program.spreadsheetId = spreadsheet.replace(rxSpreadsheetIdFromUrl, '$1'); + } + }) + .option( + '-p --production', + "run in production-mode (don't import unpublished items)" + ) + .option('-i --image-path ', 'alternative path to look for images') + .option('-C --no-cleanup', "don't run cleanup before import") + .parse(process.argv); + +const contentRoot = path.resolve(__dirname, '../../contents'); +const sheetParams = { + /*speakers: { + templateGlobals: { + template: 'pages/speaker.html.njk' + }, + dataFieldName: 'speaker', + contentPath: 'speakers' + }, + mcs: { + templateGlobals: { + template: 'pages/person.html.njk' + }, + dataFieldName: 'speaker', + contentPath: 'mcs' + }, + artists: { + templateGlobals: { + template: 'pages/person.html.njk' + }, + dataFieldName: 'speaker', + contentPath: 'artists' + },*/ + team: { + templateGlobals: { + template: 'pages/person.html.njk' + }, + dataFieldName: 'speaker', + contentPath: 'team' + }, + /*sponsors: { + templateGlobals: { + template: 'pages/sponsor.html.njk' + }, + dataFieldName: 'sponsor', + contentPath: 'sponsors' + }, + schedule: { + parseSchedule: true, + },*/ +}; + +const wwwtfrcFile = __dirname + '/../../.wwwtfrc'; +const hasRcFile = fs.existsSync(wwwtfrcFile); + +let rcFileParams = {}; +if (hasRcFile) { + rcFileParams = JSON.parse(fs.readFileSync(wwwtfrcFile)); +} + +const params = { + ...rcFileParams, + imagePath: program.imagePath, + doCleanup: program.cleanup, + publishedOnly: program.production || process.env.NODE_ENV === 'production' +}; +if (program.spreadsheetId) { + params.spreadsheetId = program.spreadsheetId; +} + +if (!params.spreadsheetId) { + console.log( + chalk.red.bold('A spreadsheet-id (or spreadsheet-url) is required.') + ); + program.outputHelp(); + process.exit(1); +} + +if (!hasRcFile) { + console.log('saving settings to', chalk.green('.wwwtfrc')); + fs.writeFileSync( + wwwtfrcFile, + JSON.stringify({spreadsheetId: params.spreadsheetId}, null, 2) + ); +} + +main(params).catch(err => console.error(err)); + +async function main(params) { + // ---- ensure the directories exist... + const requiredDirectories = ['team', 'speakers', 'talks', 'sponsors', 'images/speaker', 'images/sponsor']; + const requiredDirectoryPaths = requiredDirectories.map( + dir => `${__dirname}/../../contents/${dir}` + ); + const missingDirectories = requiredDirectoryPaths.filter( + dir => !fs.existsSync(dir) + ); + + if (!!missingDirectories.length) { + console.log(chalk.gray('creating missing directories...')); + missingDirectories.forEach(dir => mkdirp(dir)); + } + + // ---- cleanup... + if (params.doCleanup) { + console.log(chalk.gray('cleaning up...')); + + await Promise.all([ + rimraf(path.join(contentRoot, 'images/{speaker,sponsor}/*')), + rimraf(path.join(contentRoot, '{speakers,sponsors,talks}/*md')), + ]); + } + + // ---- fetch spreadsheet-data... + console.log(chalk.gray('loading spreadsheet data...')); + const sheets = simplifySpreadsheetData( + await getSheetData(params.spreadsheetId, { + readonly: true, + + async beforeOpenCallback(url) { + console.log( + chalk.white( + '\n\nšŸ” You first need to grant access to your ' + + 'google-spreadsheets to this program.\n An ' + + 'authorization-dialog will be ' + + 'opened in your browser in 5 seconds.\n\n' + ), + chalk.blue.underline(url) + ); + + return await timeout(5000); + } + }) + ); + + // ---- parse and generate markdown-files + console.log(chalk.gray('awesome, that worked.')); + Object.keys(sheets).forEach(sheetId => { + if (!sheetId) { + // Published pages create unnamed sheets. + return; + } + if (!sheetParams[sheetId]) { + console.log(chalk.red('Missing metadata for'), sheetId); + return; + } + const {templateGlobals, dataFieldName, contentPath, parseSchedule} = sheetParams[sheetId]; + if (parseSchedule) { + processSchedule(sheets[sheetId]); + return; + } + const records = processSheet(sheets[sheetId]); + + console.log(chalk.white('processing sheet %s'), chalk.yellow(sheetId)); + records + // filter unpublished records when not in dev-mode. + .filter(r => r.published || !params.publishedOnly) + + // render md-files + .forEach(async function(record) { + const filename = path.join(contentRoot, contentPath, `${record.id}.md`); + + let {content, ...data} = record; + let title = ''; + + if (!content) { + content = ' '; + } + + if (!data.name) { + data.name = data.firstname + ' ' + data.lastname; + } + + if (sheetId === 'sponsors') { + data.image = getLocalSponsorImage(params.imagePath, data); + title = data.name; + if (!data.image) { + try { + data.image = await downloadSponsorImage(data); + } catch (err) { + console.error('this is bad: ', err); + } + } + delete data.logoUrl; + } else { + data.image = getLocalSpeakerImage(params.imagePath, data); + title = `${data.name}: ${data.talkTitle}`; + if (!data.image) { + data.image = await downloadSpeakerImage(data); + } + + delete data.potraitImageUrl; + } + + const frontmatter = yaml.safeDump({ + ...templateGlobals, + title, + [dataFieldName]: data + }); + + console.log( + ' --> write markdown %s', + chalk.green(path.relative(process.cwd(), filename)) + ); + + const markdownContent = + '----\n\n' + + '# THIS FILE WAS GENERATED AUTOMATICALLY.\n' + + '# CHANGES MADE HERE WILL BE OVERWRITTEN.\n\n' + + frontmatter.trim() + + '\n\n----\n\n' + + wordwrap(content); + + fs.writeFileSync(filename, markdownContent); + }); + }); +} diff --git a/scripts/spreadsheet-import/init-jwt-auth-client.js b/scripts/spreadsheet-import/init-jwt-auth-client.js new file mode 100644 index 0000000..9cad43d --- /dev/null +++ b/scripts/spreadsheet-import/init-jwt-auth-client.js @@ -0,0 +1,29 @@ +const {promisify} = require('util'); +const google = require('googleapis'); +const {loadCredentials} = require('./credentials'); + +/** + * Initializes the auth-client. + * @param {string} scope the oauth-scope. + * @return {Promise} a promise that resolves when the authClient is ready + */ +async function initJWTAuthClient(scope) { + /* eslint-disable camelcase */ + const {client_email, private_key} = loadCredentials( + 'service-account-credentials.json' + ); + const client = new google.auth.JWT( + client_email, + null, + private_key, + scope, + null + ); + /* eslint-enable */ + + await promisify(client.authorize.bind(client))(); + + return client; +} + +module.exports = {initJWTAuthClient}; diff --git a/scripts/spreadsheet-import/init-oauth2-client.js b/scripts/spreadsheet-import/init-oauth2-client.js new file mode 100644 index 0000000..723e2e9 --- /dev/null +++ b/scripts/spreadsheet-import/init-oauth2-client.js @@ -0,0 +1,119 @@ +const crypto = require('crypto'); +const http = require('http'); +const {promisify} = require('util'); +const {parse: parseUrl} = require('url'); + +const open = require('open'); +const getPort = require('get-port'); +const google = require('googleapis'); + +const { + loadCredentials, + hasCredentials, + storeCredentials +} = require('./credentials'); + +/** + * Initialize the oauth-client for the specified scope. + * @param {string} scope the oauth-scope to request authorization for. + * @param {object} options additional options + * @return {Promise} the auth-client. + */ +async function initOAuth2Client(scope, options = {}) { + const clientSecret = loadCredentials('client_secret.json').installed; + + const port = await getPort(); + const auth = new google.auth.OAuth2( + clientSecret.client_id, + clientSecret.client_secret, + `http://localhost:${port}` + ); + + const md5 = crypto.createHash('md5'); + const scopeHash = md5.update(scope).digest('hex'); + const credentialsFile = `credentials-${scopeHash}.json`; + + if (hasCredentials(credentialsFile)) { + auth.credentials = loadCredentials(credentialsFile); + } else { + auth.credentials = await getCredentials(auth, scope, options); + storeCredentials(credentialsFile, auth.credentials); + } + + return auth; +} + +/** + * Retrieves the auth-client credentials. + * @param {google.auth.OAuth2} auth the OAuth2-instance. + * @param {string} scope the scope to get authorization for. + * @param {object} options additional options + * @param {function?} options.beforeOpenCallback an async function to be called + * with the authorization-url before the url is opened. + * @return {Promise} the credentials, including access_token, + * refresh_token and expiry_date. + */ +async function getCredentials(auth, scope, options = {}) { + const getToken = promisify(auth.getToken.bind(auth)); + + const url = auth.generateAuthUrl({ + access_type: 'offline', // eslint-disable-line camelcase + scope + }); + + const redirectUri = parseUrl(url, true).query.redirect_uri; + const port = parseUrl(redirectUri).port; + + if (options.beforeOpenCallback) { + await options.beforeOpenCallback(url); + } + + open(url); + + return await getToken(await receiveAuthorizationCode(port)); +} + +/** + * Starts an http-server to listen for the oauth2 redirectUri to be called + * containing the authorization-code. + * @param {number} port port-number for the http-server + * @return {Promise} the authorization-code. + */ +async function receiveAuthorizationCode(port) { + const server = http.createServer(); + const listen = promisify(server.listen.bind(server)); + + await listen(port, '127.0.0.1'); + + return new Promise((resolve, reject) => { + server.once('request', (request, response) => { + const {code} = parseUrl(request.url, true).query; + + if (!code) { + response.end( + '\n' + + '' + + '

Well, that\'s embarrassing.

' + + '

It won\'t be possible to spreadsheets without this ' + + ' authorization. Maybe try again?

' + + '' + ); + + reject(new Error('authorization failed.')); + } else { + response.end( + '\n' + + '' + + '

Perfect!

' + + '

You can now close this browser window.

' + + '' + ); + } + server.close(); + + resolve(parseUrl(request.url, true).query.code); + }); + }); +} + +module.exports = {initOAuth2Client}; diff --git a/scripts/spreadsheet-import/process-schedule.js b/scripts/spreadsheet-import/process-schedule.js new file mode 100644 index 0000000..b8c1fec --- /dev/null +++ b/scripts/spreadsheet-import/process-schedule.js @@ -0,0 +1,91 @@ +const fs = require('fs'); + +module.exports.processSchedule = function(sheet) { + const data = structureData(sheet); + const schedule = { + info: info(), + schedule: data, + } + const json = JSON.stringify(schedule, null, ' '); + console.info(json); + fs.writeFileSync('./schedule.json', json); + // Write with .txt filename, because wintersmith doesn't support serving + // files with the "magic" .json extension. + fs.writeFileSync('./contents/schedule-json.txt', json); +} + +const columns = [ + 'backtrack:startTime', 'backtrack:duration', 'backtrack:number', + '-', 'backtrack:who', 'backtrack:what', '-', + 'sidetrack:startTime', 'sidetrack:duration', 'sidetrack:number', + '-', 'sidetrack:who', 'sidetrack:what', '-', + 'community:startTime', 'community:what', 'community:detail', '-', + 'sponsor:startTime', 'sponsor:what', 'sponsor:detail' +]; + +const tracksMap = { + backtrack: 'Back Track', + sidetrack: 'Side Track', + community: 'Community Lounge', + sponsor: 'Sponsor Booth' +} + +function structureData(lessCrappyData) { + let day = 1; + const mergedRecords = {}; + + for (let row = 2, nRows = lessCrappyData.length; row < nRows; row++) { + + + if (!lessCrappyData[row]) { continue; } + + if (/Day 2:/.test(lessCrappyData[row][0])) { + day = 2; + } + + const tracks = {}; + for (let col = 0, nCols = lessCrappyData[row].length; col < nCols; col++) { + if (!columns[col] || columns[col] === '-') { continue; } + const [track, field] = columns[col].split(':'); + + + if (!tracks[track]) { + tracks[track] = { + day: day, + date: day == 1 ? '2018-06-02' : '2018-06-03', + track: tracksMap[track], + trackId: track + }; + } + tracks[track][field] = lessCrappyData[row][col]; + } + + Object.keys(tracks).forEach(track => { + if (!tracks[track].startTime || !tracks[track].what) { + return; + } + tracks[track].startTime = String(tracks[track].startTime).replace(':', '.'); + tracks[track].dateTime = tracks[track].date + ' ' + + tracks[track].startTime.replace('.', ':') + + ' GMT+0200'; + if (!mergedRecords[day]) { + mergedRecords[day] = {}; + } + if (!mergedRecords[day][tracks[track].startTime]) { + mergedRecords[day][tracks[track].startTime] = []; + } + mergedRecords[day][tracks[track].startTime].push(tracks[track]); + }); + } + + return mergedRecords; +} + +function info() { + const now = new Date(); + const conferenceDay = now < Date.parse('Sun Jun 02 2018 00:00:00 GMT+0200 (CEST)') ? 1 : 2; + return { + currentDay: conferenceDay, + generationTime: now.toString(), + }; +} \ No newline at end of file diff --git a/scripts/spreadsheet-import/spreadsheet-api.js b/scripts/spreadsheet-import/spreadsheet-api.js new file mode 100644 index 0000000..ad40de8 --- /dev/null +++ b/scripts/spreadsheet-import/spreadsheet-api.js @@ -0,0 +1,68 @@ +const google = require('googleapis'); + +const {initOAuth2Client} = require('./init-oauth2-client'); +const {initJWTAuthClient} = require('./init-jwt-auth-client'); +const {hasCredentials, getCredentialsPath} = require('./credentials'); + +const sheets = google.sheets('v4'); + +/** + * Loads the specified sheet via the spreadsheets-API and returns it's + * raw data. + * @param {string} documentId The id of the spreadsheets-document + * @param {string} sheetId The id of the worksheet within the document + * @returns {Promise.} The raw spreadsheet-data + */ +async function getSheetData(documentId, options = {}) { + const requestOptions = { + auth: await getAuthClient(options), + spreadsheetId: documentId, + includeGridData: true + }; + + return new Promise((resolve, reject) => { + sheets.spreadsheets.get(requestOptions, (err, result) => { + if (err) { + return reject(err); + } + return resolve(result); + }); + }); +} + +let clientPromise = null; + +/** + * Initializes the auth-client. The preferred method is to use personal oauth2, + * since it allows better management of permissions. Alternatively JWT + * (aka service accounts) is supported for automated build-environments. + * @return {Promise} a promise that resolves when the authClient is ready + */ +async function getAuthClient(options = {}) { + if (clientPromise) { + return clientPromise; + } + + const scope = + 'https://www.googleapis.com/auth/spreadsheets' + + (options.readonly ? '.readonly' : ''); + + if (hasCredentials('client_secret.json')) { + clientPromise = initOAuth2Client(scope, options); + } else if (hasCredentials('service-account-credentials.json')) { + clientPromise = initJWTAuthClient(scope); + } else { + console.error( + "šŸ” couldn't create an auth-client. Please make sure that your " + + ` credentials in ${getCredentialsPath()} are properly set up.` + ); + + throw new Error('failed to authorize'); + } + + return clientPromise; +} + +module.exports = { + getSheetData +}; diff --git a/scripts/spreadsheet-import/spreadsheet-utils.js b/scripts/spreadsheet-import/spreadsheet-utils.js new file mode 100644 index 0000000..7dc9f41 --- /dev/null +++ b/scripts/spreadsheet-import/spreadsheet-utils.js @@ -0,0 +1,114 @@ +const objectPath = require('object-path'); + +/** + * Simplifies data from the spreadsheets-API by reducing it to actual values. + * @param {Object} response - + * @return {Object} - + */ +function simplifySpreadsheetData(response) { + return response.sheets.reduce((sheets, {properties, data}) => { + sheets[properties.title] = data[0].rowData + .filter(row => row.values) + .map(row => + row.values.map(value => { + if (!value.effectiveValue) { + return null; + } else if (typeof value.effectiveValue.numberValue !== 'undefined') { + return value.effectiveValue.numberValue; + } else if (typeof value.effectiveValue.stringValue !== 'undefined') { + return value.effectiveValue.stringValue; + } + + throw new Error('neither numberValue nor stringValue exists'); + }) + ) + .filter(row => row.some(value => value !== null)); + + return sheets; + }, {}); +} + +/** + * Process (parse) data from a single sheet in the spreadsheet document. + * @param sheetData + */ +function processSheet(sheetData) { + const columnNames = sheetData[0]; + const columnTypes = sheetData[1]; + const bodyRows = sheetData.slice(2); + + const columns = parseColumnHeaders(columnNames, columnTypes); + + return bodyRows.map(parseDataRow.bind(null, columns)); +} + +/** + * Parses and validates the column-headers. + * @param columnNames + * @param columnTypes + * @return {Column[]} + */ +function parseColumnHeaders(columnNames, columnTypes) { + const columns = []; + const columnsByName = {}; + + for (let i = 0; i < columnNames.length; i++) { + const name = columnNames[i]; + const type = columnTypes[i]; + + if (!name || name.startsWith('//')) { + continue; + } + + const column = { + dataIndex: i, + name, + type + }; + + // validate: make sure there isn't already a column with the same fieldname + const conflictingColumn = columnsByName[name]; + if (conflictingColumn) { + throw new Error( + `āš ļø name-conflict: column "${column.name}" (cell ` + + `${String.fromCharCode(65 + i)}1) has the same ` + + `fieldname as column "${conflictingColumn.header}" + (cell ${String.fromCharCode(65 + conflictingColumn.dataIndex)}1)` + ); + } + + columnsByName[name] = column; + columns.push(column); + } + + return columns; +} + +/** + * Parses a single record from the spreadsheet. + * @param {Column[]} columns the column-specifications from the header + * @param {string[]} row the data-row from the spreadsheet + * @returns {Object} the parsed record + */ +function parseDataRow(columns, row) { + const record = {}; + + for (const column of columns) { + const {dataIndex, name, type} = column; + let value = row[dataIndex]; + + if (typeof value === 'undefined') { + continue; + } + + if (type === 'boolean') { + value = Boolean(value); + } + + objectPath.set(record, name, value); + } + + return record; +} + +module.exports = {simplifySpreadsheetData, processSheet}; diff --git a/secrets.tar.enc b/secrets.tar.enc new file mode 100644 index 0000000..90a2b85 Binary files /dev/null and b/secrets.tar.enc differ diff --git a/templates/pages/about.html.njk b/templates/pages/about.html.njk index 4f288e9..ae4ca6a 100644 --- a/templates/pages/about.html.njk +++ b/templates/pages/about.html.njk @@ -37,6 +37,56 @@ -{% endblock %} + {% macro teamMember(contents, speaker) %} +
+ {% if speaker.links.twitter %} + + {% endif %} + {% include '../partials/speaker-picture.html.njk' %} +

+ {{ speaker.firstname }} {{ speaker.lastname }} +

+

+ {% if speaker.twitterHandle %} + {{ speaker.twitterHandle }}
+ {% endif %} +

+ {% if speaker.links.twitter %} + + {% endif %} +
+ {% endmacro %} + + {% set team = contents.team._.pages %} + +
+
+

Organizing team

+
+
+
+ {% for page in team %} + {% set speaker = page.metadata.speaker %} + {% if speaker.core %} + {{ teamMember(contents, speaker) }} + {% endif %} + {% endfor %} +
+
+
+

Volunteer team

+
+
+ +
+ {% for page in team %} + {% set speaker = page.metadata.speaker %} + {% if not speaker.core %} + {{ teamMember(contents, speaker) }} + {% endif %} + {% endfor %} +
+ +{% endblock %} diff --git a/templates/partials/speaker-picture.html.njk b/templates/partials/speaker-picture.html.njk new file mode 100644 index 0000000..b7d914e --- /dev/null +++ b/templates/partials/speaker-picture.html.njk @@ -0,0 +1,11 @@ +{% if speaker.image.filename %} +
+ Portrait photo of {{ speaker.firstname }} {{ speaker.lastname }} +
+{% else %} +
+ +
+{% endif %}