(Fork) Use new Maxmind download URLs and Basic authentication scheme

README.md

@@ -6,14 +6,16 @@ Maxmind's GeoLite2 Free Databases download helper.
 
 ### Access Key
 
-**IMPORTANT** You must setup `MAXMIND_LICENSE_KEY` environment variable be able to download databases. To do so, go to the https://www.maxmind.com/en/geolite2/signup, create a free account and generate new license key.
+**IMPORTANT** You must set up `MAXMIND_ACCOUNT_ID` and `MAXMIND_LICENSE_KEY` environment variables to be able to download databases. To do so, go to the https://www.maxmind.com/en/geolite2/signup, create a free account and generate new license key.
 
-If you don't have access to the environment variables during installation, you can provide license key via `package.json`:
+If you don't have access to the environment variables during installation, you can provide config via `package.json`:
 
 ```jsonc
 {
   ...
   "geolite2": {
+    // specify the account id
+    "account-id": "<your account id>",
     // specify the key
     "license-key": "<your license key>",
     // ... or specify the file where key is located:

scripts/postinstall.js

@@ -3,7 +3,7 @@ const https = require('https');
 const zlib = require('zlib');
 const tar = require('tar');
 const path = require('path');
-const { getLicense, getSelectedDbs } = require('../utils');
+const { getAccountId, getLicense, getSelectedDbs } = require('../utils');
 
 let licenseKey;
 try {
@@ -13,16 +13,26 @@ try {
   console.error(e.message);
 }
 
-if (!licenseKey) {
-  console.error(`Error: License key is not configured.\n
+let accountId;
+try {
+  accountId = getAccountId();
+} catch (e) {
+  console.error('geolite2: Error retrieving Maxmind Account ID');
+  console.error(e.message);
+}
+
+if (!licenseKey || !accountId) {
+  console.error(`Error: License Key or Account ID is not configured.\n
   You need to signup for a _free_ Maxmind account to get a license key.
   Go to https://www.maxmind.com/en/geolite2/signup, obtain your key and
   put it in the MAXMIND_LICENSE_KEY environment variable.
 
-  If you don not have access to env vars, put this config in your package.json
+  If you do not have access to env vars, put this config in your package.json
   file (at the root level) like this:
 
   "geolite2": {
+    // specify the account id
+    "account-id": "<your account id>",
     // specify the key
     "license-key": "<your license key>",
     // ... or specify the file where key is located:
@@ -33,7 +43,7 @@ if (!licenseKey) {
 }
 
 const link = (edition) =>
-  `https://download.maxmind.com/app/geoip_download?edition_id=${edition}&license_key=${licenseKey}&suffix=tar.gz`;
+  `https://download.maxmind.com/geoip/databases/${edition}/download?suffix=tar.gz`;
 
 const selected = getSelectedDbs();
 const editionIds = ['City', 'Country', 'ASN']
@@ -44,25 +54,40 @@ const downloadPath = path.join(__dirname, '..', 'dbs');
 
 if (!fs.existsSync(downloadPath)) fs.mkdirSync(downloadPath);
 
-const download = (url) =>
-  new Promise((resolve) => {
-    https.get(url, (response) => {
-      resolve(response.pipe(zlib.createGunzip({})));
-    });
+const request = async (url, options) => {
+  const response = await new Promise((resolve, reject) => {
+    https
+      .request(url, {
+        auth: `${accountId}:${licenseKey}`,
+        ...options
+      }, (response) => {
+        resolve(response);
+      })
+      .on('error', (err) => reject(err))
+      .end();
   });
 
+  if (response.statusCode >= 300 && response.statusCode < 400 && response.headers.location) {
+    // Handle redirect
+    return request(response.headers.location, {
+      ...options,
+      auth: null
+    });
+  }
+
+  if (response.statusCode !== 200) {
+    throw new Error(`Request failed to ${url} - ${response.statusCode} ${response.statusMessage}`);
+  }
+
+  return response;
+};
+
 // https://dev.maxmind.com/geoip/updating-databases?lang=en#checking-for-the-latest-release-date
 const isOutdated = async (dbPath, url) => {
   if (!fs.existsSync(dbPath)) return true;
 
-  const remoteLastModified = await new Promise((resolve, reject) => {
-    https
-      .request(url, { method: 'HEAD' }, (res) =>
-        resolve(Date.parse(res.headers['last-modified']))
-      )
-      .on('error', (err) => reject(err))
-      .end();
-  });
+  const response = await request(url, { method: 'HEAD' });
+  const remoteLastModified = Date.parse(response.headers['last-modified']);
   const localLastModified = fs.statSync(dbPath).mtimeMs;
 
   return localLastModified < remoteLastModified;
@@ -80,14 +105,24 @@ const main = async () => {
     }
     console.log(' > %s: Is either missing or outdated, downloading', editionId);
 
-    const result = await download(link(editionId));
-    result.pipe(tar.t()).on('entry', (entry) => {
-      if (entry.path.endsWith('.mmdb')) {
-        const dstFilename = path.join(downloadPath, path.basename(entry.path));
-        entry.pipe(fs.createWriteStream(dstFilename));
-        entry.on('end', () => {});
-      }
-    });
+    const response = await request(link(editionId));
+    const entryPromises = [];
+    await new Promise((resolve, reject) => response
+      .pipe(zlib.createGunzip())
+      .pipe(tar.t())
+      .on('entry', (entry) => {
+        if (entry.path.endsWith('.mmdb')) {
+          const dstFilename = path.join(downloadPath, path.basename(entry.path));
+          console.log(`writing ${dstFilename} ...`);
+          entryPromises.push(new Promise((resolve, reject) => {
+            entry.pipe(fs.createWriteStream(dstFilename)).on('finish', resolve).on('error', reject);
+          }));
+        }
+      })
+      .on('end', resolve)
+      .on('error', reject)
+    );
+    await Promise.all(entryPromises);
   }
 };
 

utils.js

@@ -34,6 +34,16 @@ const getConfig = () => {
   return configWithDir.config;
 };
 
+const getAccountId = () => {
+  const envId = process.env.MAXMIND_ACCOUNT_ID;
+  if (envId) return envId;
+
+  const config = getConfig();
+  if (!config) return;
+
+  return config['account-id'];
+}
+
 const getLicense = () => {
   const envKey = process.env.MAXMIND_LICENSE_KEY;
   if (envKey) return envKey;
@@ -94,6 +104,7 @@ const getSelectedDbs = () => {
 
 module.exports = {
   getConfig,
+  getAccountId,
   getLicense,
   getSelectedDbs,
 };