fix: Use new Maxmind download URLs and Basic authentication scheme

Use new Maxmind download URLs and Basic authentication scheme

README.md

@@ -6,14 +6,16 @@ Maxmind's GeoLite2 Free Databases download helper.
 
 ### Access Key
 
-**IMPORTANT** You must setup `MAXMIND_LICENSE_KEY` environment variable be able to download databases. To do so, go to the https://www.maxmind.com/en/geolite2/signup, create a free account and generate new license key.
+**IMPORTANT** You must set up `MAXMIND_ACCOUNT_ID` and `MAXMIND_LICENSE_KEY` environment variables to be able to download databases. To do so, go to the https://www.maxmind.com/en/geolite2/signup, create a free account and generate new license key.
 
-If you don't have access to the environment variables during installation, you can provide license key via `package.json`:
+If you don't have access to the environment variables during installation, you can provide config via `package.json`:
 
 ```jsonc
 {
   ...
   "geolite2": {
+    // specify the account id
+    "account-id": "<your account id>",
     // specify the key
     "license-key": "<your license key>",
     // ... or specify the file where key is located:
@@ -25,6 +27,8 @@ If you don't have access to the environment variables during installation, you c
 
 Beware of security risks of adding keys and secrets to your repository!
 
+**Note:** For backwards compatibility, the account ID is currently optional. When not provided we fall back to using legacy Maxmind download URLs with only the license key. However, this behavior may become unsupported in the future so adding an account ID is recommended. 
+
 ### Selecting databases to download
 
 You can select the dbs you want downloaded by adding a `selected-dbs` property on `geolite2` via `package.json`.

package.json

@@ -30,6 +30,7 @@
   },
   "homepage": "https://github.com/runk/node-geolite2#readme",
   "dependencies": {
+    "node-fetch": "^2.7.0",
     "tar": "^5.0.5"
   },
   "devDependencies": {

scripts/postinstall.js

@@ -1,9 +1,9 @@
 const fs = require('fs');
-const https = require('https');
 const zlib = require('zlib');
 const tar = require('tar');
 const path = require('path');
-const { getLicense, getSelectedDbs } = require('../utils');
+const fetch = require('node-fetch');
+const { getAccountId, getLicense, getSelectedDbs } = require('../utils');
 
 let licenseKey;
 try {
@@ -13,16 +13,27 @@ try {
   console.error(e.message);
 }
 
+let accountId;
+try {
+  accountId = getAccountId();
+} catch (e) {
+  console.error('geolite2: Error retrieving Maxmind Account ID');
+  console.error(e.message);
+}
+
 if (!licenseKey) {
-  console.error(`Error: License key is not configured.\n
+  console.error(`Error: License Key is not configured.\n
   You need to signup for a _free_ Maxmind account to get a license key.
-  Go to https://www.maxmind.com/en/geolite2/signup, obtain your key and
-  put it in the MAXMIND_LICENSE_KEY environment variable.
+  Go to https://www.maxmind.com/en/geolite2/signup, obtain your account ID and
+  license key and put them in the MAXMIND_ACCOUNT_ID and MAXMIND_LICENSE_KEY
+  environment variables.
 
-  If you don not have access to env vars, put this config in your package.json
+  If you do not have access to env vars, put this config in your package.json
   file (at the root level) like this:
 
   "geolite2": {
+    // specify the account id
+    "account-id": "<your account id>",
     // specify the key
     "license-key": "<your license key>",
     // ... or specify the file where key is located:
@@ -32,8 +43,12 @@ if (!licenseKey) {
   process.exit(1);
 }
 
+// If an account ID is set, use the new URL path with Basic auth.
+// Otherwise, fall back to the legacy URL path with license key as query parameter.
 const link = (edition) =>
-  `https://download.maxmind.com/app/geoip_download?edition_id=${edition}&license_key=${licenseKey}&suffix=tar.gz`;
+  accountId
+    ? `https://download.maxmind.com/geoip/databases/${edition}/download?suffix=tar.gz`
+    : `https://download.maxmind.com/app/geoip_download?edition_id=${edition}&license_key=${licenseKey}&suffix=tar.gz`;
 
 const selected = getSelectedDbs();
 const editionIds = ['City', 'Country', 'ASN']
@@ -44,25 +59,34 @@ const downloadPath = path.join(__dirname, '..', 'dbs');
 
 if (!fs.existsSync(downloadPath)) fs.mkdirSync(downloadPath);
 
-const download = (url) =>
-  new Promise((resolve) => {
-    https.get(url, (response) => {
-      resolve(response.pipe(zlib.createGunzip({})));
-    });
+const request = async (url, options) => {
+  const response = await fetch(url, {
+    headers: accountId
+      ? {
+          Authorization: `Basic ${Buffer.from(
+            `${accountId}:${licenseKey}`
+          ).toString('base64')}`,
+        }
+      : undefined,
+    redirect: 'follow',
+    ...options,
   });
 
+  if (!response.ok) {
+    throw new Error(
+      `Failed to fetch ${url}: ${response.status} ${response.statusText}`
+    );
+  }
+
+  return response;
+};
+
 // https://dev.maxmind.com/geoip/updating-databases?lang=en#checking-for-the-latest-release-date
 const isOutdated = async (dbPath, url) => {
   if (!fs.existsSync(dbPath)) return true;
 
-  const remoteLastModified = await new Promise((resolve, reject) => {
-    https
-      .request(url, { method: 'HEAD' }, (res) =>
-        resolve(Date.parse(res.headers['last-modified']))
-      )
-      .on('error', (err) => reject(err))
-      .end();
-  });
+  const response = await request(url, { method: 'HEAD' });
+  const remoteLastModified = Date.parse(response.headers['last-modified']);
   const localLastModified = fs.statSync(dbPath).mtimeMs;
 
   return localLastModified < remoteLastModified;
@@ -80,20 +104,40 @@ const main = async () => {
     }
     console.log(' > %s: Is either missing or outdated, downloading', editionId);
 
-    const result = await download(link(editionId));
-    result.pipe(tar.t()).on('entry', (entry) => {
-      if (entry.path.endsWith('.mmdb')) {
-        const dstFilename = path.join(downloadPath, path.basename(entry.path));
-        entry.pipe(fs.createWriteStream(dstFilename));
-        entry.on('end', () => {});
-      }
-    });
+    const response = await request(link(editionId));
+    const entryPromises = [];
+    await new Promise((resolve, reject) =>
+      response.body
+        .pipe(zlib.createGunzip())
+        .pipe(tar.t())
+        .on('entry', (entry) => {
+          if (entry.path.endsWith('.mmdb')) {
+            const dstFilename = path.join(
+              downloadPath,
+              path.basename(entry.path)
+            );
+            console.log(`writing ${dstFilename} ...`);
+            entryPromises.push(
+              new Promise((resolve, reject) => {
+                entry
+                  .pipe(fs.createWriteStream(dstFilename))
+                  .on('finish', resolve)
+                  .on('error', reject);
+              })
+            );
+          }
+        })
+        .on('end', resolve)
+        .on('error', reject)
+    );
+    await Promise.all(entryPromises);
   }
 };
 
 main()
   .then(() => {
     // success
+    process.exit(0);
   })
   .catch((err) => {
     console.error(err);

utils.js

@@ -21,11 +21,11 @@ const getConfigWithDir = () => {
   }
 
   console.log(
-    "WARN: geolite2 cannot find project's package.json file, using default configuration.\n" +
-    'WARN: geolite2 expects to have maxmind licence key to be present in `MAXMIND_LICENSE_KEY` env variable when package.json is unavailable.'
+    "WARN: geolite2 cannot find configuration in package.json file, using defaults.\n" +
+    "WARN: geolite2 expects to have 'MAXMIND_ACCOUNT_ID' and 'MAXMIND_LICENSE_KEY' to be present in environment variables when package.json is unavailable.",
   );
   console.log(
-    'WARN: geolite2 expected package.json to be preset at a parent of:\n%s',
+    'WARN: geolite2 expected package.json to be present at a parent of:\n%s',
     cwd
   );
 };
@@ -36,6 +36,16 @@ const getConfig = () => {
   return configWithDir.config;
 };
 
+const getAccountId = () => {
+  const envId = process.env.MAXMIND_ACCOUNT_ID;
+  if (envId) return envId;
+
+  const config = getConfig();
+  if (!config) return;
+
+  return config['account-id'];
+}
+
 const getLicense = () => {
   const envKey = process.env.MAXMIND_LICENSE_KEY;
   if (envKey) return envKey;
@@ -100,6 +110,7 @@ const getSelectedDbs = () => {
 
 module.exports = {
   getConfig,
+  getAccountId,
   getLicense,
   getSelectedDbs,
 };