feat(driver-bench): add driver benchmarks to dbx-tools

package.json

@@ -3,7 +3,7 @@
   "private": true,
   "workspaces": [
     "packages/bson-bench",
-    "packages/eslint-config"
+    "packages/driver-bench"
   ],
   "devDependencies": {
     "@tsconfig/node16": "^16.1.3",
@@ -27,7 +27,7 @@
     "typescript-cached-transpile": "^0.0.6"
   },
   "scripts": {
-    "check:eslint": "eslint -v && eslint --max-warnings=0 --ext '.js,.ts' packages/**/src packages/**/test",
+    "check:eslint": "eslint -v && eslint --max-warnings=0 --ext '.js,.ts,.mts' packages/**/src packages/**/test",
     "fix:eslint": "npm run check:eslint -- --fix"
   }
 }

packages/driver-bench/.gitignore

@@ -0,0 +1,2 @@
+node_modules/
+lib/

packages/driver-bench/LICENSE

@@ -0,0 +1,13 @@
+ Copyright © 2023 MongoDB, Inc
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

packages/driver-bench/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "driver-bench",
+  "version": "1.0.0",
+  "description": "",
+  "module": "./lib/mod.mjs",
+  "bin": {
+    "mndb": "./lib/cli.mjs"
+  },
+  "scripts": {
+    "test": "echo \"Warning: no test specified\" && exit 0",
+    "prepare": "tsc"
+  },
+  "keywords": [],
+  "author": "The MongoDB NodeJS Team <dbx-node@mongodb.com>",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "bson": "^6.7.0"
+  }
+}

packages/driver-bench/readme.md

@@ -0,0 +1,21 @@
+# MongoDB Driver benchmark tool
+
+Node.js driver's implementation of the specification benchmarks.
+The intention is to keep the benchmarks agonostic of driver version in order to facilitate comparisons.
+
+```sh
+env MONGODB_DRIVER_PATH=.../mongodb/lib/index.js node ./lib/cli.mjs
+```
+
+## CLI work in progress
+
+### Wishlist
+
+- [ ] Control driver path with CLI flags
+- [ ] Request that a version from npm be downloaded and tested
+- [ ] Request that a version from git be cloned, built and tested
+- [ ] Make comparison easier by renaming output files and pretty printing them
+  - [ ] Make comparison _even_ easier by fetching results from EVG.
+    - Currently, our results are not made available in a nice way. Predictable S3 URL?
+- [ ] Export an API that makes it easy to script running benches against more than one copy of the driver
+  - Again in service of comparison: `expect(bench(path)).to.be.faster.than(bench(otherPath))`

packages/driver-bench/src/cli.mts

@@ -0,0 +1,24 @@
+#!/usr/bin/env node
+
+import child_process from 'node:child_process';
+import events from 'node:events';
+import module from 'node:module';
+import path from 'node:path';
+import process from 'node:process';
+
+let require;
+
+let { MONGODB_DRIVER_PATH = '' } = process.env;
+const { DEBUG_BENCH = 'no' } = process.env;
+
+if (MONGODB_DRIVER_PATH === '') {
+  require ??= module.createRequire(import.meta.dirname);
+  MONGODB_DRIVER_PATH = require.resolve('mongodb');
+}
+
+const benchmark = child_process.fork(path.join(import.meta.dirname, './driverBench/index.mjs'), {
+  execArgv: DEBUG_BENCH === 'yes' ? ['--enable-source-maps'] : [],
+  stdio: 'inherit',
+  env: { MONGODB_DRIVER_PATH }
+});
+await events.once(benchmark, 'exit');

packages/driver-bench/src/driverBench/common.mts

@@ -0,0 +1,106 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import process from 'node:process';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+
+const { MONGODB_DRIVER_PATH = '' } = process.env;
+const { MongoClient, GridFSBucket } = await import(
+  MONGODB_DRIVER_PATH.length !== 0 ? MONGODB_DRIVER_PATH : 'mongodb'
+);
+
+const DB_NAME = 'perftest';
+const COLLECTION_NAME = 'corpus';
+
+const SPEC_DIRECTORY = path.resolve(import.meta.dirname, '..', '..', 'src', 'driverBench', 'spec');
+
+export function loadSpecFile(filePath): Buffer;
+export function loadSpecFile(filePath, encoding): Buffer;
+export function loadSpecFile(filePath, encoding: 'utf8'): string;
+export function loadSpecFile(filePath, encoding?: BufferEncoding): string | Buffer {
+  const fp = [SPEC_DIRECTORY].concat(filePath);
+  return fs.readFileSync(path.join(...fp), encoding);
+}
+
+export function loadSpecString(filePath) {
+  return loadSpecFile(filePath, 'utf8');
+}
+
+export function makeClient() {
+  this.client = new MongoClient(process.env.MONGODB_URI || 'mongodb://127.0.0.1:27017', {
+    serverSelectionTimeoutMS: 2000
+  });
+}
+
+export function connectClient() {
+  return this.client.connect();
+}
+
+export function disconnectClient() {
+  this.client.close();
+}
+
+export function initDb() {
+  this.db = this.client.db(DB_NAME);
+}
+
+export function dropDb() {
+  return this.db.dropDatabase();
+}
+
+export function createCollection() {
+  return this.db.createCollection(COLLECTION_NAME);
+}
+
+export function initCollection() {
+  this.collection = this.db.collection(COLLECTION_NAME);
+}
+
+export function dropCollection() {
+  return this.collection.drop().catch(e => {
+    if (e.code !== 26 /* NamespaceNotFound */) {
+      throw e;
+    }
+  });
+}
+
+export function initBucket() {
+  this.bucket = new GridFSBucket(this.db);
+}
+
+export function dropBucket() {
+  return this.bucket && this.bucket.drop();
+}
+
+export function makeLoadJSON(name) {
+  return function () {
+    this.doc = JSON.parse(loadSpecString(['single_and_multi_document', name]));
+  };
+}
+
+export function makeLoadTweets(makeId) {
+  return function () {
+    const doc = this.doc;
+    const tweets = [];
+    for (let _id = 1; _id <= 10000; _id += 1) {
+      tweets.push(Object.assign({}, doc, makeId ? { _id } : {}));
+    }
+
+    return this.collection.insertMany(tweets);
+  };
+}
+
+export function makeLoadInsertDocs(numberOfOperations) {
+  return function () {
+    this.docs = [];
+    for (let i = 0; i < numberOfOperations; i += 1) {
+      this.docs.push(Object.assign({}, this.doc));
+    }
+  };
+}
+
+export async function writeSingleByteFileToBucket() {
+  const stream = this.bucket.openUploadStream('setup-file.txt');
+  const oneByteFile = Readable.from('a');
+  return pipeline(oneByteFile, stream);
+}

packages/driver-bench/src/driverBench/index.mts

@@ -0,0 +1,101 @@
+import { writeFile } from 'node:fs/promises';
+import os from 'node:os';
+import { inspect } from 'node:util';
+
+import { Runner } from '../mongoBench/index.mjs';
+import {
+  makeMultiBench,
+  makeParallelBenchmarks,
+  makeSingleBench
+} from '../mongoBench/suites/index.mjs';
+
+const bsonType = 'js-bson';
+// TODO(NODE-4606): test against different driver configurations in CI
+const hw = os.cpus();
+const ram = os.totalmem() / 1024 ** 3;
+const platform = { name: hw[0].model, cores: hw.length, ram: `${ram}GB` };
+
+const systemInfo = () =>
+  [
+    `\n- cpu: ${platform.name}`,
+    `- cores: ${platform.cores}`,
+    `- arch: ${os.arch()}`,
+    `- os: ${process.platform} (${os.release()})`,
+    `- ram: ${platform.ram}\n`
+  ].join('\n');
+console.log(systemInfo());
+
+function average(arr) {
+  return arr.reduce((x, y) => x + y, 0) / arr.length;
+}
+
+const benchmarkRunner = new Runner()
+  .suite('singleBench', suite => makeSingleBench(suite))
+  .suite('multiBench', suite => makeMultiBench(suite))
+  .suite('parallel', suite => makeParallelBenchmarks(suite));
+
+benchmarkRunner
+  .run()
+  .then(microBench => {
+    const singleBench = average([
+      microBench.singleBench.findOne,
+      microBench.singleBench.smallDocInsertOne,
+      microBench.singleBench.largeDocInsertOne
+    ]);
+    const multiBench = average(Object.values(microBench.multiBench));
+
+    const parallelBench = average([
+      microBench.parallel.ldjsonMultiFileUpload,
+      microBench.parallel.ldjsonMultiFileExport,
+      microBench.parallel.gridfsMultiFileUpload,
+      microBench.parallel.gridfsMultiFileDownload
+    ]);
+
+    const readBench = average([
+      microBench.singleBench.findOne,
+      microBench.multiBench.findManyAndEmptyCursor,
+      microBench.multiBench.gridFsDownload,
+      microBench.parallel.gridfsMultiFileDownload,
+      microBench.parallel.ldjsonMultiFileExport
+    ]);
+    const writeBench = average([
+      microBench.singleBench.smallDocInsertOne,
+      microBench.singleBench.largeDocInsertOne,
+      microBench.multiBench.smallDocBulkInsert,
+      microBench.multiBench.largeDocBulkInsert,
+      microBench.multiBench.gridFsUpload,
+      microBench.parallel.ldjsonMultiFileUpload,
+      microBench.parallel.gridfsMultiFileUpload
+    ]);
+
+    const driverBench = average([readBench, writeBench]);
+
+    const benchmarkResults = {
+      singleBench,
+      multiBench,
+      parallelBench,
+      readBench,
+      writeBench,
+      driverBench,
+      ...microBench.parallel,
+      ...microBench.bsonBench,
+      ...microBench.singleBench,
+      ...microBench.multiBench
+    };
+
+    return Object.entries(benchmarkResults).map(([benchmarkName, result]) => {
+      return {
+        info: {
+          test_name: benchmarkName,
+          tags: [bsonType]
+        },
+        metrics: [{ name: 'megabytes_per_second', value: result }]
+      };
+    });
+  })
+  .then(data => {
+    const results = JSON.stringify(data, undefined, 2);
+    console.log(inspect(data, { depth: Infinity, colors: true }));
+    return writeFile('results.json', results);
+  })
+  .catch(err => console.error(err));