Merge pull request #140 from nymag/generic_db-adapters

Generic db adapters

lib/services/adapters/levelup.js

@@ -0,0 +1,186 @@
+/**
+ * Mostly just converts callbacks to Promises
+ *
+ * @module
+ */
+
+'use strict';
+
+// for now, use memory.
+var _ = require('lodash'),
+  db = require('levelup')('whatever', { db: require('memdown') }),
+  bluebird = require('bluebird'),
+  jsonTransform = require('./../../streams/json-transform'),
+  chalk = require('chalk');
+
+/**
+ * Use ES6 promises
+ * @returns {{}}
+ */
+function defer() {
+  var def = bluebird.defer();
+  def.apply = function (err, result) {
+    if (err) {
+      def.reject(err);
+    } else {
+      def.resolve(result);
+    }
+  };
+  return def;
+}
+
+/**
+ * @param {string} key
+ * @param {string} value
+ * @returns {Promise}
+ */
+module.exports.put = function (key, value) {
+  var deferred = defer();
+  db.put(key, value, deferred.apply);
+  return deferred.promise;
+};
+
+/**
+ * @param {string} key
+ * @returns {Promise}
+ */
+module.exports.get = function (key) {
+  var deferred = defer();
+  db.get(key, deferred.apply);
+  return deferred.promise;
+};
+
+/**
+ * @param {string} key
+ * @returns {Promise}
+ */
+module.exports.del = function (key) {
+  var deferred = defer();
+  db.del(key, deferred.apply);
+  return deferred.promise;
+};
+
+/**
+ * Get a read stream of all the keys.
+ *
+ * @example db.list({prefix: '/components/hey'})
+ *
+ * WARNING:  Try to always end with the same character (like a /) or be completely consistent with your prefixing
+ * because the '/' character is right in the middle of the sorting range of characters.  You will get weird results
+ * if you're not careful with your ending character.  For example, `/components/text` will also return the entries of
+ * `/components/text-box`, so the search should really be `/components/text/`.
+ *
+ * @param {object} [options]  Defaults to limit of 10.
+ * @returns {ReadStream}
+ */
+module.exports.list = function (options) {
+  options = _.defaults(options || {}, {
+    limit: 10,
+    keys: true,
+    values: true,
+    fillCache: false
+  });
+
+  // The prefix option is a shortcut for a greaterThan and lessThan range.
+  if (options.prefix) {
+    // \x00 is the first possible alphanumeric character, and \xFF is the last
+    options.gte = options.prefix + '\x00';
+    options.lte = options.prefix + '\xff';
+  }
+
+  var readStream,
+    transformOptions = {
+      objectMode: options.values,
+      isArray: options.isArray
+    };
+
+  // if keys but no values, or values but no keys, always return as array.
+  if ((options.keys && !options.values) || (!options.keys && options.values)) {
+    transformOptions.isArray = true;
+  }
+
+  readStream = db.createReadStream(options);
+
+  if (_.isFunction(options.transforms)) {
+    options.transforms = options.transforms();
+  }
+
+  // apply all transforms
+  if (options.transforms) {
+    readStream = _.reduce(options.transforms, function (readStream, transform) {
+      return readStream.pipe(transform);
+    }, readStream);
+  }
+
+  return readStream.pipe(jsonTransform(transformOptions));
+};
+
+/**
+ * @param {Array} ops
+ * @param {object} [options]
+ * @returns {Promise}
+ */
+module.exports.batch = function (ops, options) {
+  var deferred = defer();
+  db.batch(ops, options || {}, deferred.apply);
+  return deferred.promise;
+};
+
+/**
+ * Clear all records from the DB.  (useful for unit testing)
+ */
+module.exports.clear = function () {
+  var errors = [],
+    ops = [],
+    deferred = defer();
+  db.createReadStream({
+    keys:true,
+    fillCache: false,
+    limit: -1
+  }).on('data', function (data) {
+    ops.push({ type: 'del', key: data.key});
+  }).on('error', function (error) {
+    errors.push(error);
+  }).on('end', function () {
+    if (errors.length) {
+      deferred.apply(_.first(errors));
+    } else {
+      db.batch(ops, deferred.apply);
+    }
+  });
+
+
+  return deferred.promise;
+};
+
+/**
+ * Format a series of batch operations in a consistent way.
+ *
+ * @param {[{type: string, key: string, value: string}]} ops
+ * @returns {[string]}
+ */
+module.exports.formatBatchOperations = function (ops) {
+  return _.map(ops, function (op) {
+    var str,
+      value = op.value;
+    try {
+      value = require('util').inspect(JSON.parse(value), { showHidden: false, depth: 5, colors: true });
+      if (value.indexOf('\n') !== -1) {
+        value = '\n' + value;
+      }
+    } catch (x) {
+      // do nothing
+    } finally {
+      str = ' ' + chalk.blue(op.key + ': ') + chalk.dim(value);
+    }
+    return str;
+  }).join('\n');
+};
+
+module.exports.getDB = function () {
+  return db;
+};
+
+module.exports.setDB = function (value) {
+  db = value;
+};

lib/services/adapters/levelup.test.js

@@ -0,0 +1,198 @@
+'use strict';
+
+var _ = require('lodash'),
+  filename = __filename.split('/').pop().split('.').shift(),
+  lib = require('./' + filename),
+  bluebird = require('bluebird'),
+  expect = require('chai').expect,
+  sinon = require('sinon'),
+  NotFoundError = require('levelup').NotFoundError;
+
+describe(_.startCase(filename), function () {
+  var sandbox;
+
+  beforeEach(function () {
+    sandbox = sinon.sandbox.create();
+    return lib.clear();
+  });
+
+  afterEach(function () {
+    sandbox.restore();
+  });
+
+  after(function () {
+    return lib.clear();
+  });
+
+  it('can put and get strings', function () {
+    return lib.put('1', '2').then(function () {
+      return lib.get('1');
+    }).done(function (result) {
+      expect(result).to.equal('2');
+    });
+  });
+
+  it('can put and del strings', function () {
+    return lib.put('1', '2').then(function () {
+      return lib.del('1');
+    }).done(function (result) {
+      expect(result).to.equal(undefined);
+    });
+  });
+
+  it('cannot get deleted strings', function (done) {
+    lib.put('1', '2').then(function () {
+      return lib.del('1');
+    }).then(function () {
+      return lib.get('1');
+    }).done(function (result) {
+      done(result); // should not happen
+    }, function (err) {
+      expect(err.name).to.equal('NotFoundError');
+      done();
+    });
+  });
+
+  describe('clear', function () {
+    var fn = lib[this.title];
+
+    it('handles db errors as promise', function (done) {
+      var on, mockOn;
+
+      // fake pipe;
+      on = function () { return on; };
+      on.on = on;
+      sandbox.stub(lib.getDB(), 'createReadStream', _.constant(on));
+      mockOn = sandbox.mock(on);
+      mockOn.expects('on').withArgs('data', sinon.match.func).yields('some data').exactly(1).returns(on);
+      mockOn.expects('on').withArgs('error', sinon.match.func).yields('whatever').exactly(1).returns(on);
+      mockOn.expects('on').withArgs('end', sinon.match.func).yields().exactly(1).returns(on);
+
+      fn().done(function (result) {
+        done(result); // should not happen
+      }, function () {
+        sandbox.verify();
+        done();
+      });
+    });
+
+    it('deletes all records', function () {
+      return lib.put('1', '2').then(function () {
+        return fn();
+      }).then(function () {
+        return bluebird.settle([lib.get('1'), lib.get('2')]);
+      }).spread(function (get1, get2) {
+
+        expect(get1.isRejected()).to.equal(true);
+        expect(get2.isRejected()).to.equal(true);
+
+      });
+    });
+  });
+
+  describe('list', function () {
+    var fn = lib[this.title];
+
+    function pipeToPromise(pipe) {
+      var str = '',
+        d = bluebird.defer();
+      pipe.on('data', function (data) { str += data; })
+        .on('error', d.reject)
+        .on('end', function () {
+          d.resolve(str);
+        });
+      return d.promise;
+    }
+
+    it('default behaviour', function () {
+      return bluebird.join(lib.put('1', '2'), lib.put('3', '4'), lib.put('5', '6'))
+        .then(function () {
+          return pipeToPromise(fn());
+        }).then(function (str) {
+          expect(str).to.equal('{"1":"2","3":"4","5":"6"}');
+        });
+    });
+
+    it('can get keys-only in array structure', function () {
+      return bluebird.join(lib.put('1', '2'), lib.put('3', '4'), lib.put('5', '6'))
+        .then(function () {
+          return pipeToPromise(fn({keys: true, values: false}));
+        }).then(function (str) {
+          expect(str).to.equal('["1","3","5"]');
+        });
+    });
+
+    it('can get values-only in array structure', function () {
+      return bluebird.join(lib.put('1', '2'), lib.put('3', '4'), lib.put('5', '6'))
+        .then(function () {
+          return pipeToPromise(fn({keys: false, values: true}));
+        }).then(function (str) {
+          expect(str).to.equal('["2","4","6"]');
+        });
+    });
+
+    it('can get key-value in object structure in array', function () {
+      return bluebird.join(lib.put('1', '2'), lib.put('3', '4'), lib.put('5', '6'))
+        .then(function () {
+          return pipeToPromise(fn({isArray: true}));
+        }).then(function (str) {
+          expect(str).to.equal('[{"key":"1","value":"2"},{"key":"3","value":"4"},{"key":"5","value":"6"}]');
+        });
+    });
+
+    it('can return empty data safely for arrays', function () {
+      return pipeToPromise(fn({isArray: true})).then(function (str) {
+        expect(str).to.equal('[]');
+      });
+    });
+
+    it('can return empty data safely for objects', function () {
+      return pipeToPromise(fn({isArray: false})).then(function (str) {
+        expect(str).to.equal('{}');
+      });
+    });
+  });
+
+  describe('formatBatchOperations', function () {
+    var fn = lib[this.title];
+
+    it('does not throw on empty batch', function () {
+      expect(function () {
+        fn([]);
+      }).to.not.throw();
+    });
+
+    it('does not throw on single operation', function () {
+      expect(function () {
+        fn([{type: 'put', key: 'a', value: '{}'}]);
+      }).to.not.throw();
+    });
+
+    it('does not throw on non-object value', function () {
+      expect(function () {
+        fn([{type: 'put', key: 'a', value: 'str'}]);
+      }).to.not.throw();
+    });
+
+    it('does not throw on large object value', function () {
+      expect(function () {
+        var obj = {};
+        _.times(100, function (index) { obj[index] = index; });
+
+        fn([{type: 'put', key: 'a', value: JSON.stringify(obj)}]);
+      }).to.not.throw();
+    });
+  });
+
+  describe('setDB', function () {
+    var fn = lib[this.title];
+
+    it('sets', function () {
+      expect(function () {
+        var db = lib.getDB();
+
+        fn(db);
+      }).to.not.throw();
+    });
+  });
+});