From a8ab8ef400cf2e6efafd6e2530c2d02ff876240b Mon Sep 17 00:00:00 2001
From: Joshua Thomas <joshua.thomas@gmail.com>
Date: Thu, 23 Jul 2015 07:32:47 -0700
Subject: [PATCH] fix for #78 - when a posting transaction has been created,
 remove it from search

---
 init/misc/sync-es.js       | 42 +++++++++++++++++++++++--
 lib/data/elasticsearch.js  |  2 +-
 lib/data/postings.js       | 63 ++++++++++++++++++++++++++------------
 lib/models/transactions.js | 22 +++++++++++++
 4 files changed, 107 insertions(+), 22 deletions(-)

diff --git a/init/misc/sync-es.js b/init/misc/sync-es.js
index 9124dee..ad32363 100644
--- a/init/misc/sync-es.js
+++ b/init/misc/sync-es.js
@@ -19,6 +19,38 @@ var
 module.exports = (function (app) {
 	'use strict';
 
+	var nonIndexedPostings = [];
+
+	/**
+	 * Collection recent transactions for filtering postings from search
+	 **/
+	function collectionRecentTransactions (callback) {
+		var
+			now = new Date(),
+			past = new Date().setDate(now.getDate() - 30);
+
+		app.log.info('retrieving recent transactions for posting exclusions');
+
+		app
+			.db
+			.collection('transactions')
+			.find({
+				createdAt : { $gte : past }
+			}, { postingId : true })
+			.toArray(function (err, transactions) {
+				if (err) {
+					return callback(err);
+				}
+
+				transactions.forEach(function (transaction) {
+					nonIndexedPostings.push(transaction.postingId);
+				});
+
+				app.log.info('found %d posting exclusions', nonIndexedPostings.length);
+				return callback();
+			});
+	}
+
 	/**
 	 * Connect to Elasticsearch
 	 **/
@@ -135,7 +167,8 @@ module.exports = (function (app) {
 		config : loadConfig,
 		es : ['config', 'logging', connectToElasticsearch],
 		logging : ['config', createLogger],
-		mongo : ['config', 'logging', connectToMongo]
+		mongo : ['config', 'logging', connectToMongo],
+		recentTransactions : ['mongo', collectionRecentTransactions],
 	}, function (err) {
 		if (err) {
 			(app.log || console).error(err);
@@ -164,7 +197,12 @@ module.exports = (function (app) {
 				app
 					.db
 					.collection('postings')
-					.find({})
+					// don't grab postings we shouldn't search index
+					.find({
+						postingId : {
+							$nin : nonIndexedPostings
+						}
+					})
 					.skip(skip)
 					.limit(limit)
 					.toArray(function (err, postings) {
diff --git a/lib/data/elasticsearch.js b/lib/data/elasticsearch.js
index 3459881..fde890d 100644
--- a/lib/data/elasticsearch.js
+++ b/lib/data/elasticsearch.js
@@ -1,5 +1,5 @@
 var
-	tokenizer = require('./tokenizer'),
+	// tokenizer = require('./tokenizer'),
 
 	ES_SUPPORTED_KEYWORDS = [
 		'beginsWith',
diff --git a/lib/data/postings.js b/lib/data/postings.js
index 0fbcf6d..357cb61 100644
--- a/lib/data/postings.js
+++ b/lib/data/postings.js
@@ -659,7 +659,12 @@ module.exports = function (app, es, self) {
 			});
 	};
 
-	self.remove = function (postingId, callback) {
+	self.remove = function (postingId, removeAllRecords, callback) {
+		if (typeof callback === 'undefined' && typeof removeAllRecords === 'function') {
+			callback = removeAllRecords;
+			removeAllRecords = true;
+		}
+
 		var verr;
 
 		Posting
@@ -670,27 +675,47 @@ module.exports = function (app, es, self) {
 					return callback(verr);
 				}
 
-				posting.remove(function (err) {
-					if (err) {
-						verr =
-							new VError(err, 'removal of posting %s has failed', postingId);
-						return callback(verr);
-					}
+				async.series([
+
+					// remove from Mongo (if removeAllRecords === true)
+					function (done) {
+						if (!removeAllRecords) {
+							return setImmediate(done);
+						}
+
+						posting.remove(function (err) {
+							if (err) {
+								verr =
+									new VError(err, 'removal of posting %s has failed', postingId);
+
+								return done(verr);
+							}
+
+							return done();
+						});
+					},
 
 					// remove from Elasticsearch
-					es.delete({
-						_id : postingId,
-						_type : DEFAULT_POSTING_TYPE
-					}, function (err) {
-						if (err) {
-							app.log.warn(
-								'unable to remove posting %s from Elasticsearch',
-								postingId);
-							app.log.warn(err);
+					function (done) {
+						es.delete({
+							_id : postingId,
+							_type : DEFAULT_POSTING_TYPE
+						}, function (err) {
+							if (err) {
+								// only warn - this isn't a critical breaking behavior
+								app.log.warn(
+									'unable to remove posting %s from Elasticsearch',
+									postingId);
+								app.log.warn(err);
+							}
 
-							return;
-						}
-					});
+							return done();
+						});
+					}
+				], function (err) {
+					if (err) {
+						return callback(err);
+					}
 
 					return callback(null, posting.toObject({ transform : true }));
 				});
diff --git a/lib/models/transactions.js b/lib/models/transactions.js
index 63295c9..ff83e88 100644
--- a/lib/models/transactions.js
+++ b/lib/models/transactions.js
@@ -178,6 +178,7 @@ module.exports = function (app, data, services, self) {
 						});
 				},
 
+				// create the transaction
 				function (done) {
 					// create an ID for the transaction
 					transactionId = uuid.v4().replace(/-/g, '');
@@ -205,6 +206,27 @@ module.exports = function (app, data, services, self) {
 								countdown(startTime, new Date(), countdown.MILLISECONDS).toString());
 
 							// return
+							return done(null, newTransaction);
+						});
+				},
+
+				// remove posting from search
+				function (newTransaction, done) {
+					app.log.trace(
+						'removing posting %s from search',
+						newTransaction.postingId);
+
+					data.postings.remove(
+						newTransaction.postingId,
+						false, // only remove from search index
+						function (err) {
+							if (err) {
+								// not an error worth sending an error response for
+								app.log.warn(
+									'unable to remove posting %s from search',
+									newTransaction.postingId);
+							}
+
 							return done(null, newTransaction);
 						});
 				}