Skip to content

Commit

Permalink
fix for #78 - when a posting transaction has been created, remove it …
Browse files Browse the repository at this point in the history
…from search
  • Loading branch information
brozeph committed Jul 23, 2015
1 parent cdda40c commit a8ab8ef
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 22 deletions.
42 changes: 40 additions & 2 deletions init/misc/sync-es.js
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,38 @@ var
module.exports = (function (app) {
'use strict';

var nonIndexedPostings = [];

/**
* Collection recent transactions for filtering postings from search
**/
function collectionRecentTransactions (callback) {
var
now = new Date(),
past = new Date().setDate(now.getDate() - 30);

app.log.info('retrieving recent transactions for posting exclusions');

app
.db
.collection('transactions')
.find({
createdAt : { $gte : past }
}, { postingId : true })
.toArray(function (err, transactions) {
if (err) {
return callback(err);
}

transactions.forEach(function (transaction) {
nonIndexedPostings.push(transaction.postingId);
});

app.log.info('found %d posting exclusions', nonIndexedPostings.length);
return callback();
});
}

/**
* Connect to Elasticsearch
**/
Expand Down Expand Up @@ -135,7 +167,8 @@ module.exports = (function (app) {
config : loadConfig,
es : ['config', 'logging', connectToElasticsearch],
logging : ['config', createLogger],
mongo : ['config', 'logging', connectToMongo]
mongo : ['config', 'logging', connectToMongo],
recentTransactions : ['mongo', collectionRecentTransactions],
}, function (err) {
if (err) {
(app.log || console).error(err);
Expand Down Expand Up @@ -164,7 +197,12 @@ module.exports = (function (app) {
app
.db
.collection('postings')
.find({})
// don't grab postings we shouldn't search index
.find({
postingId : {
$nin : nonIndexedPostings
}
})
.skip(skip)
.limit(limit)
.toArray(function (err, postings) {
Expand Down
2 changes: 1 addition & 1 deletion lib/data/elasticsearch.js
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
var
tokenizer = require('./tokenizer'),
// tokenizer = require('./tokenizer'),

ES_SUPPORTED_KEYWORDS = [
'beginsWith',
Expand Down
63 changes: 44 additions & 19 deletions lib/data/postings.js
Original file line number Diff line number Diff line change
Expand Up @@ -659,7 +659,12 @@ module.exports = function (app, es, self) {
});
};

self.remove = function (postingId, callback) {
self.remove = function (postingId, removeAllRecords, callback) {
if (typeof callback === 'undefined' && typeof removeAllRecords === 'function') {
callback = removeAllRecords;
removeAllRecords = true;
}

var verr;

Posting
Expand All @@ -670,27 +675,47 @@ module.exports = function (app, es, self) {
return callback(verr);
}

posting.remove(function (err) {
if (err) {
verr =
new VError(err, 'removal of posting %s has failed', postingId);
return callback(verr);
}
async.series([

// remove from Mongo (if removeAllRecords === true)
function (done) {
if (!removeAllRecords) {
return setImmediate(done);
}

posting.remove(function (err) {
if (err) {
verr =
new VError(err, 'removal of posting %s has failed', postingId);

return done(verr);
}

return done();
});
},

// remove from Elasticsearch
es.delete({
_id : postingId,
_type : DEFAULT_POSTING_TYPE
}, function (err) {
if (err) {
app.log.warn(
'unable to remove posting %s from Elasticsearch',
postingId);
app.log.warn(err);
function (done) {
es.delete({
_id : postingId,
_type : DEFAULT_POSTING_TYPE
}, function (err) {
if (err) {
// only warn - this isn't a critical breaking behavior
app.log.warn(
'unable to remove posting %s from Elasticsearch',
postingId);
app.log.warn(err);
}

return;
}
});
return done();
});
}
], function (err) {
if (err) {
return callback(err);
}

return callback(null, posting.toObject({ transform : true }));
});
Expand Down
22 changes: 22 additions & 0 deletions lib/models/transactions.js
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ module.exports = function (app, data, services, self) {
});
},

// create the transaction
function (done) {
// create an ID for the transaction
transactionId = uuid.v4().replace(/-/g, '');
Expand Down Expand Up @@ -205,6 +206,27 @@ module.exports = function (app, data, services, self) {
countdown(startTime, new Date(), countdown.MILLISECONDS).toString());

// return
return done(null, newTransaction);
});
},

// remove posting from search
function (newTransaction, done) {
app.log.trace(
'removing posting %s from search',
newTransaction.postingId);

data.postings.remove(
newTransaction.postingId,
false, // only remove from search index
function (err) {
if (err) {
// not an error worth sending an error response for
app.log.warn(
'unable to remove posting %s from search',
newTransaction.postingId);
}

return done(null, newTransaction);
});
}
Expand Down

0 comments on commit a8ab8ef

Please sign in to comment.