diff --git a/files_find/Files Find/bin/src/app.js b/files_find/Files Find/bin/src/app.js index 0af40ee..3fab9e3 100644 --- a/files_find/Files Find/bin/src/app.js +++ b/files_find/Files Find/bin/src/app.js @@ -1,21 +1,38 @@ var dir_recursive = require("recursive-readdir"); const fs = require("fs"); +const path = require("path"); var dir_func = fs.readdir; -const path = require('path'); var comparer = require("natural-compare-lite"); const matcher = require('matcher'); +const fspromises = require('fs/promises'); +const lockfile = require('proper-lockfile'); //prepare logging var log_file = fs.createWriteStream(__dirname + '/debug.log', {flags : 'w'}); log_file.write("Arguments: " + JSON.stringify(process.argv) + "\n") +function loginfo(...what){ + log_file.write(what.join(" ") + "\n"); + process.stdout.write(what.join(" ") + "\n") +} + +console.log = loginfo; +console.error = loginfo; + //read input job ticket let rawdata = fs.readFileSync(process.argv[2],'utf8').replace(/^\uFEFF/, ""); //get rid of UTF8 BOM log_file.write("JOBTICKET"); -//log_file.write(rawdata); +log_file.write(rawdata); let o_job = JSON.parse(rawdata); console.log(o_job["proc_data"]); +//INTERNAL VARS +var cache_dir = o_job.cache_dir; +loginfo("cache_dir: " + cache_dir); +var watch_db_dir = path.join(cache_dir,"files_find"); +var watch_db_file = path.join(watch_db_dir,"watch_"+o_job.nodes.next.id); +loginfo("watch_db_file: " + watch_db_file); + //GET OUTPUTS - make them available as global objects var o_out_files = filterById(o_job["proc_data"]["outputs"],"output_found_array"); var o_out_count = filterById(o_job["proc_data"]["outputs"],"output_found_count"); @@ -78,14 +95,20 @@ if (recursive){ } console.log(dir_func) -//search the filesysem var all_files =[]; -findfiles(); +main(); -function findfiles(){ +//MAIN FUNCTION +async function main(){ + console.log("Findfiles start "+root_path+"\n"); - dir_func(root_path, function (err, files) { - if (err){ + + dir_func(root_path, async function (err, files) { + + //todo: only do this when lock is enabled + lockWatchDb(); + + if (err){ console.log("Error \n" + err + "\n"); return; } @@ -167,6 +190,11 @@ function findfiles(){ //skip sizes all_files = filter_size(all_files); + + //skip already known files + + all_files = await processWatchDb(all_files); + //fill single output file output parameters var o_out_array = filterByTerm(o_job["proc_data"]["outputs"],"output_files"); for (i=0;i setTimeout(resolve, ms)); +} + function sort_output_array(a_allfiles){ console.log("Sorting by",sort_order) if (sort_order.match("Date Created")){ @@ -320,4 +352,53 @@ function format_input_path(what){ throw(what + " does not exist"); } return what; +} + +//WATCHDATABASE + +async function lockWatchDb(){ + if (!fs.existsSync(watch_db_dir)){ + console.log("Creating watchdb dir",watch_db_dir); + fspromises.mkdir(watch_db_dir, {}).catch((err) => { + console.error("Error, could not create folder",watch_db_dir) + }) + } + if (!fs.existsSync(watch_db_file)){ + console.log("Creating watchdb dir",watch_db_file); + await fspromises.writeFile(watch_db_file,"[]"); + }else{ + //wait until we can write to the db File + var is_locked = true; + await lockfile.lock(watch_db_file,{stale:5000,retries:10});//after 1 minute we continue with lock + console.log("Locked the watchdb"); + + } +} + +async function processWatchDb(residentFiles){ + //delete entries not in residentFiles but in oldFiles + + var oldFiles = await fspromises.readFile(watch_db_file,"utf-8"); + console.log("oldfiles",oldFiles); + var oldFiles = JSON.parse(oldFiles); + //build "new files" (difference between original files in watchdb and oldFiles_without_stale) + + + let staleFiles = oldFiles.filter(x => !residentFiles.includes(x)); //files do not exist anymore + let oldFiles_without_stale = oldFiles.filter(x => !staleFiles.includes(x)); + + + //store sum of oldFiles_without_stale and unknown files + let newFiles = residentFiles.filter(x => !oldFiles_without_stale.includes(x)); + + let toRemember = [...oldFiles_without_stale,...newFiles]; + console.log("remembering",toRemember.length,"files") + await fspromises.writeFile(watch_db_file,JSON.stringify(toRemember,null,2),"utf-8"); + + + console.log("newFiles",JSON.stringify(newFiles)); + return newFiles; + + + } \ No newline at end of file diff --git a/files_find/Files Find/bin/src/node_modules/.package-lock.json b/files_find/Files Find/bin/src/node_modules/.package-lock.json index 950b623..d8b64ab 100644 --- a/files_find/Files Find/bin/src/node_modules/.package-lock.json +++ b/files_find/Files Find/bin/src/node_modules/.package-lock.json @@ -1,7 +1,7 @@ { "name": "src", "version": "1.0.0", - "lockfileVersion": 2, + "lockfileVersion": 3, "requires": true, "packages": { "node_modules/balanced-match": { @@ -34,6 +34,11 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/graceful-fs": { + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" + }, "node_modules/matcher": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", @@ -61,6 +66,16 @@ "resolved": "https://registry.npmjs.org/natural-compare-lite/-/natural-compare-lite-1.4.0.tgz", "integrity": "sha1-F7CVgZiJef3a/gIB6TG6kzyWy7Q=" }, + "node_modules/proper-lockfile": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/proper-lockfile/-/proper-lockfile-4.1.2.tgz", + "integrity": "sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==", + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + } + }, "node_modules/recursive-readdir": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/recursive-readdir/-/recursive-readdir-2.2.2.tgz", @@ -71,6 +86,19 @@ "engines": { "node": ">=0.10.0" } + }, + "node_modules/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==", + "engines": { + "node": ">= 4" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" } } } diff --git a/files_find/Files Find/bin/src/node_modules/graceful-fs/LICENSE b/files_find/Files Find/bin/src/node_modules/graceful-fs/LICENSE new file mode 100644 index 0000000..e906a25 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/graceful-fs/LICENSE @@ -0,0 +1,15 @@ +The ISC License + +Copyright (c) 2011-2022 Isaac Z. Schlueter, Ben Noordhuis, and Contributors + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR +IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/files_find/Files Find/bin/src/node_modules/graceful-fs/README.md b/files_find/Files Find/bin/src/node_modules/graceful-fs/README.md new file mode 100644 index 0000000..82d6e4d --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/graceful-fs/README.md @@ -0,0 +1,143 @@ +# graceful-fs + +graceful-fs functions as a drop-in replacement for the fs module, +making various improvements. + +The improvements are meant to normalize behavior across different +platforms and environments, and to make filesystem access more +resilient to errors. + +## Improvements over [fs module](https://nodejs.org/api/fs.html) + +* Queues up `open` and `readdir` calls, and retries them once + something closes if there is an EMFILE error from too many file + descriptors. +* fixes `lchmod` for Node versions prior to 0.6.2. +* implements `fs.lutimes` if possible. Otherwise it becomes a noop. +* ignores `EINVAL` and `EPERM` errors in `chown`, `fchown` or + `lchown` if the user isn't root. +* makes `lchmod` and `lchown` become noops, if not available. +* retries reading a file if `read` results in EAGAIN error. + +On Windows, it retries renaming a file for up to one second if `EACCESS` +or `EPERM` error occurs, likely because antivirus software has locked +the directory. + +## USAGE + +```javascript +// use just like fs +var fs = require('graceful-fs') + +// now go and do stuff with it... +fs.readFile('some-file-or-whatever', (err, data) => { + // Do stuff here. +}) +``` + +## Sync methods + +This module cannot intercept or handle `EMFILE` or `ENFILE` errors from sync +methods. If you use sync methods which open file descriptors then you are +responsible for dealing with any errors. + +This is a known limitation, not a bug. + +## Global Patching + +If you want to patch the global fs module (or any other fs-like +module) you can do this: + +```javascript +// Make sure to read the caveat below. +var realFs = require('fs') +var gracefulFs = require('graceful-fs') +gracefulFs.gracefulify(realFs) +``` + +This should only ever be done at the top-level application layer, in +order to delay on EMFILE errors from any fs-using dependencies. You +should **not** do this in a library, because it can cause unexpected +delays in other parts of the program. + +## Changes + +This module is fairly stable at this point, and used by a lot of +things. That being said, because it implements a subtle behavior +change in a core part of the node API, even modest changes can be +extremely breaking, and the versioning is thus biased towards +bumping the major when in doubt. + +The main change between major versions has been switching between +providing a fully-patched `fs` module vs monkey-patching the node core +builtin, and the approach by which a non-monkey-patched `fs` was +created. + +The goal is to trade `EMFILE` errors for slower fs operations. So, if +you try to open a zillion files, rather than crashing, `open` +operations will be queued up and wait for something else to `close`. + +There are advantages to each approach. Monkey-patching the fs means +that no `EMFILE` errors can possibly occur anywhere in your +application, because everything is using the same core `fs` module, +which is patched. However, it can also obviously cause undesirable +side-effects, especially if the module is loaded multiple times. + +Implementing a separate-but-identical patched `fs` module is more +surgical (and doesn't run the risk of patching multiple times), but +also imposes the challenge of keeping in sync with the core module. + +The current approach loads the `fs` module, and then creates a +lookalike object that has all the same methods, except a few that are +patched. It is safe to use in all versions of Node from 0.8 through +7.0. + +### v4 + +* Do not monkey-patch the fs module. This module may now be used as a + drop-in dep, and users can opt into monkey-patching the fs builtin + if their app requires it. + +### v3 + +* Monkey-patch fs, because the eval approach no longer works on recent + node. +* fixed possible type-error throw if rename fails on windows +* verify that we *never* get EMFILE errors +* Ignore ENOSYS from chmod/chown +* clarify that graceful-fs must be used as a drop-in + +### v2.1.0 + +* Use eval rather than monkey-patching fs. +* readdir: Always sort the results +* win32: requeue a file if error has an OK status + +### v2.0 + +* A return to monkey patching +* wrap process.cwd + +### v1.1 + +* wrap readFile +* Wrap fs.writeFile. +* readdir protection +* Don't clobber the fs builtin +* Handle fs.read EAGAIN errors by trying again +* Expose the curOpen counter +* No-op lchown/lchmod if not implemented +* fs.rename patch only for win32 +* Patch fs.rename to handle AV software on Windows +* Close #4 Chown should not fail on einval or eperm if non-root +* Fix isaacs/fstream#1 Only wrap fs one time +* Fix #3 Start at 1024 max files, then back off on EMFILE +* lutimes that doens't blow up on Linux +* A full on-rewrite using a queue instead of just swallowing the EMFILE error +* Wrap Read/Write streams as well + +### 1.0 + +* Update engines for node 0.6 +* Be lstat-graceful on Windows +* first diff --git a/files_find/Files Find/bin/src/node_modules/graceful-fs/clone.js b/files_find/Files Find/bin/src/node_modules/graceful-fs/clone.js new file mode 100644 index 0000000..dff3cc8 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/graceful-fs/clone.js @@ -0,0 +1,23 @@ +'use strict' + +module.exports = clone + +var getPrototypeOf = Object.getPrototypeOf || function (obj) { + return obj.__proto__ +} + +function clone (obj) { + if (obj === null || typeof obj !== 'object') + return obj + + if (obj instanceof Object) + var copy = { __proto__: getPrototypeOf(obj) } + else + var copy = Object.create(null) + + Object.getOwnPropertyNames(obj).forEach(function (key) { + Object.defineProperty(copy, key, Object.getOwnPropertyDescriptor(obj, key)) + }) + + return copy +} diff --git a/files_find/Files Find/bin/src/node_modules/graceful-fs/graceful-fs.js b/files_find/Files Find/bin/src/node_modules/graceful-fs/graceful-fs.js new file mode 100644 index 0000000..8d5b89e --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/graceful-fs/graceful-fs.js @@ -0,0 +1,448 @@ +var fs = require('fs') +var polyfills = require('./polyfills.js') +var legacy = require('./legacy-streams.js') +var clone = require('./clone.js') + +var util = require('util') + +/* istanbul ignore next - node 0.x polyfill */ +var gracefulQueue +var previousSymbol + +/* istanbul ignore else - node 0.x polyfill */ +if (typeof Symbol === 'function' && typeof Symbol.for === 'function') { + gracefulQueue = Symbol.for('graceful-fs.queue') + // This is used in testing by future versions + previousSymbol = Symbol.for('graceful-fs.previous') +} else { + gracefulQueue = '___graceful-fs.queue' + previousSymbol = '___graceful-fs.previous' +} + +function noop () {} + +function publishQueue(context, queue) { + Object.defineProperty(context, gracefulQueue, { + get: function() { + return queue + } + }) +} + +var debug = noop +if (util.debuglog) + debug = util.debuglog('gfs4') +else if (/\bgfs4\b/i.test(process.env.NODE_DEBUG || '')) + debug = function() { + var m = util.format.apply(util, arguments) + m = 'GFS4: ' + m.split(/\n/).join('\nGFS4: ') + console.error(m) + } + +// Once time initialization +if (!fs[gracefulQueue]) { + // This queue can be shared by multiple loaded instances + var queue = global[gracefulQueue] || [] + publishQueue(fs, queue) + + // Patch fs.close/closeSync to shared queue version, because we need + // to retry() whenever a close happens *anywhere* in the program. + // This is essential when multiple graceful-fs instances are + // in play at the same time. + fs.close = (function (fs$close) { + function close (fd, cb) { + return fs$close.call(fs, fd, function (err) { + // This function uses the graceful-fs shared queue + if (!err) { + resetQueue() + } + + if (typeof cb === 'function') + cb.apply(this, arguments) + }) + } + + Object.defineProperty(close, previousSymbol, { + value: fs$close + }) + return close + })(fs.close) + + fs.closeSync = (function (fs$closeSync) { + function closeSync (fd) { + // This function uses the graceful-fs shared queue + fs$closeSync.apply(fs, arguments) + resetQueue() + } + + Object.defineProperty(closeSync, previousSymbol, { + value: fs$closeSync + }) + return closeSync + })(fs.closeSync) + + if (/\bgfs4\b/i.test(process.env.NODE_DEBUG || '')) { + process.on('exit', function() { + debug(fs[gracefulQueue]) + require('assert').equal(fs[gracefulQueue].length, 0) + }) + } +} + +if (!global[gracefulQueue]) { + publishQueue(global, fs[gracefulQueue]); +} + +module.exports = patch(clone(fs)) +if (process.env.TEST_GRACEFUL_FS_GLOBAL_PATCH && !fs.__patched) { + module.exports = patch(fs) + fs.__patched = true; +} + +function patch (fs) { + // Everything that references the open() function needs to be in here + polyfills(fs) + fs.gracefulify = patch + + fs.createReadStream = createReadStream + fs.createWriteStream = createWriteStream + var fs$readFile = fs.readFile + fs.readFile = readFile + function readFile (path, options, cb) { + if (typeof options === 'function') + cb = options, options = null + + return go$readFile(path, options, cb) + + function go$readFile (path, options, cb, startTime) { + return fs$readFile(path, options, function (err) { + if (err && (err.code === 'EMFILE' || err.code === 'ENFILE')) + enqueue([go$readFile, [path, options, cb], err, startTime || Date.now(), Date.now()]) + else { + if (typeof cb === 'function') + cb.apply(this, arguments) + } + }) + } + } + + var fs$writeFile = fs.writeFile + fs.writeFile = writeFile + function writeFile (path, data, options, cb) { + if (typeof options === 'function') + cb = options, options = null + + return go$writeFile(path, data, options, cb) + + function go$writeFile (path, data, options, cb, startTime) { + return fs$writeFile(path, data, options, function (err) { + if (err && (err.code === 'EMFILE' || err.code === 'ENFILE')) + enqueue([go$writeFile, [path, data, options, cb], err, startTime || Date.now(), Date.now()]) + else { + if (typeof cb === 'function') + cb.apply(this, arguments) + } + }) + } + } + + var fs$appendFile = fs.appendFile + if (fs$appendFile) + fs.appendFile = appendFile + function appendFile (path, data, options, cb) { + if (typeof options === 'function') + cb = options, options = null + + return go$appendFile(path, data, options, cb) + + function go$appendFile (path, data, options, cb, startTime) { + return fs$appendFile(path, data, options, function (err) { + if (err && (err.code === 'EMFILE' || err.code === 'ENFILE')) + enqueue([go$appendFile, [path, data, options, cb], err, startTime || Date.now(), Date.now()]) + else { + if (typeof cb === 'function') + cb.apply(this, arguments) + } + }) + } + } + + var fs$copyFile = fs.copyFile + if (fs$copyFile) + fs.copyFile = copyFile + function copyFile (src, dest, flags, cb) { + if (typeof flags === 'function') { + cb = flags + flags = 0 + } + return go$copyFile(src, dest, flags, cb) + + function go$copyFile (src, dest, flags, cb, startTime) { + return fs$copyFile(src, dest, flags, function (err) { + if (err && (err.code === 'EMFILE' || err.code === 'ENFILE')) + enqueue([go$copyFile, [src, dest, flags, cb], err, startTime || Date.now(), Date.now()]) + else { + if (typeof cb === 'function') + cb.apply(this, arguments) + } + }) + } + } + + var fs$readdir = fs.readdir + fs.readdir = readdir + var noReaddirOptionVersions = /^v[0-5]\./ + function readdir (path, options, cb) { + if (typeof options === 'function') + cb = options, options = null + + var go$readdir = noReaddirOptionVersions.test(process.version) + ? function go$readdir (path, options, cb, startTime) { + return fs$readdir(path, fs$readdirCallback( + path, options, cb, startTime + )) + } + : function go$readdir (path, options, cb, startTime) { + return fs$readdir(path, options, fs$readdirCallback( + path, options, cb, startTime + )) + } + + return go$readdir(path, options, cb) + + function fs$readdirCallback (path, options, cb, startTime) { + return function (err, files) { + if (err && (err.code === 'EMFILE' || err.code === 'ENFILE')) + enqueue([ + go$readdir, + [path, options, cb], + err, + startTime || Date.now(), + Date.now() + ]) + else { + if (files && files.sort) + files.sort() + + if (typeof cb === 'function') + cb.call(this, err, files) + } + } + } + } + + if (process.version.substr(0, 4) === 'v0.8') { + var legStreams = legacy(fs) + ReadStream = legStreams.ReadStream + WriteStream = legStreams.WriteStream + } + + var fs$ReadStream = fs.ReadStream + if (fs$ReadStream) { + ReadStream.prototype = Object.create(fs$ReadStream.prototype) + ReadStream.prototype.open = ReadStream$open + } + + var fs$WriteStream = fs.WriteStream + if (fs$WriteStream) { + WriteStream.prototype = Object.create(fs$WriteStream.prototype) + WriteStream.prototype.open = WriteStream$open + } + + Object.defineProperty(fs, 'ReadStream', { + get: function () { + return ReadStream + }, + set: function (val) { + ReadStream = val + }, + enumerable: true, + configurable: true + }) + Object.defineProperty(fs, 'WriteStream', { + get: function () { + return WriteStream + }, + set: function (val) { + WriteStream = val + }, + enumerable: true, + configurable: true + }) + + // legacy names + var FileReadStream = ReadStream + Object.defineProperty(fs, 'FileReadStream', { + get: function () { + return FileReadStream + }, + set: function (val) { + FileReadStream = val + }, + enumerable: true, + configurable: true + }) + var FileWriteStream = WriteStream + Object.defineProperty(fs, 'FileWriteStream', { + get: function () { + return FileWriteStream + }, + set: function (val) { + FileWriteStream = val + }, + enumerable: true, + configurable: true + }) + + function ReadStream (path, options) { + if (this instanceof ReadStream) + return fs$ReadStream.apply(this, arguments), this + else + return ReadStream.apply(Object.create(ReadStream.prototype), arguments) + } + + function ReadStream$open () { + var that = this + open(that.path, that.flags, that.mode, function (err, fd) { + if (err) { + if (that.autoClose) + that.destroy() + + that.emit('error', err) + } else { + that.fd = fd + that.emit('open', fd) + that.read() + } + }) + } + + function WriteStream (path, options) { + if (this instanceof WriteStream) + return fs$WriteStream.apply(this, arguments), this + else + return WriteStream.apply(Object.create(WriteStream.prototype), arguments) + } + + function WriteStream$open () { + var that = this + open(that.path, that.flags, that.mode, function (err, fd) { + if (err) { + that.destroy() + that.emit('error', err) + } else { + that.fd = fd + that.emit('open', fd) + } + }) + } + + function createReadStream (path, options) { + return new fs.ReadStream(path, options) + } + + function createWriteStream (path, options) { + return new fs.WriteStream(path, options) + } + + var fs$open = fs.open + fs.open = open + function open (path, flags, mode, cb) { + if (typeof mode === 'function') + cb = mode, mode = null + + return go$open(path, flags, mode, cb) + + function go$open (path, flags, mode, cb, startTime) { + return fs$open(path, flags, mode, function (err, fd) { + if (err && (err.code === 'EMFILE' || err.code === 'ENFILE')) + enqueue([go$open, [path, flags, mode, cb], err, startTime || Date.now(), Date.now()]) + else { + if (typeof cb === 'function') + cb.apply(this, arguments) + } + }) + } + } + + return fs +} + +function enqueue (elem) { + debug('ENQUEUE', elem[0].name, elem[1]) + fs[gracefulQueue].push(elem) + retry() +} + +// keep track of the timeout between retry() calls +var retryTimer + +// reset the startTime and lastTime to now +// this resets the start of the 60 second overall timeout as well as the +// delay between attempts so that we'll retry these jobs sooner +function resetQueue () { + var now = Date.now() + for (var i = 0; i < fs[gracefulQueue].length; ++i) { + // entries that are only a length of 2 are from an older version, don't + // bother modifying those since they'll be retried anyway. + if (fs[gracefulQueue][i].length > 2) { + fs[gracefulQueue][i][3] = now // startTime + fs[gracefulQueue][i][4] = now // lastTime + } + } + // call retry to make sure we're actively processing the queue + retry() +} + +function retry () { + // clear the timer and remove it to help prevent unintended concurrency + clearTimeout(retryTimer) + retryTimer = undefined + + if (fs[gracefulQueue].length === 0) + return + + var elem = fs[gracefulQueue].shift() + var fn = elem[0] + var args = elem[1] + // these items may be unset if they were added by an older graceful-fs + var err = elem[2] + var startTime = elem[3] + var lastTime = elem[4] + + // if we don't have a startTime we have no way of knowing if we've waited + // long enough, so go ahead and retry this item now + if (startTime === undefined) { + debug('RETRY', fn.name, args) + fn.apply(null, args) + } else if (Date.now() - startTime >= 60000) { + // it's been more than 60 seconds total, bail now + debug('TIMEOUT', fn.name, args) + var cb = args.pop() + if (typeof cb === 'function') + cb.call(null, err) + } else { + // the amount of time between the last attempt and right now + var sinceAttempt = Date.now() - lastTime + // the amount of time between when we first tried, and when we last tried + // rounded up to at least 1 + var sinceStart = Math.max(lastTime - startTime, 1) + // backoff. wait longer than the total time we've been retrying, but only + // up to a maximum of 100ms + var desiredDelay = Math.min(sinceStart * 1.2, 100) + // it's been long enough since the last retry, do it again + if (sinceAttempt >= desiredDelay) { + debug('RETRY', fn.name, args) + fn.apply(null, args.concat([startTime])) + } else { + // if we can't do this job yet, push it to the end of the queue + // and let the next iteration check again + fs[gracefulQueue].push(elem) + } + } + + // schedule our next run if one isn't already scheduled + if (retryTimer === undefined) { + retryTimer = setTimeout(retry, 0) + } +} diff --git a/files_find/Files Find/bin/src/node_modules/graceful-fs/legacy-streams.js b/files_find/Files Find/bin/src/node_modules/graceful-fs/legacy-streams.js new file mode 100644 index 0000000..d617b50 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/graceful-fs/legacy-streams.js @@ -0,0 +1,118 @@ +var Stream = require('stream').Stream + +module.exports = legacy + +function legacy (fs) { + return { + ReadStream: ReadStream, + WriteStream: WriteStream + } + + function ReadStream (path, options) { + if (!(this instanceof ReadStream)) return new ReadStream(path, options); + + Stream.call(this); + + var self = this; + + this.path = path; + this.fd = null; + this.readable = true; + this.paused = false; + + this.flags = 'r'; + this.mode = 438; /*=0666*/ + this.bufferSize = 64 * 1024; + + options = options || {}; + + // Mixin options into this + var keys = Object.keys(options); + for (var index = 0, length = keys.length; index < length; index++) { + var key = keys[index]; + this[key] = options[key]; + } + + if (this.encoding) this.setEncoding(this.encoding); + + if (this.start !== undefined) { + if ('number' !== typeof this.start) { + throw TypeError('start must be a Number'); + } + if (this.end === undefined) { + this.end = Infinity; + } else if ('number' !== typeof this.end) { + throw TypeError('end must be a Number'); + } + + if (this.start > this.end) { + throw new Error('start must be <= end'); + } + + this.pos = this.start; + } + + if (this.fd !== null) { + process.nextTick(function() { + self._read(); + }); + return; + } + + fs.open(this.path, this.flags, this.mode, function (err, fd) { + if (err) { + self.emit('error', err); + self.readable = false; + return; + } + + self.fd = fd; + self.emit('open', fd); + self._read(); + }) + } + + function WriteStream (path, options) { + if (!(this instanceof WriteStream)) return new WriteStream(path, options); + + Stream.call(this); + + this.path = path; + this.fd = null; + this.writable = true; + + this.flags = 'w'; + this.encoding = 'binary'; + this.mode = 438; /*=0666*/ + this.bytesWritten = 0; + + options = options || {}; + + // Mixin options into this + var keys = Object.keys(options); + for (var index = 0, length = keys.length; index < length; index++) { + var key = keys[index]; + this[key] = options[key]; + } + + if (this.start !== undefined) { + if ('number' !== typeof this.start) { + throw TypeError('start must be a Number'); + } + if (this.start < 0) { + throw new Error('start must be >= zero'); + } + + this.pos = this.start; + } + + this.busy = false; + this._queue = []; + + if (this.fd === null) { + this._open = fs.open; + this._queue.push([this._open, this.path, this.flags, this.mode, undefined]); + this.flush(); + } + } +} diff --git a/files_find/Files Find/bin/src/node_modules/graceful-fs/package.json b/files_find/Files Find/bin/src/node_modules/graceful-fs/package.json new file mode 100644 index 0000000..87babf0 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/graceful-fs/package.json @@ -0,0 +1,53 @@ +{ + "name": "graceful-fs", + "description": "A drop-in replacement for fs, making various improvements.", + "version": "4.2.11", + "repository": { + "type": "git", + "url": "https://github.com/isaacs/node-graceful-fs" + }, + "main": "graceful-fs.js", + "directories": { + "test": "test" + }, + "scripts": { + "preversion": "npm test", + "postversion": "npm publish", + "postpublish": "git push origin --follow-tags", + "test": "nyc --silent node test.js | tap -c -", + "posttest": "nyc report" + }, + "keywords": [ + "fs", + "module", + "reading", + "retry", + "retries", + "queue", + "error", + "errors", + "handling", + "EMFILE", + "EAGAIN", + "EINVAL", + "EPERM", + "EACCESS" + ], + "license": "ISC", + "devDependencies": { + "import-fresh": "^2.0.0", + "mkdirp": "^0.5.0", + "rimraf": "^2.2.8", + "tap": "^16.3.4" + }, + "files": [ + "fs.js", + "graceful-fs.js", + "legacy-streams.js", + "polyfills.js", + "clone.js" + ], + "tap": { + "reporter": "classic" + } +} diff --git a/files_find/Files Find/bin/src/node_modules/graceful-fs/polyfills.js b/files_find/Files Find/bin/src/node_modules/graceful-fs/polyfills.js new file mode 100644 index 0000000..453f1a9 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/graceful-fs/polyfills.js @@ -0,0 +1,355 @@ +var constants = require('constants') + +var origCwd = process.cwd +var cwd = null + +var platform = process.env.GRACEFUL_FS_PLATFORM || process.platform + +process.cwd = function() { + if (!cwd) + cwd = origCwd.call(process) + return cwd +} +try { + process.cwd() +} catch (er) {} + +// This check is needed until node.js 12 is required +if (typeof process.chdir === 'function') { + var chdir = process.chdir + process.chdir = function (d) { + cwd = null + chdir.call(process, d) + } + if (Object.setPrototypeOf) Object.setPrototypeOf(process.chdir, chdir) +} + +module.exports = patch + +function patch (fs) { + // (re-)implement some things that are known busted or missing. + + // lchmod, broken prior to 0.6.2 + // back-port the fix here. + if (constants.hasOwnProperty('O_SYMLINK') && + process.version.match(/^v0\.6\.[0-2]|^v0\.5\./)) { + patchLchmod(fs) + } + + // lutimes implementation, or no-op + if (!fs.lutimes) { + patchLutimes(fs) + } + + // https://github.com/isaacs/node-graceful-fs/issues/4 + // Chown should not fail on einval or eperm if non-root. + // It should not fail on enosys ever, as this just indicates + // that a fs doesn't support the intended operation. + + fs.chown = chownFix(fs.chown) + fs.fchown = chownFix(fs.fchown) + fs.lchown = chownFix(fs.lchown) + + fs.chmod = chmodFix(fs.chmod) + fs.fchmod = chmodFix(fs.fchmod) + fs.lchmod = chmodFix(fs.lchmod) + + fs.chownSync = chownFixSync(fs.chownSync) + fs.fchownSync = chownFixSync(fs.fchownSync) + fs.lchownSync = chownFixSync(fs.lchownSync) + + fs.chmodSync = chmodFixSync(fs.chmodSync) + fs.fchmodSync = chmodFixSync(fs.fchmodSync) + fs.lchmodSync = chmodFixSync(fs.lchmodSync) + + fs.stat = statFix(fs.stat) + fs.fstat = statFix(fs.fstat) + fs.lstat = statFix(fs.lstat) + + fs.statSync = statFixSync(fs.statSync) + fs.fstatSync = statFixSync(fs.fstatSync) + fs.lstatSync = statFixSync(fs.lstatSync) + + // if lchmod/lchown do not exist, then make them no-ops + if (fs.chmod && !fs.lchmod) { + fs.lchmod = function (path, mode, cb) { + if (cb) process.nextTick(cb) + } + fs.lchmodSync = function () {} + } + if (fs.chown && !fs.lchown) { + fs.lchown = function (path, uid, gid, cb) { + if (cb) process.nextTick(cb) + } + fs.lchownSync = function () {} + } + + // on Windows, A/V software can lock the directory, causing this + // to fail with an EACCES or EPERM if the directory contains newly + // created files. Try again on failure, for up to 60 seconds. + + // Set the timeout this long because some Windows Anti-Virus, such as Parity + // bit9, may lock files for up to a minute, causing npm package install + // failures. Also, take care to yield the scheduler. Windows scheduling gives + // CPU to a busy looping process, which can cause the program causing the lock + // contention to be starved of CPU by node, so the contention doesn't resolve. + if (platform === "win32") { + fs.rename = typeof fs.rename !== 'function' ? fs.rename + : (function (fs$rename) { + function rename (from, to, cb) { + var start = Date.now() + var backoff = 0; + fs$rename(from, to, function CB (er) { + if (er + && (er.code === "EACCES" || er.code === "EPERM" || er.code === "EBUSY") + && Date.now() - start < 60000) { + setTimeout(function() { + fs.stat(to, function (stater, st) { + if (stater && stater.code === "ENOENT") + fs$rename(from, to, CB); + else + cb(er) + }) + }, backoff) + if (backoff < 100) + backoff += 10; + return; + } + if (cb) cb(er) + }) + } + if (Object.setPrototypeOf) Object.setPrototypeOf(rename, fs$rename) + return rename + })(fs.rename) + } + + // if read() returns EAGAIN, then just try it again. + fs.read = typeof fs.read !== 'function' ? fs.read + : (function (fs$read) { + function read (fd, buffer, offset, length, position, callback_) { + var callback + if (callback_ && typeof callback_ === 'function') { + var eagCounter = 0 + callback = function (er, _, __) { + if (er && er.code === 'EAGAIN' && eagCounter < 10) { + eagCounter ++ + return fs$read.call(fs, fd, buffer, offset, length, position, callback) + } + callback_.apply(this, arguments) + } + } + return fs$read.call(fs, fd, buffer, offset, length, position, callback) + } + + // This ensures `util.promisify` works as it does for native `fs.read`. + if (Object.setPrototypeOf) Object.setPrototypeOf(read, fs$read) + return read + })(fs.read) + + fs.readSync = typeof fs.readSync !== 'function' ? fs.readSync + : (function (fs$readSync) { return function (fd, buffer, offset, length, position) { + var eagCounter = 0 + while (true) { + try { + return fs$readSync.call(fs, fd, buffer, offset, length, position) + } catch (er) { + if (er.code === 'EAGAIN' && eagCounter < 10) { + eagCounter ++ + continue + } + throw er + } + } + }})(fs.readSync) + + function patchLchmod (fs) { + fs.lchmod = function (path, mode, callback) { + fs.open( path + , constants.O_WRONLY | constants.O_SYMLINK + , mode + , function (err, fd) { + if (err) { + if (callback) callback(err) + return + } + // prefer to return the chmod error, if one occurs, + // but still try to close, and report closing errors if they occur. + fs.fchmod(fd, mode, function (err) { + fs.close(fd, function(err2) { + if (callback) callback(err || err2) + }) + }) + }) + } + + fs.lchmodSync = function (path, mode) { + var fd = fs.openSync(path, constants.O_WRONLY | constants.O_SYMLINK, mode) + + // prefer to return the chmod error, if one occurs, + // but still try to close, and report closing errors if they occur. + var threw = true + var ret + try { + ret = fs.fchmodSync(fd, mode) + threw = false + } finally { + if (threw) { + try { + fs.closeSync(fd) + } catch (er) {} + } else { + fs.closeSync(fd) + } + } + return ret + } + } + + function patchLutimes (fs) { + if (constants.hasOwnProperty("O_SYMLINK") && fs.futimes) { + fs.lutimes = function (path, at, mt, cb) { + fs.open(path, constants.O_SYMLINK, function (er, fd) { + if (er) { + if (cb) cb(er) + return + } + fs.futimes(fd, at, mt, function (er) { + fs.close(fd, function (er2) { + if (cb) cb(er || er2) + }) + }) + }) + } + + fs.lutimesSync = function (path, at, mt) { + var fd = fs.openSync(path, constants.O_SYMLINK) + var ret + var threw = true + try { + ret = fs.futimesSync(fd, at, mt) + threw = false + } finally { + if (threw) { + try { + fs.closeSync(fd) + } catch (er) {} + } else { + fs.closeSync(fd) + } + } + return ret + } + + } else if (fs.futimes) { + fs.lutimes = function (_a, _b, _c, cb) { if (cb) process.nextTick(cb) } + fs.lutimesSync = function () {} + } + } + + function chmodFix (orig) { + if (!orig) return orig + return function (target, mode, cb) { + return orig.call(fs, target, mode, function (er) { + if (chownErOk(er)) er = null + if (cb) cb.apply(this, arguments) + }) + } + } + + function chmodFixSync (orig) { + if (!orig) return orig + return function (target, mode) { + try { + return orig.call(fs, target, mode) + } catch (er) { + if (!chownErOk(er)) throw er + } + } + } + + + function chownFix (orig) { + if (!orig) return orig + return function (target, uid, gid, cb) { + return orig.call(fs, target, uid, gid, function (er) { + if (chownErOk(er)) er = null + if (cb) cb.apply(this, arguments) + }) + } + } + + function chownFixSync (orig) { + if (!orig) return orig + return function (target, uid, gid) { + try { + return orig.call(fs, target, uid, gid) + } catch (er) { + if (!chownErOk(er)) throw er + } + } + } + + function statFix (orig) { + if (!orig) return orig + // Older versions of Node erroneously returned signed integers for + // uid + gid. + return function (target, options, cb) { + if (typeof options === 'function') { + cb = options + options = null + } + function callback (er, stats) { + if (stats) { + if (stats.uid < 0) stats.uid += 0x100000000 + if (stats.gid < 0) stats.gid += 0x100000000 + } + if (cb) cb.apply(this, arguments) + } + return options ? orig.call(fs, target, options, callback) + : orig.call(fs, target, callback) + } + } + + function statFixSync (orig) { + if (!orig) return orig + // Older versions of Node erroneously returned signed integers for + // uid + gid. + return function (target, options) { + var stats = options ? orig.call(fs, target, options) + : orig.call(fs, target) + if (stats) { + if (stats.uid < 0) stats.uid += 0x100000000 + if (stats.gid < 0) stats.gid += 0x100000000 + } + return stats; + } + } + + // ENOSYS means that the fs doesn't support the op. Just ignore + // that, because it doesn't matter. + // + // if there's no getuid, or if getuid() is something other + // than 0, and the error is EINVAL or EPERM, then just ignore + // it. + // + // This specific case is a silent failure in cp, install, tar, + // and most other unix tools that manage permissions. + // + // When running as root, or if other types of errors are + // encountered, then it's strict. + function chownErOk (er) { + if (!er) + return true + + if (er.code === "ENOSYS") + return true + + var nonroot = !process.getuid || process.getuid() !== 0 + if (nonroot) { + if (er.code === "EINVAL" || er.code === "EPERM") + return true + } + + return false + } +} diff --git a/files_find/Files Find/bin/src/node_modules/proper-lockfile/CHANGELOG.md b/files_find/Files Find/bin/src/node_modules/proper-lockfile/CHANGELOG.md new file mode 100644 index 0000000..e44d953 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/proper-lockfile/CHANGELOG.md @@ -0,0 +1,108 @@ +# Change Log + +All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. + + +## [4.1.2](https://github.com/moxystudio/node-proper-lockfile/compare/v4.1.1...v4.1.2) (2021-01-25) + + +### Bug Fixes + +* fix node 14 updating graceful-fs ([#102](https://github.com/moxystudio/node-proper-lockfile/issues/102)) ([b0d988e](https://github.com/moxystudio/node-proper-lockfile/commit/b0d988e)) + + + + +## [4.1.1](https://github.com/moxystudio/node-proper-lockfile/compare/v4.1.0...v4.1.1) (2019-04-03) + + +### Bug Fixes + +* fix mtime precision on some filesystems ([#88](https://github.com/moxystudio/node-proper-lockfile/issues/88)) ([f266158](https://github.com/moxystudio/node-proper-lockfile/commit/f266158)), closes [#82](https://github.com/moxystudio/node-proper-lockfile/issues/82) [#87](https://github.com/moxystudio/node-proper-lockfile/issues/87) + + + + +# [4.1.0](https://github.com/moxystudio/node-proper-lockfile/compare/v4.0.0...v4.1.0) (2019-03-18) + + +### Features + +* allow second precision in mtime comparison ([#78](https://github.com/moxystudio/node-proper-lockfile/issues/78)) ([b2816a6](https://github.com/moxystudio/node-proper-lockfile/commit/b2816a6)) + + + + +# [4.0.0](https://github.com/moxystudio/node-proper-lockfile/compare/v3.2.0...v4.0.0) (2019-03-12) + + +### Bug Fixes + +* fix typo in error message ([#68](https://github.com/moxystudio/node-proper-lockfile/issues/68)) ([b91cb55](https://github.com/moxystudio/node-proper-lockfile/commit/b91cb55)) + + +### Features + +* make staleness check more robust ([#74](https://github.com/moxystudio/node-proper-lockfile/issues/74)) ([9cc0973](https://github.com/moxystudio/node-proper-lockfile/commit/9cc0973)), closes [#71](https://github.com/moxystudio/node-proper-lockfile/issues/71) [/github.com/ipfs/js-ipfs-repo/issues/188#issuecomment-468682971](https://github.com//github.com/ipfs/js-ipfs-repo/issues/188/issues/issuecomment-468682971) + + +### BREAKING CHANGES + +* We were marking the lock as compromised when system went into sleep or if the event loop was busy taking too long to run the internals timers, Now we keep track of the mtime updated by the current process, and if we lose some cycles in the update process but recover and the mtime is still ours we do not mark the lock as compromised. + + + + +# [3.2.0](https://github.com/moxystudio/node-proper-lockfile/compare/v3.1.0...v3.2.0) (2018-11-19) + + +### Features + +* add lock path option ([#66](https://github.com/moxystudio/node-proper-lockfile/issues/66)) ([32f1b8d](https://github.com/moxystudio/node-proper-lockfile/commit/32f1b8d)) + + + + +# [3.1.0](https://github.com/moxystudio/node-proper-lockfile/compare/v3.0.2...v3.1.0) (2018-11-15) + + +### Bug Fixes + +* **package:** update retry to version 0.12.0 ([#50](https://github.com/moxystudio/node-proper-lockfile/issues/50)) ([d400b98](https://github.com/moxystudio/node-proper-lockfile/commit/d400b98)) + + +### Features + +* add signal exit ([#65](https://github.com/moxystudio/node-proper-lockfile/issues/65)) ([f20bc45](https://github.com/moxystudio/node-proper-lockfile/commit/f20bc45)) + + + + +## [3.0.2](https://github.com/moxystudio/node-proper-lockfile/compare/v3.0.1...v3.0.2) (2018-01-30) + + + + +## [3.0.1](https://github.com/moxystudio/node-proper-lockfile/compare/v3.0.0...v3.0.1) (2018-01-20) + + +### Bug Fixes + +* restore ability to use lockfile() directly ([0ef8fbc](https://github.com/moxystudio/node-proper-lockfile/commit/0ef8fbc)) + + + + +# [3.0.0](https://github.com/moxystudio/node-proper-lockfile/compare/v2.0.1...v3.0.0) (2018-01-20) + + +### Chores + +* update project to latest node lts ([b1d43e5](https://github.com/moxystudio/node-proper-lockfile/commit/b1d43e5)) + + +### BREAKING CHANGES + +* remove callback support +* use of node lts language features such as object spread +* compromised function in lock() has been moved to an option diff --git a/files_find/Files Find/bin/src/node_modules/proper-lockfile/LICENSE b/files_find/Files Find/bin/src/node_modules/proper-lockfile/LICENSE new file mode 100644 index 0000000..8407b9a --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/proper-lockfile/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2018 Made With MOXY Lda + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/files_find/Files Find/bin/src/node_modules/proper-lockfile/README.md b/files_find/Files Find/bin/src/node_modules/proper-lockfile/README.md new file mode 100644 index 0000000..9355730 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/proper-lockfile/README.md @@ -0,0 +1,183 @@ +# proper-lockfile + +[![NPM version][npm-image]][npm-url] [![Downloads][downloads-image]][npm-url] [![Build Status][travis-image]][travis-url] [![Coverage Status][codecov-image]][codecov-url] [![Dependency status][david-dm-image]][david-dm-url] [![Dev Dependency status][david-dm-dev-image]][david-dm-dev-url] + +[npm-url]:https://npmjs.org/package/proper-lockfile +[downloads-image]:https://img.shields.io/npm/dm/proper-lockfile.svg +[npm-image]:https://img.shields.io/npm/v/proper-lockfile.svg +[travis-url]:https://travis-ci.org/moxystudio/node-proper-lockfile +[travis-image]:https://img.shields.io/travis/moxystudio/node-proper-lockfile/master.svg +[codecov-url]:https://codecov.io/gh/moxystudio/node-proper-lockfile +[codecov-image]:https://img.shields.io/codecov/c/github/moxystudio/node-proper-lockfile/master.svg +[david-dm-url]:https://david-dm.org/moxystudio/node-proper-lockfile +[david-dm-image]:https://img.shields.io/david/moxystudio/node-proper-lockfile.svg +[david-dm-dev-url]:https://david-dm.org/moxystudio/node-proper-lockfile?type=dev +[david-dm-dev-image]:https://img.shields.io/david/dev/moxystudio/node-proper-lockfile.svg + +An inter-process and inter-machine lockfile utility that works on a local or network file system. + + +## Installation + +`$ npm install proper-lockfile` + + +## Design + +There are various ways to achieve [file locking](http://en.wikipedia.org/wiki/File_locking). + +This library utilizes the `mkdir` strategy which works atomically on any kind of file system, even network based ones. +The lockfile path is based on the file path you are trying to lock by suffixing it with `.lock`. + +When a lock is successfully acquired, the lockfile's `mtime` (modified time) is periodically updated to prevent staleness. This allows to effectively check if a lock is stale by checking its `mtime` against a stale threshold. If the update of the mtime fails several times, the lock might be compromised. The `mtime` is [supported](http://en.wikipedia.org/wiki/Comparison_of_file_systems) in almost every `filesystem`. + + +### Comparison + +This library is similar to [lockfile](https://github.com/isaacs/lockfile) but the latter has some drawbacks: + +- It relies on `open` with `O_EXCL` flag which has problems in network file systems. `proper-lockfile` uses `mkdir` which doesn't have this issue. + +> O_EXCL is broken on NFS file systems; programs which rely on it for performing locking tasks will contain a race condition. + +- The lockfile staleness check is done via `ctime` (creation time) which is unsuitable for long running processes. `proper-lockfile` constantly updates lockfiles `mtime` to do proper staleness check. + +- It does not check if the lockfile was compromised which can lead to undesirable situations. `proper-lockfile` checks the lockfile when updating the `mtime`. + +- It has a default value of `0` for the stale option which isn't good because any crash or process kill that the package can't handle gracefully will leave the lock active forever. + + +### Compromised + +`proper-lockfile` does not detect cases in which: + +- A `lockfile` is manually removed and someone else acquires the lock right after +- Different `stale`/`update` values are being used for the same file, possibly causing two locks to be acquired on the same file + +`proper-lockfile` detects cases in which: + +- Updates to the `lockfile` fail +- Updates take longer than expected, possibly causing the lock to become stale for a certain amount of time + + +As you see, the first two are a consequence of bad usage. Technically, it was possible to detect the first two but it would introduce complexity and eventual race conditions. + + +## Usage + +### .lock(file, [options]) + +Tries to acquire a lock on `file` or rejects the promise on error. + +If the lock succeeds, a `release` function is provided that should be called when you want to release the lock. The `release` function also rejects the promise on error (e.g. when the lock was already compromised). + +Available options: + +- `stale`: Duration in milliseconds in which the lock is considered stale, defaults to `10000` (minimum value is `5000`) +- `update`: The interval in milliseconds in which the lockfile's `mtime` will be updated, defaults to `stale/2` (minimum value is `1000`, maximum value is `stale/2`) +- `retries`: The number of retries or a [retry](https://www.npmjs.org/package/retry) options object, defaults to `0` +- `realpath`: Resolve symlinks using realpath, defaults to `true` (note that if `true`, the `file` must exist previously) +- `fs`: A custom fs to use, defaults to `graceful-fs` +- `onCompromised`: Called if the lock gets compromised, defaults to a function that simply throws the error which will probably cause the process to die +- `lockfilePath`: Custom lockfile path. e.g.: If you want to lock a directory and create the lock file inside it, you can pass `file` as `` and `options.lockfilePath` as `/dir.lock` + + +```js +const lockfile = require('proper-lockfile'); + +lockfile.lock('some/file') +.then((release) => { + // Do something while the file is locked + + // Call the provided release function when you're done, + // which will also return a promise + return release(); +}) +.catch((e) => { + // either lock could not be acquired + // or releasing it failed + console.error(e) +}); + +// Alternatively, you may use lockfile('some/file') directly. +``` + + +### .unlock(file, [options]) + +Releases a previously acquired lock on `file` or rejects the promise on error. + +Whenever possible you should use the `release` function instead (as exemplified above). Still there are cases in which it's hard to keep a reference to it around code. In those cases `unlock()` might be handy. + +Available options: + +- `realpath`: Resolve symlinks using realpath, defaults to `true` (note that if `true`, the `file` must exist previously) +- `fs`: A custom fs to use, defaults to `graceful-fs` +- `lockfilePath`: Custom lockfile path. e.g.: If you want to lock a directory and create the lock file inside it, you can pass `file` as `` and `options.lockfilePath` as `/dir.lock` + + +```js +const lockfile = require('proper-lockfile'); + +lockfile.lock('some/file') +.then(() => { + // Do something while the file is locked + + // Later.. + return lockfile.unlock('some/file'); +}); +``` + +### .check(file, [options]) + +Check if the file is locked and its lockfile is not stale, rejects the promise on error. + +Available options: + +- `stale`: Duration in milliseconds in which the lock is considered stale, defaults to `10000` (minimum value is `5000`) +- `realpath`: Resolve symlinks using realpath, defaults to `true` (note that if `true`, the `file` must exist previously) +- `fs`: A custom fs to use, defaults to `graceful-fs` +- `lockfilePath`: Custom lockfile path. e.g.: If you want to lock a directory and create the lock file inside it, you can pass `file` as `` and `options.lockfilePath` as `/dir.lock` + + +```js +const lockfile = require('proper-lockfile'); + +lockfile.check('some/file') +.then((isLocked) => { + // isLocked will be true if 'some/file' is locked, false otherwise +}); +``` + +### .lockSync(file, [options]) + +Sync version of `.lock()`. +Returns the `release` function or throws on error. + +### .unlockSync(file, [options]) + +Sync version of `.unlock()`. +Throws on error. + +### .checkSync(file, [options]) + +Sync version of `.check()`. +Returns a boolean or throws on error. + + +## Graceful exit + +`proper-lockfile` automatically removes locks if the process exits, except if the process is killed with SIGKILL or it crashes due to a VM fatal error (e.g.: out of memory). + + +## Tests + +`$ npm test` +`$ npm test -- --watch` during development + +The test suite is very extensive. There's even a stress test to guarantee exclusiveness of locks. + + +## License + +Released under the [MIT License](https://www.opensource.org/licenses/mit-license.php). diff --git a/files_find/Files Find/bin/src/node_modules/proper-lockfile/index.js b/files_find/Files Find/bin/src/node_modules/proper-lockfile/index.js new file mode 100644 index 0000000..0c3ec0a --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/proper-lockfile/index.js @@ -0,0 +1,40 @@ +'use strict'; + +const lockfile = require('./lib/lockfile'); +const { toPromise, toSync, toSyncOptions } = require('./lib/adapter'); + +async function lock(file, options) { + const release = await toPromise(lockfile.lock)(file, options); + + return toPromise(release); +} + +function lockSync(file, options) { + const release = toSync(lockfile.lock)(file, toSyncOptions(options)); + + return toSync(release); +} + +function unlock(file, options) { + return toPromise(lockfile.unlock)(file, options); +} + +function unlockSync(file, options) { + return toSync(lockfile.unlock)(file, toSyncOptions(options)); +} + +function check(file, options) { + return toPromise(lockfile.check)(file, options); +} + +function checkSync(file, options) { + return toSync(lockfile.check)(file, toSyncOptions(options)); +} + +module.exports = lock; +module.exports.lock = lock; +module.exports.unlock = unlock; +module.exports.lockSync = lockSync; +module.exports.unlockSync = unlockSync; +module.exports.check = check; +module.exports.checkSync = checkSync; diff --git a/files_find/Files Find/bin/src/node_modules/proper-lockfile/lib/adapter.js b/files_find/Files Find/bin/src/node_modules/proper-lockfile/lib/adapter.js new file mode 100644 index 0000000..8578967 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/proper-lockfile/lib/adapter.js @@ -0,0 +1,85 @@ +'use strict'; + +const fs = require('graceful-fs'); + +function createSyncFs(fs) { + const methods = ['mkdir', 'realpath', 'stat', 'rmdir', 'utimes']; + const newFs = { ...fs }; + + methods.forEach((method) => { + newFs[method] = (...args) => { + const callback = args.pop(); + let ret; + + try { + ret = fs[`${method}Sync`](...args); + } catch (err) { + return callback(err); + } + + callback(null, ret); + }; + }); + + return newFs; +} + +// ---------------------------------------------------------- + +function toPromise(method) { + return (...args) => new Promise((resolve, reject) => { + args.push((err, result) => { + if (err) { + reject(err); + } else { + resolve(result); + } + }); + + method(...args); + }); +} + +function toSync(method) { + return (...args) => { + let err; + let result; + + args.push((_err, _result) => { + err = _err; + result = _result; + }); + + method(...args); + + if (err) { + throw err; + } + + return result; + }; +} + +function toSyncOptions(options) { + // Shallow clone options because we are oging to mutate them + options = { ...options }; + + // Transform fs to use the sync methods instead + options.fs = createSyncFs(options.fs || fs); + + // Retries are not allowed because it requires the flow to be sync + if ( + (typeof options.retries === 'number' && options.retries > 0) || + (options.retries && typeof options.retries.retries === 'number' && options.retries.retries > 0) + ) { + throw Object.assign(new Error('Cannot use retries with the sync api'), { code: 'ESYNC' }); + } + + return options; +} + +module.exports = { + toPromise, + toSync, + toSyncOptions, +}; diff --git a/files_find/Files Find/bin/src/node_modules/proper-lockfile/lib/lockfile.js b/files_find/Files Find/bin/src/node_modules/proper-lockfile/lib/lockfile.js new file mode 100644 index 0000000..97b6637 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/proper-lockfile/lib/lockfile.js @@ -0,0 +1,342 @@ +'use strict'; + +const path = require('path'); +const fs = require('graceful-fs'); +const retry = require('retry'); +const onExit = require('signal-exit'); +const mtimePrecision = require('./mtime-precision'); + +const locks = {}; + +function getLockFile(file, options) { + return options.lockfilePath || `${file}.lock`; +} + +function resolveCanonicalPath(file, options, callback) { + if (!options.realpath) { + return callback(null, path.resolve(file)); + } + + // Use realpath to resolve symlinks + // It also resolves relative paths + options.fs.realpath(file, callback); +} + +function acquireLock(file, options, callback) { + const lockfilePath = getLockFile(file, options); + + // Use mkdir to create the lockfile (atomic operation) + options.fs.mkdir(lockfilePath, (err) => { + if (!err) { + // At this point, we acquired the lock! + // Probe the mtime precision + return mtimePrecision.probe(lockfilePath, options.fs, (err, mtime, mtimePrecision) => { + // If it failed, try to remove the lock.. + /* istanbul ignore if */ + if (err) { + options.fs.rmdir(lockfilePath, () => {}); + + return callback(err); + } + + callback(null, mtime, mtimePrecision); + }); + } + + // If error is not EEXIST then some other error occurred while locking + if (err.code !== 'EEXIST') { + return callback(err); + } + + // Otherwise, check if lock is stale by analyzing the file mtime + if (options.stale <= 0) { + return callback(Object.assign(new Error('Lock file is already being held'), { code: 'ELOCKED', file })); + } + + options.fs.stat(lockfilePath, (err, stat) => { + if (err) { + // Retry if the lockfile has been removed (meanwhile) + // Skip stale check to avoid recursiveness + if (err.code === 'ENOENT') { + return acquireLock(file, { ...options, stale: 0 }, callback); + } + + return callback(err); + } + + if (!isLockStale(stat, options)) { + return callback(Object.assign(new Error('Lock file is already being held'), { code: 'ELOCKED', file })); + } + + // If it's stale, remove it and try again! + // Skip stale check to avoid recursiveness + removeLock(file, options, (err) => { + if (err) { + return callback(err); + } + + acquireLock(file, { ...options, stale: 0 }, callback); + }); + }); + }); +} + +function isLockStale(stat, options) { + return stat.mtime.getTime() < Date.now() - options.stale; +} + +function removeLock(file, options, callback) { + // Remove lockfile, ignoring ENOENT errors + options.fs.rmdir(getLockFile(file, options), (err) => { + if (err && err.code !== 'ENOENT') { + return callback(err); + } + + callback(); + }); +} + +function updateLock(file, options) { + const lock = locks[file]; + + // Just for safety, should never happen + /* istanbul ignore if */ + if (lock.updateTimeout) { + return; + } + + lock.updateDelay = lock.updateDelay || options.update; + lock.updateTimeout = setTimeout(() => { + lock.updateTimeout = null; + + // Stat the file to check if mtime is still ours + // If it is, we can still recover from a system sleep or a busy event loop + options.fs.stat(lock.lockfilePath, (err, stat) => { + const isOverThreshold = lock.lastUpdate + options.stale < Date.now(); + + // If it failed to update the lockfile, keep trying unless + // the lockfile was deleted or we are over the threshold + if (err) { + if (err.code === 'ENOENT' || isOverThreshold) { + return setLockAsCompromised(file, lock, Object.assign(err, { code: 'ECOMPROMISED' })); + } + + lock.updateDelay = 1000; + + return updateLock(file, options); + } + + const isMtimeOurs = lock.mtime.getTime() === stat.mtime.getTime(); + + if (!isMtimeOurs) { + return setLockAsCompromised( + file, + lock, + Object.assign( + new Error('Unable to update lock within the stale threshold'), + { code: 'ECOMPROMISED' } + )); + } + + const mtime = mtimePrecision.getMtime(lock.mtimePrecision); + + options.fs.utimes(lock.lockfilePath, mtime, mtime, (err) => { + const isOverThreshold = lock.lastUpdate + options.stale < Date.now(); + + // Ignore if the lock was released + if (lock.released) { + return; + } + + // If it failed to update the lockfile, keep trying unless + // the lockfile was deleted or we are over the threshold + if (err) { + if (err.code === 'ENOENT' || isOverThreshold) { + return setLockAsCompromised(file, lock, Object.assign(err, { code: 'ECOMPROMISED' })); + } + + lock.updateDelay = 1000; + + return updateLock(file, options); + } + + // All ok, keep updating.. + lock.mtime = mtime; + lock.lastUpdate = Date.now(); + lock.updateDelay = null; + updateLock(file, options); + }); + }); + }, lock.updateDelay); + + // Unref the timer so that the nodejs process can exit freely + // This is safe because all acquired locks will be automatically released + // on process exit + + // We first check that `lock.updateTimeout.unref` exists because some users + // may be using this module outside of NodeJS (e.g., in an electron app), + // and in those cases `setTimeout` return an integer. + /* istanbul ignore else */ + if (lock.updateTimeout.unref) { + lock.updateTimeout.unref(); + } +} + +function setLockAsCompromised(file, lock, err) { + // Signal the lock has been released + lock.released = true; + + // Cancel lock mtime update + // Just for safety, at this point updateTimeout should be null + /* istanbul ignore if */ + if (lock.updateTimeout) { + clearTimeout(lock.updateTimeout); + } + + if (locks[file] === lock) { + delete locks[file]; + } + + lock.options.onCompromised(err); +} + +// ---------------------------------------------------------- + +function lock(file, options, callback) { + /* istanbul ignore next */ + options = { + stale: 10000, + update: null, + realpath: true, + retries: 0, + fs, + onCompromised: (err) => { throw err; }, + ...options, + }; + + options.retries = options.retries || 0; + options.retries = typeof options.retries === 'number' ? { retries: options.retries } : options.retries; + options.stale = Math.max(options.stale || 0, 2000); + options.update = options.update == null ? options.stale / 2 : options.update || 0; + options.update = Math.max(Math.min(options.update, options.stale / 2), 1000); + + // Resolve to a canonical file path + resolveCanonicalPath(file, options, (err, file) => { + if (err) { + return callback(err); + } + + // Attempt to acquire the lock + const operation = retry.operation(options.retries); + + operation.attempt(() => { + acquireLock(file, options, (err, mtime, mtimePrecision) => { + if (operation.retry(err)) { + return; + } + + if (err) { + return callback(operation.mainError()); + } + + // We now own the lock + const lock = locks[file] = { + lockfilePath: getLockFile(file, options), + mtime, + mtimePrecision, + options, + lastUpdate: Date.now(), + }; + + // We must keep the lock fresh to avoid staleness + updateLock(file, options); + + callback(null, (releasedCallback) => { + if (lock.released) { + return releasedCallback && + releasedCallback(Object.assign(new Error('Lock is already released'), { code: 'ERELEASED' })); + } + + // Not necessary to use realpath twice when unlocking + unlock(file, { ...options, realpath: false }, releasedCallback); + }); + }); + }); + }); +} + +function unlock(file, options, callback) { + options = { + fs, + realpath: true, + ...options, + }; + + // Resolve to a canonical file path + resolveCanonicalPath(file, options, (err, file) => { + if (err) { + return callback(err); + } + + // Skip if the lock is not acquired + const lock = locks[file]; + + if (!lock) { + return callback(Object.assign(new Error('Lock is not acquired/owned by you'), { code: 'ENOTACQUIRED' })); + } + + lock.updateTimeout && clearTimeout(lock.updateTimeout); // Cancel lock mtime update + lock.released = true; // Signal the lock has been released + delete locks[file]; // Delete from locks + + removeLock(file, options, callback); + }); +} + +function check(file, options, callback) { + options = { + stale: 10000, + realpath: true, + fs, + ...options, + }; + + options.stale = Math.max(options.stale || 0, 2000); + + // Resolve to a canonical file path + resolveCanonicalPath(file, options, (err, file) => { + if (err) { + return callback(err); + } + + // Check if lockfile exists + options.fs.stat(getLockFile(file, options), (err, stat) => { + if (err) { + // If does not exist, file is not locked. Otherwise, callback with error + return err.code === 'ENOENT' ? callback(null, false) : callback(err); + } + + // Otherwise, check if lock is stale by analyzing the file mtime + return callback(null, !isLockStale(stat, options)); + }); + }); +} + +function getLocks() { + return locks; +} + +// Remove acquired locks on exit +/* istanbul ignore next */ +onExit(() => { + for (const file in locks) { + const options = locks[file].options; + + try { options.fs.rmdirSync(getLockFile(file, options)); } catch (e) { /* Empty */ } + } +}); + +module.exports.lock = lock; +module.exports.unlock = unlock; +module.exports.check = check; +module.exports.getLocks = getLocks; diff --git a/files_find/Files Find/bin/src/node_modules/proper-lockfile/lib/mtime-precision.js b/files_find/Files Find/bin/src/node_modules/proper-lockfile/lib/mtime-precision.js new file mode 100644 index 0000000..b82a3ce --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/proper-lockfile/lib/mtime-precision.js @@ -0,0 +1,55 @@ +'use strict'; + +const cacheSymbol = Symbol(); + +function probe(file, fs, callback) { + const cachedPrecision = fs[cacheSymbol]; + + if (cachedPrecision) { + return fs.stat(file, (err, stat) => { + /* istanbul ignore if */ + if (err) { + return callback(err); + } + + callback(null, stat.mtime, cachedPrecision); + }); + } + + // Set mtime by ceiling Date.now() to seconds + 5ms so that it's "not on the second" + const mtime = new Date((Math.ceil(Date.now() / 1000) * 1000) + 5); + + fs.utimes(file, mtime, mtime, (err) => { + /* istanbul ignore if */ + if (err) { + return callback(err); + } + + fs.stat(file, (err, stat) => { + /* istanbul ignore if */ + if (err) { + return callback(err); + } + + const precision = stat.mtime.getTime() % 1000 === 0 ? 's' : 'ms'; + + // Cache the precision in a non-enumerable way + Object.defineProperty(fs, cacheSymbol, { value: precision }); + + callback(null, stat.mtime, precision); + }); + }); +} + +function getMtime(precision) { + let now = Date.now(); + + if (precision === 's') { + now = Math.ceil(now / 1000) * 1000; + } + + return new Date(now); +} + +module.exports.probe = probe; +module.exports.getMtime = getMtime; diff --git a/files_find/Files Find/bin/src/node_modules/proper-lockfile/package.json b/files_find/Files Find/bin/src/node_modules/proper-lockfile/package.json new file mode 100644 index 0000000..b74750c --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/proper-lockfile/package.json @@ -0,0 +1,71 @@ +{ + "name": "proper-lockfile", + "version": "4.1.2", + "description": "A inter-process and inter-machine lockfile utility that works on a local or network file system", + "keywords": [ + "lock", + "locking", + "file", + "lockfile", + "fs", + "cross-process" + ], + "author": "André Cruz ", + "homepage": "https://github.com/moxystudio/node-proper-lockfile", + "repository": { + "type": "git", + "url": "git@github.com:moxystudio/node-proper-lockfile.git" + }, + "license": "MIT", + "main": "index.js", + "files": [ + "lib" + ], + "scripts": { + "lint": "eslint .", + "test": "jest --env node --coverage --runInBand", + "prerelease": "npm t && npm run lint", + "release": "standard-version", + "postrelease": "git push --follow-tags origin HEAD && npm publish" + }, + "husky": { + "hooks": { + "commit-msg": "commitlint -E HUSKY_GIT_PARAMS", + "pre-commit": "lint-staged" + } + }, + "lint-staged": { + "*.js": [ + "eslint --fix", + "git add" + ] + }, + "commitlint": { + "extends": [ + "@commitlint/config-conventional" + ] + }, + "dependencies": { + "graceful-fs": "^4.2.4", + "retry": "^0.12.0", + "signal-exit": "^3.0.2" + }, + "devDependencies": { + "@commitlint/cli": "^7.0.0", + "@commitlint/config-conventional": "^7.0.1", + "@segment/clear-timeouts": "^2.0.0", + "delay": "^4.1.0", + "eslint": "^5.3.0", + "eslint-config-moxy": "^7.1.0", + "execa": "^1.0.0", + "husky": "^1.1.4", + "jest": "^24.5.0", + "lint-staged": "^8.0.4", + "mkdirp": "^0.5.1", + "p-defer": "^2.1.0", + "rimraf": "^2.6.2", + "stable": "^0.1.8", + "standard-version": "^5.0.0", + "thread-sleep": "^2.1.0" + } +} diff --git a/files_find/Files Find/bin/src/node_modules/retry/.npmignore b/files_find/Files Find/bin/src/node_modules/retry/.npmignore new file mode 100644 index 0000000..432f285 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/.npmignore @@ -0,0 +1,3 @@ +/node_modules/* +npm-debug.log +coverage diff --git a/files_find/Files Find/bin/src/node_modules/retry/.travis.yml b/files_find/Files Find/bin/src/node_modules/retry/.travis.yml new file mode 100644 index 0000000..bcde212 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/.travis.yml @@ -0,0 +1,15 @@ +language: node_js +node_js: + - "4" +before_install: + - pip install --user codecov +after_success: + - codecov --file coverage/lcov.info --disable search +# travis encrypt [subdomain]:[api token]@[room id] +# notifications: +# email: false +# campfire: +# rooms: +# secure: xyz +# on_failure: always +# on_success: always diff --git a/files_find/Files Find/bin/src/node_modules/retry/License b/files_find/Files Find/bin/src/node_modules/retry/License new file mode 100644 index 0000000..0b58de3 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/License @@ -0,0 +1,21 @@ +Copyright (c) 2011: +Tim Koschützki (tim@debuggable.com) +Felix Geisendörfer (felix@debuggable.com) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/files_find/Files Find/bin/src/node_modules/retry/Makefile b/files_find/Files Find/bin/src/node_modules/retry/Makefile new file mode 100644 index 0000000..1968d8f --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/Makefile @@ -0,0 +1,18 @@ +SHELL := /bin/bash + +release-major: test + npm version major -m "Release %s" + git push + npm publish + +release-minor: test + npm version minor -m "Release %s" + git push + npm publish + +release-patch: test + npm version patch -m "Release %s" + git push + npm publish + +.PHONY: test release-major release-minor release-patch diff --git a/files_find/Files Find/bin/src/node_modules/retry/README.md b/files_find/Files Find/bin/src/node_modules/retry/README.md new file mode 100644 index 0000000..16e28ec --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/README.md @@ -0,0 +1,227 @@ + +[![Build Status](https://secure.travis-ci.org/tim-kos/node-retry.png?branch=master)](http://travis-ci.org/tim-kos/node-retry "Check this project's build status on TravisCI") +[![codecov](https://codecov.io/gh/tim-kos/node-retry/branch/master/graph/badge.svg)](https://codecov.io/gh/tim-kos/node-retry) + + +# retry + +Abstraction for exponential and custom retry strategies for failed operations. + +## Installation + + npm install retry + +## Current Status + +This module has been tested and is ready to be used. + +## Tutorial + +The example below will retry a potentially failing `dns.resolve` operation +`10` times using an exponential backoff strategy. With the default settings, this +means the last attempt is made after `17 minutes and 3 seconds`. + +``` javascript +var dns = require('dns'); +var retry = require('retry'); + +function faultTolerantResolve(address, cb) { + var operation = retry.operation(); + + operation.attempt(function(currentAttempt) { + dns.resolve(address, function(err, addresses) { + if (operation.retry(err)) { + return; + } + + cb(err ? operation.mainError() : null, addresses); + }); + }); +} + +faultTolerantResolve('nodejs.org', function(err, addresses) { + console.log(err, addresses); +}); +``` + +Of course you can also configure the factors that go into the exponential +backoff. See the API documentation below for all available settings. +currentAttempt is an int representing the number of attempts so far. + +``` javascript +var operation = retry.operation({ + retries: 5, + factor: 3, + minTimeout: 1 * 1000, + maxTimeout: 60 * 1000, + randomize: true, +}); +``` + +## API + +### retry.operation([options]) + +Creates a new `RetryOperation` object. `options` is the same as `retry.timeouts()`'s `options`, with two additions: + +* `forever`: Whether to retry forever, defaults to `false`. +* `unref`: Whether to [unref](https://nodejs.org/api/timers.html#timers_unref) the setTimeout's, defaults to `false`. +* `maxRetryTime`: The maximum time (in milliseconds) that the retried operation is allowed to run. Default is `Infinity`. + +### retry.timeouts([options]) + +Returns an array of timeouts. All time `options` and return values are in +milliseconds. If `options` is an array, a copy of that array is returned. + +`options` is a JS object that can contain any of the following keys: + +* `retries`: The maximum amount of times to retry the operation. Default is `10`. Seting this to `1` means `do it once, then retry it once`. +* `factor`: The exponential factor to use. Default is `2`. +* `minTimeout`: The number of milliseconds before starting the first retry. Default is `1000`. +* `maxTimeout`: The maximum number of milliseconds between two retries. Default is `Infinity`. +* `randomize`: Randomizes the timeouts by multiplying with a factor between `1` to `2`. Default is `false`. + +The formula used to calculate the individual timeouts is: + +``` +Math.min(random * minTimeout * Math.pow(factor, attempt), maxTimeout) +``` + +Have a look at [this article][article] for a better explanation of approach. + +If you want to tune your `factor` / `times` settings to attempt the last retry +after a certain amount of time, you can use wolfram alpha. For example in order +to tune for `10` attempts in `5 minutes`, you can use this equation: + +![screenshot](https://github.com/tim-kos/node-retry/raw/master/equation.gif) + +Explaining the various values from left to right: + +* `k = 0 ... 9`: The `retries` value (10) +* `1000`: The `minTimeout` value in ms (1000) +* `x^k`: No need to change this, `x` will be your resulting factor +* `5 * 60 * 1000`: The desired total amount of time for retrying in ms (5 minutes) + +To make this a little easier for you, use wolfram alpha to do the calculations: + + + +[article]: http://dthain.blogspot.com/2009/02/exponential-backoff-in-distributed.html + +### retry.createTimeout(attempt, opts) + +Returns a new `timeout` (integer in milliseconds) based on the given parameters. + +`attempt` is an integer representing for which retry the timeout should be calculated. If your retry operation was executed 4 times you had one attempt and 3 retries. If you then want to calculate a new timeout, you should set `attempt` to 4 (attempts are zero-indexed). + +`opts` can include `factor`, `minTimeout`, `randomize` (boolean) and `maxTimeout`. They are documented above. + +`retry.createTimeout()` is used internally by `retry.timeouts()` and is public for you to be able to create your own timeouts for reinserting an item, see [issue #13](https://github.com/tim-kos/node-retry/issues/13). + +### retry.wrap(obj, [options], [methodNames]) + +Wrap all functions of the `obj` with retry. Optionally you can pass operation options and +an array of method names which need to be wrapped. + +``` +retry.wrap(obj) + +retry.wrap(obj, ['method1', 'method2']) + +retry.wrap(obj, {retries: 3}) + +retry.wrap(obj, {retries: 3}, ['method1', 'method2']) +``` +The `options` object can take any options that the usual call to `retry.operation` can take. + +### new RetryOperation(timeouts, [options]) + +Creates a new `RetryOperation` where `timeouts` is an array where each value is +a timeout given in milliseconds. + +Available options: +* `forever`: Whether to retry forever, defaults to `false`. +* `unref`: Wether to [unref](https://nodejs.org/api/timers.html#timers_unref) the setTimeout's, defaults to `false`. + +If `forever` is true, the following changes happen: +* `RetryOperation.errors()` will only output an array of one item: the last error. +* `RetryOperation` will repeatedly use the `timeouts` array. Once all of its timeouts have been used up, it restarts with the first timeout, then uses the second and so on. + +#### retryOperation.errors() + +Returns an array of all errors that have been passed to `retryOperation.retry()` so far. The +returning array has the errors ordered chronologically based on when they were passed to +`retryOperation.retry()`, which means the first passed error is at index zero and the last is +at the last index. + +#### retryOperation.mainError() + +A reference to the error object that occured most frequently. Errors are +compared using the `error.message` property. + +If multiple error messages occured the same amount of time, the last error +object with that message is returned. + +If no errors occured so far, the value is `null`. + +#### retryOperation.attempt(fn, timeoutOps) + +Defines the function `fn` that is to be retried and executes it for the first +time right away. The `fn` function can receive an optional `currentAttempt` callback that represents the number of attempts to execute `fn` so far. + +Optionally defines `timeoutOps` which is an object having a property `timeout` in miliseconds and a property `cb` callback function. +Whenever your retry operation takes longer than `timeout` to execute, the timeout callback function `cb` is called. + + +#### retryOperation.try(fn) + +This is an alias for `retryOperation.attempt(fn)`. This is deprecated. Please use `retryOperation.attempt(fn)` instead. + +#### retryOperation.start(fn) + +This is an alias for `retryOperation.attempt(fn)`. This is deprecated. Please use `retryOperation.attempt(fn)` instead. + +#### retryOperation.retry(error) + +Returns `false` when no `error` value is given, or the maximum amount of retries +has been reached. + +Otherwise it returns `true`, and retries the operation after the timeout for +the current attempt number. + +#### retryOperation.stop() + +Allows you to stop the operation being retried. Useful for aborting the operation on a fatal error etc. + +#### retryOperation.reset() + +Resets the internal state of the operation object, so that you can call `attempt()` again as if this was a new operation object. + +#### retryOperation.attempts() + +Returns an int representing the number of attempts it took to call `fn` before it was successful. + +## License + +retry is licensed under the MIT license. + + +# Changelog + +0.10.0 Adding `stop` functionality, thanks to @maxnachlinger. + +0.9.0 Adding `unref` functionality, thanks to @satazor. + +0.8.0 Implementing retry.wrap. + +0.7.0 Some bug fixes and made retry.createTimeout() public. Fixed issues [#10](https://github.com/tim-kos/node-retry/issues/10), [#12](https://github.com/tim-kos/node-retry/issues/12), and [#13](https://github.com/tim-kos/node-retry/issues/13). + +0.6.0 Introduced optional timeOps parameter for the attempt() function which is an object having a property timeout in milliseconds and a property cb callback function. Whenever your retry operation takes longer than timeout to execute, the timeout callback function cb is called. + +0.5.0 Some minor refactoring. + +0.4.0 Changed retryOperation.try() to retryOperation.attempt(). Deprecated the aliases start() and try() for it. + +0.3.0 Added retryOperation.start() which is an alias for retryOperation.try(). + +0.2.0 Added attempts() function and parameter to retryOperation.try() representing the number of attempts it took to call fn(). diff --git a/files_find/Files Find/bin/src/node_modules/retry/equation.gif b/files_find/Files Find/bin/src/node_modules/retry/equation.gif new file mode 100644 index 0000000..9710723 Binary files /dev/null and b/files_find/Files Find/bin/src/node_modules/retry/equation.gif differ diff --git a/files_find/Files Find/bin/src/node_modules/retry/example/dns.js b/files_find/Files Find/bin/src/node_modules/retry/example/dns.js new file mode 100644 index 0000000..446729b --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/example/dns.js @@ -0,0 +1,31 @@ +var dns = require('dns'); +var retry = require('../lib/retry'); + +function faultTolerantResolve(address, cb) { + var opts = { + retries: 2, + factor: 2, + minTimeout: 1 * 1000, + maxTimeout: 2 * 1000, + randomize: true + }; + var operation = retry.operation(opts); + + operation.attempt(function(currentAttempt) { + dns.resolve(address, function(err, addresses) { + if (operation.retry(err)) { + return; + } + + cb(operation.mainError(), operation.errors(), addresses); + }); + }); +} + +faultTolerantResolve('nodejs.org', function(err, errors, addresses) { + console.warn('err:'); + console.log(err); + + console.warn('addresses:'); + console.log(addresses); +}); \ No newline at end of file diff --git a/files_find/Files Find/bin/src/node_modules/retry/example/stop.js b/files_find/Files Find/bin/src/node_modules/retry/example/stop.js new file mode 100644 index 0000000..e1ceafe --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/example/stop.js @@ -0,0 +1,40 @@ +var retry = require('../lib/retry'); + +function attemptAsyncOperation(someInput, cb) { + var opts = { + retries: 2, + factor: 2, + minTimeout: 1 * 1000, + maxTimeout: 2 * 1000, + randomize: true + }; + var operation = retry.operation(opts); + + operation.attempt(function(currentAttempt) { + failingAsyncOperation(someInput, function(err, result) { + + if (err && err.message === 'A fatal error') { + operation.stop(); + return cb(err); + } + + if (operation.retry(err)) { + return; + } + + cb(operation.mainError(), operation.errors(), result); + }); + }); +} + +attemptAsyncOperation('test input', function(err, errors, result) { + console.warn('err:'); + console.log(err); + + console.warn('result:'); + console.log(result); +}); + +function failingAsyncOperation(input, cb) { + return setImmediate(cb.bind(null, new Error('A fatal error'))); +} diff --git a/files_find/Files Find/bin/src/node_modules/retry/index.js b/files_find/Files Find/bin/src/node_modules/retry/index.js new file mode 100644 index 0000000..ee62f3a --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/index.js @@ -0,0 +1 @@ +module.exports = require('./lib/retry'); \ No newline at end of file diff --git a/files_find/Files Find/bin/src/node_modules/retry/lib/retry.js b/files_find/Files Find/bin/src/node_modules/retry/lib/retry.js new file mode 100644 index 0000000..dcb5768 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/lib/retry.js @@ -0,0 +1,100 @@ +var RetryOperation = require('./retry_operation'); + +exports.operation = function(options) { + var timeouts = exports.timeouts(options); + return new RetryOperation(timeouts, { + forever: options && options.forever, + unref: options && options.unref, + maxRetryTime: options && options.maxRetryTime + }); +}; + +exports.timeouts = function(options) { + if (options instanceof Array) { + return [].concat(options); + } + + var opts = { + retries: 10, + factor: 2, + minTimeout: 1 * 1000, + maxTimeout: Infinity, + randomize: false + }; + for (var key in options) { + opts[key] = options[key]; + } + + if (opts.minTimeout > opts.maxTimeout) { + throw new Error('minTimeout is greater than maxTimeout'); + } + + var timeouts = []; + for (var i = 0; i < opts.retries; i++) { + timeouts.push(this.createTimeout(i, opts)); + } + + if (options && options.forever && !timeouts.length) { + timeouts.push(this.createTimeout(i, opts)); + } + + // sort the array numerically ascending + timeouts.sort(function(a,b) { + return a - b; + }); + + return timeouts; +}; + +exports.createTimeout = function(attempt, opts) { + var random = (opts.randomize) + ? (Math.random() + 1) + : 1; + + var timeout = Math.round(random * opts.minTimeout * Math.pow(opts.factor, attempt)); + timeout = Math.min(timeout, opts.maxTimeout); + + return timeout; +}; + +exports.wrap = function(obj, options, methods) { + if (options instanceof Array) { + methods = options; + options = null; + } + + if (!methods) { + methods = []; + for (var key in obj) { + if (typeof obj[key] === 'function') { + methods.push(key); + } + } + } + + for (var i = 0; i < methods.length; i++) { + var method = methods[i]; + var original = obj[method]; + + obj[method] = function retryWrapper(original) { + var op = exports.operation(options); + var args = Array.prototype.slice.call(arguments, 1); + var callback = args.pop(); + + args.push(function(err) { + if (op.retry(err)) { + return; + } + if (err) { + arguments[0] = op.mainError(); + } + callback.apply(this, arguments); + }); + + op.attempt(function() { + original.apply(obj, args); + }); + }.bind(obj, original); + obj[method].options = options; + } +}; diff --git a/files_find/Files Find/bin/src/node_modules/retry/lib/retry_operation.js b/files_find/Files Find/bin/src/node_modules/retry/lib/retry_operation.js new file mode 100644 index 0000000..1e56469 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/lib/retry_operation.js @@ -0,0 +1,158 @@ +function RetryOperation(timeouts, options) { + // Compatibility for the old (timeouts, retryForever) signature + if (typeof options === 'boolean') { + options = { forever: options }; + } + + this._originalTimeouts = JSON.parse(JSON.stringify(timeouts)); + this._timeouts = timeouts; + this._options = options || {}; + this._maxRetryTime = options && options.maxRetryTime || Infinity; + this._fn = null; + this._errors = []; + this._attempts = 1; + this._operationTimeout = null; + this._operationTimeoutCb = null; + this._timeout = null; + this._operationStart = null; + + if (this._options.forever) { + this._cachedTimeouts = this._timeouts.slice(0); + } +} +module.exports = RetryOperation; + +RetryOperation.prototype.reset = function() { + this._attempts = 1; + this._timeouts = this._originalTimeouts; +} + +RetryOperation.prototype.stop = function() { + if (this._timeout) { + clearTimeout(this._timeout); + } + + this._timeouts = []; + this._cachedTimeouts = null; +}; + +RetryOperation.prototype.retry = function(err) { + if (this._timeout) { + clearTimeout(this._timeout); + } + + if (!err) { + return false; + } + var currentTime = new Date().getTime(); + if (err && currentTime - this._operationStart >= this._maxRetryTime) { + this._errors.unshift(new Error('RetryOperation timeout occurred')); + return false; + } + + this._errors.push(err); + + var timeout = this._timeouts.shift(); + if (timeout === undefined) { + if (this._cachedTimeouts) { + // retry forever, only keep last error + this._errors.splice(this._errors.length - 1, this._errors.length); + this._timeouts = this._cachedTimeouts.slice(0); + timeout = this._timeouts.shift(); + } else { + return false; + } + } + + var self = this; + var timer = setTimeout(function() { + self._attempts++; + + if (self._operationTimeoutCb) { + self._timeout = setTimeout(function() { + self._operationTimeoutCb(self._attempts); + }, self._operationTimeout); + + if (self._options.unref) { + self._timeout.unref(); + } + } + + self._fn(self._attempts); + }, timeout); + + if (this._options.unref) { + timer.unref(); + } + + return true; +}; + +RetryOperation.prototype.attempt = function(fn, timeoutOps) { + this._fn = fn; + + if (timeoutOps) { + if (timeoutOps.timeout) { + this._operationTimeout = timeoutOps.timeout; + } + if (timeoutOps.cb) { + this._operationTimeoutCb = timeoutOps.cb; + } + } + + var self = this; + if (this._operationTimeoutCb) { + this._timeout = setTimeout(function() { + self._operationTimeoutCb(); + }, self._operationTimeout); + } + + this._operationStart = new Date().getTime(); + + this._fn(this._attempts); +}; + +RetryOperation.prototype.try = function(fn) { + console.log('Using RetryOperation.try() is deprecated'); + this.attempt(fn); +}; + +RetryOperation.prototype.start = function(fn) { + console.log('Using RetryOperation.start() is deprecated'); + this.attempt(fn); +}; + +RetryOperation.prototype.start = RetryOperation.prototype.try; + +RetryOperation.prototype.errors = function() { + return this._errors; +}; + +RetryOperation.prototype.attempts = function() { + return this._attempts; +}; + +RetryOperation.prototype.mainError = function() { + if (this._errors.length === 0) { + return null; + } + + var counts = {}; + var mainError = null; + var mainErrorCount = 0; + + for (var i = 0; i < this._errors.length; i++) { + var error = this._errors[i]; + var message = error.message; + var count = (counts[message] || 0) + 1; + + counts[message] = count; + + if (count >= mainErrorCount) { + mainError = error; + mainErrorCount = count; + } + } + + return mainError; +}; diff --git a/files_find/Files Find/bin/src/node_modules/retry/package.json b/files_find/Files Find/bin/src/node_modules/retry/package.json new file mode 100644 index 0000000..73c7259 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/package.json @@ -0,0 +1,32 @@ +{ + "author": "Tim Koschützki (http://debuggable.com/)", + "name": "retry", + "description": "Abstraction for exponential and custom retry strategies for failed operations.", + "license": "MIT", + "version": "0.12.0", + "homepage": "https://github.com/tim-kos/node-retry", + "repository": { + "type": "git", + "url": "git://github.com/tim-kos/node-retry.git" + }, + "directories": { + "lib": "./lib" + }, + "main": "index", + "engines": { + "node": ">= 4" + }, + "dependencies": {}, + "devDependencies": { + "fake": "0.2.0", + "istanbul": "^0.4.5", + "tape": "^4.8.0" + }, + "scripts": { + "test": "./node_modules/.bin/istanbul cover ./node_modules/tape/bin/tape ./test/integration/*.js", + "release:major": "env SEMANTIC=major npm run release", + "release:minor": "env SEMANTIC=minor npm run release", + "release:patch": "env SEMANTIC=patch npm run release", + "release": "npm version ${SEMANTIC:-patch} -m \"Release %s\" && git push && git push --tags && npm publish" + } +} diff --git a/files_find/Files Find/bin/src/node_modules/retry/test/common.js b/files_find/Files Find/bin/src/node_modules/retry/test/common.js new file mode 100644 index 0000000..2247206 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/test/common.js @@ -0,0 +1,10 @@ +var common = module.exports; +var path = require('path'); + +var rootDir = path.join(__dirname, '..'); +common.dir = { + lib: rootDir + '/lib' +}; + +common.assert = require('assert'); +common.fake = require('fake'); \ No newline at end of file diff --git a/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-forever.js b/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-forever.js new file mode 100644 index 0000000..b41307c --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-forever.js @@ -0,0 +1,24 @@ +var common = require('../common'); +var assert = common.assert; +var retry = require(common.dir.lib + '/retry'); + +(function testForeverUsesFirstTimeout() { + var operation = retry.operation({ + retries: 0, + minTimeout: 100, + maxTimeout: 100, + forever: true + }); + + operation.attempt(function(numAttempt) { + console.log('>numAttempt', numAttempt); + var err = new Error("foo"); + if (numAttempt == 10) { + operation.stop(); + } + + if (operation.retry(err)) { + return; + } + }); +})(); diff --git a/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-retry-operation.js b/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-retry-operation.js new file mode 100644 index 0000000..e351bb6 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-retry-operation.js @@ -0,0 +1,258 @@ +var common = require('../common'); +var assert = common.assert; +var fake = common.fake.create(); +var retry = require(common.dir.lib + '/retry'); + +(function testReset() { + var error = new Error('some error'); + var operation = retry.operation([1, 2, 3]); + var attempts = 0; + + var finalCallback = fake.callback('finalCallback'); + fake.expectAnytime(finalCallback); + + var expectedFinishes = 1; + var finishes = 0; + + var fn = function() { + operation.attempt(function(currentAttempt) { + attempts++; + assert.equal(currentAttempt, attempts); + if (operation.retry(error)) { + return; + } + + finishes++ + assert.equal(expectedFinishes, finishes); + assert.strictEqual(attempts, 4); + assert.strictEqual(operation.attempts(), attempts); + assert.strictEqual(operation.mainError(), error); + + if (finishes < 2) { + attempts = 0; + expectedFinishes++; + operation.reset(); + fn() + } else { + finalCallback(); + } + }); + }; + + fn(); +})(); + +(function testErrors() { + var operation = retry.operation(); + + var error = new Error('some error'); + var error2 = new Error('some other error'); + operation._errors.push(error); + operation._errors.push(error2); + + assert.deepEqual(operation.errors(), [error, error2]); +})(); + +(function testMainErrorReturnsMostFrequentError() { + var operation = retry.operation(); + var error = new Error('some error'); + var error2 = new Error('some other error'); + + operation._errors.push(error); + operation._errors.push(error2); + operation._errors.push(error); + + assert.strictEqual(operation.mainError(), error); +})(); + +(function testMainErrorReturnsLastErrorOnEqualCount() { + var operation = retry.operation(); + var error = new Error('some error'); + var error2 = new Error('some other error'); + + operation._errors.push(error); + operation._errors.push(error2); + + assert.strictEqual(operation.mainError(), error2); +})(); + +(function testAttempt() { + var operation = retry.operation(); + var fn = new Function(); + + var timeoutOpts = { + timeout: 1, + cb: function() {} + }; + operation.attempt(fn, timeoutOpts); + + assert.strictEqual(fn, operation._fn); + assert.strictEqual(timeoutOpts.timeout, operation._operationTimeout); + assert.strictEqual(timeoutOpts.cb, operation._operationTimeoutCb); +})(); + +(function testRetry() { + var error = new Error('some error'); + var operation = retry.operation([1, 2, 3]); + var attempts = 0; + + var finalCallback = fake.callback('finalCallback'); + fake.expectAnytime(finalCallback); + + var fn = function() { + operation.attempt(function(currentAttempt) { + attempts++; + assert.equal(currentAttempt, attempts); + if (operation.retry(error)) { + return; + } + + assert.strictEqual(attempts, 4); + assert.strictEqual(operation.attempts(), attempts); + assert.strictEqual(operation.mainError(), error); + finalCallback(); + }); + }; + + fn(); +})(); + +(function testRetryForever() { + var error = new Error('some error'); + var operation = retry.operation({ retries: 3, forever: true }); + var attempts = 0; + + var finalCallback = fake.callback('finalCallback'); + fake.expectAnytime(finalCallback); + + var fn = function() { + operation.attempt(function(currentAttempt) { + attempts++; + assert.equal(currentAttempt, attempts); + if (attempts !== 6 && operation.retry(error)) { + return; + } + + assert.strictEqual(attempts, 6); + assert.strictEqual(operation.attempts(), attempts); + assert.strictEqual(operation.mainError(), error); + finalCallback(); + }); + }; + + fn(); +})(); + +(function testRetryForeverNoRetries() { + var error = new Error('some error'); + var delay = 50 + var operation = retry.operation({ + retries: null, + forever: true, + minTimeout: delay, + maxTimeout: delay + }); + + var attempts = 0; + var startTime = new Date().getTime(); + + var finalCallback = fake.callback('finalCallback'); + fake.expectAnytime(finalCallback); + + var fn = function() { + operation.attempt(function(currentAttempt) { + attempts++; + assert.equal(currentAttempt, attempts); + if (attempts !== 4 && operation.retry(error)) { + return; + } + + var endTime = new Date().getTime(); + var minTime = startTime + (delay * 3); + var maxTime = minTime + 20 // add a little headroom for code execution time + assert(endTime >= minTime) + assert(endTime < maxTime) + assert.strictEqual(attempts, 4); + assert.strictEqual(operation.attempts(), attempts); + assert.strictEqual(operation.mainError(), error); + finalCallback(); + }); + }; + + fn(); +})(); + +(function testStop() { + var error = new Error('some error'); + var operation = retry.operation([1, 2, 3]); + var attempts = 0; + + var finalCallback = fake.callback('finalCallback'); + fake.expectAnytime(finalCallback); + + var fn = function() { + operation.attempt(function(currentAttempt) { + attempts++; + assert.equal(currentAttempt, attempts); + + if (attempts === 2) { + operation.stop(); + + assert.strictEqual(attempts, 2); + assert.strictEqual(operation.attempts(), attempts); + assert.strictEqual(operation.mainError(), error); + finalCallback(); + } + + if (operation.retry(error)) { + return; + } + }); + }; + + fn(); +})(); + +(function testMaxRetryTime() { + var error = new Error('some error'); + var maxRetryTime = 30; + var operation = retry.operation({ + minTimeout: 1, + maxRetryTime: maxRetryTime + }); + var attempts = 0; + + var finalCallback = fake.callback('finalCallback'); + fake.expectAnytime(finalCallback); + + var longAsyncFunction = function (wait, callback){ + setTimeout(callback, wait); + }; + + var fn = function() { + var startTime = new Date().getTime(); + operation.attempt(function(currentAttempt) { + attempts++; + assert.equal(currentAttempt, attempts); + + if (attempts !== 2) { + if (operation.retry(error)) { + return; + } + } else { + var curTime = new Date().getTime(); + longAsyncFunction(maxRetryTime - (curTime - startTime - 1), function(){ + if (operation.retry(error)) { + assert.fail('timeout should be occurred'); + return; + } + + assert.strictEqual(operation.mainError(), error); + finalCallback(); + }); + } + }); + }; + + fn(); +})(); diff --git a/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-retry-wrap.js b/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-retry-wrap.js new file mode 100644 index 0000000..3d2b6bf --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-retry-wrap.js @@ -0,0 +1,101 @@ +var common = require('../common'); +var assert = common.assert; +var fake = common.fake.create(); +var retry = require(common.dir.lib + '/retry'); + +function getLib() { + return { + fn1: function() {}, + fn2: function() {}, + fn3: function() {} + }; +} + +(function wrapAll() { + var lib = getLib(); + retry.wrap(lib); + assert.equal(lib.fn1.name, 'bound retryWrapper'); + assert.equal(lib.fn2.name, 'bound retryWrapper'); + assert.equal(lib.fn3.name, 'bound retryWrapper'); +}()); + +(function wrapAllPassOptions() { + var lib = getLib(); + retry.wrap(lib, {retries: 2}); + assert.equal(lib.fn1.name, 'bound retryWrapper'); + assert.equal(lib.fn2.name, 'bound retryWrapper'); + assert.equal(lib.fn3.name, 'bound retryWrapper'); + assert.equal(lib.fn1.options.retries, 2); + assert.equal(lib.fn2.options.retries, 2); + assert.equal(lib.fn3.options.retries, 2); +}()); + +(function wrapDefined() { + var lib = getLib(); + retry.wrap(lib, ['fn2', 'fn3']); + assert.notEqual(lib.fn1.name, 'bound retryWrapper'); + assert.equal(lib.fn2.name, 'bound retryWrapper'); + assert.equal(lib.fn3.name, 'bound retryWrapper'); +}()); + +(function wrapDefinedAndPassOptions() { + var lib = getLib(); + retry.wrap(lib, {retries: 2}, ['fn2', 'fn3']); + assert.notEqual(lib.fn1.name, 'bound retryWrapper'); + assert.equal(lib.fn2.name, 'bound retryWrapper'); + assert.equal(lib.fn3.name, 'bound retryWrapper'); + assert.equal(lib.fn2.options.retries, 2); + assert.equal(lib.fn3.options.retries, 2); +}()); + +(function runWrappedWithoutError() { + var callbackCalled; + var lib = {method: function(a, b, callback) { + assert.equal(a, 1); + assert.equal(b, 2); + assert.equal(typeof callback, 'function'); + callback(); + }}; + retry.wrap(lib); + lib.method(1, 2, function() { + callbackCalled = true; + }); + assert.ok(callbackCalled); +}()); + +(function runWrappedSeveralWithoutError() { + var callbacksCalled = 0; + var lib = { + fn1: function (a, callback) { + assert.equal(a, 1); + assert.equal(typeof callback, 'function'); + callback(); + }, + fn2: function (a, callback) { + assert.equal(a, 2); + assert.equal(typeof callback, 'function'); + callback(); + } + }; + retry.wrap(lib, {}, ['fn1', 'fn2']); + lib.fn1(1, function() { + callbacksCalled++; + }); + lib.fn2(2, function() { + callbacksCalled++; + }); + assert.equal(callbacksCalled, 2); +}()); + +(function runWrappedWithError() { + var callbackCalled; + var lib = {method: function(callback) { + callback(new Error('Some error')); + }}; + retry.wrap(lib, {retries: 1}); + lib.method(function(err) { + callbackCalled = true; + assert.ok(err instanceof Error); + }); + assert.ok(!callbackCalled); +}()); diff --git a/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-timeouts.js b/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-timeouts.js new file mode 100644 index 0000000..7206b0f --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/retry/test/integration/test-timeouts.js @@ -0,0 +1,69 @@ +var common = require('../common'); +var assert = common.assert; +var retry = require(common.dir.lib + '/retry'); + +(function testDefaultValues() { + var timeouts = retry.timeouts(); + + assert.equal(timeouts.length, 10); + assert.equal(timeouts[0], 1000); + assert.equal(timeouts[1], 2000); + assert.equal(timeouts[2], 4000); +})(); + +(function testDefaultValuesWithRandomize() { + var minTimeout = 5000; + var timeouts = retry.timeouts({ + minTimeout: minTimeout, + randomize: true + }); + + assert.equal(timeouts.length, 10); + assert.ok(timeouts[0] > minTimeout); + assert.ok(timeouts[1] > timeouts[0]); + assert.ok(timeouts[2] > timeouts[1]); +})(); + +(function testPassedTimeoutsAreUsed() { + var timeoutsArray = [1000, 2000, 3000]; + var timeouts = retry.timeouts(timeoutsArray); + assert.deepEqual(timeouts, timeoutsArray); + assert.notStrictEqual(timeouts, timeoutsArray); +})(); + +(function testTimeoutsAreWithinBoundaries() { + var minTimeout = 1000; + var maxTimeout = 10000; + var timeouts = retry.timeouts({ + minTimeout: minTimeout, + maxTimeout: maxTimeout + }); + for (var i = 0; i < timeouts; i++) { + assert.ok(timeouts[i] >= minTimeout); + assert.ok(timeouts[i] <= maxTimeout); + } +})(); + +(function testTimeoutsAreIncremental() { + var timeouts = retry.timeouts(); + var lastTimeout = timeouts[0]; + for (var i = 0; i < timeouts; i++) { + assert.ok(timeouts[i] > lastTimeout); + lastTimeout = timeouts[i]; + } +})(); + +(function testTimeoutsAreIncrementalForFactorsLessThanOne() { + var timeouts = retry.timeouts({ + retries: 3, + factor: 0.5 + }); + + var expected = [250, 500, 1000]; + assert.deepEqual(expected, timeouts); +})(); + +(function testRetries() { + var timeouts = retry.timeouts({retries: 2}); + assert.strictEqual(timeouts.length, 2); +})(); diff --git a/files_find/Files Find/bin/src/node_modules/signal-exit/LICENSE.txt b/files_find/Files Find/bin/src/node_modules/signal-exit/LICENSE.txt new file mode 100644 index 0000000..eead04a --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/signal-exit/LICENSE.txt @@ -0,0 +1,16 @@ +The ISC License + +Copyright (c) 2015, Contributors + +Permission to use, copy, modify, and/or distribute this software +for any purpose with or without fee is hereby granted, provided +that the above copyright notice and this permission notice +appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE +LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/files_find/Files Find/bin/src/node_modules/signal-exit/README.md b/files_find/Files Find/bin/src/node_modules/signal-exit/README.md new file mode 100644 index 0000000..f9c7c00 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/signal-exit/README.md @@ -0,0 +1,39 @@ +# signal-exit + +[![Build Status](https://travis-ci.org/tapjs/signal-exit.png)](https://travis-ci.org/tapjs/signal-exit) +[![Coverage](https://coveralls.io/repos/tapjs/signal-exit/badge.svg?branch=master)](https://coveralls.io/r/tapjs/signal-exit?branch=master) +[![NPM version](https://img.shields.io/npm/v/signal-exit.svg)](https://www.npmjs.com/package/signal-exit) +[![Standard Version](https://img.shields.io/badge/release-standard%20version-brightgreen.svg)](https://github.com/conventional-changelog/standard-version) + +When you want to fire an event no matter how a process exits: + +* reaching the end of execution. +* explicitly having `process.exit(code)` called. +* having `process.kill(pid, sig)` called. +* receiving a fatal signal from outside the process + +Use `signal-exit`. + +```js +var onExit = require('signal-exit') + +onExit(function (code, signal) { + console.log('process exited!') +}) +``` + +## API + +`var remove = onExit(function (code, signal) {}, options)` + +The return value of the function is a function that will remove the +handler. + +Note that the function *only* fires for signals if the signal would +cause the process to exit. That is, there are no other listeners, and +it is a fatal signal. + +## Options + +* `alwaysLast`: Run this handler after any other signal or exit + handlers. This causes `process.emit` to be monkeypatched. diff --git a/files_find/Files Find/bin/src/node_modules/signal-exit/index.js b/files_find/Files Find/bin/src/node_modules/signal-exit/index.js new file mode 100644 index 0000000..93703f3 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/signal-exit/index.js @@ -0,0 +1,202 @@ +// Note: since nyc uses this module to output coverage, any lines +// that are in the direct sync flow of nyc's outputCoverage are +// ignored, since we can never get coverage for them. +// grab a reference to node's real process object right away +var process = global.process + +const processOk = function (process) { + return process && + typeof process === 'object' && + typeof process.removeListener === 'function' && + typeof process.emit === 'function' && + typeof process.reallyExit === 'function' && + typeof process.listeners === 'function' && + typeof process.kill === 'function' && + typeof process.pid === 'number' && + typeof process.on === 'function' +} + +// some kind of non-node environment, just no-op +/* istanbul ignore if */ +if (!processOk(process)) { + module.exports = function () { + return function () {} + } +} else { + var assert = require('assert') + var signals = require('./signals.js') + var isWin = /^win/i.test(process.platform) + + var EE = require('events') + /* istanbul ignore if */ + if (typeof EE !== 'function') { + EE = EE.EventEmitter + } + + var emitter + if (process.__signal_exit_emitter__) { + emitter = process.__signal_exit_emitter__ + } else { + emitter = process.__signal_exit_emitter__ = new EE() + emitter.count = 0 + emitter.emitted = {} + } + + // Because this emitter is a global, we have to check to see if a + // previous version of this library failed to enable infinite listeners. + // I know what you're about to say. But literally everything about + // signal-exit is a compromise with evil. Get used to it. + if (!emitter.infinite) { + emitter.setMaxListeners(Infinity) + emitter.infinite = true + } + + module.exports = function (cb, opts) { + /* istanbul ignore if */ + if (!processOk(global.process)) { + return function () {} + } + assert.equal(typeof cb, 'function', 'a callback must be provided for exit handler') + + if (loaded === false) { + load() + } + + var ev = 'exit' + if (opts && opts.alwaysLast) { + ev = 'afterexit' + } + + var remove = function () { + emitter.removeListener(ev, cb) + if (emitter.listeners('exit').length === 0 && + emitter.listeners('afterexit').length === 0) { + unload() + } + } + emitter.on(ev, cb) + + return remove + } + + var unload = function unload () { + if (!loaded || !processOk(global.process)) { + return + } + loaded = false + + signals.forEach(function (sig) { + try { + process.removeListener(sig, sigListeners[sig]) + } catch (er) {} + }) + process.emit = originalProcessEmit + process.reallyExit = originalProcessReallyExit + emitter.count -= 1 + } + module.exports.unload = unload + + var emit = function emit (event, code, signal) { + /* istanbul ignore if */ + if (emitter.emitted[event]) { + return + } + emitter.emitted[event] = true + emitter.emit(event, code, signal) + } + + // { : , ... } + var sigListeners = {} + signals.forEach(function (sig) { + sigListeners[sig] = function listener () { + /* istanbul ignore if */ + if (!processOk(global.process)) { + return + } + // If there are no other listeners, an exit is coming! + // Simplest way: remove us and then re-send the signal. + // We know that this will kill the process, so we can + // safely emit now. + var listeners = process.listeners(sig) + if (listeners.length === emitter.count) { + unload() + emit('exit', null, sig) + /* istanbul ignore next */ + emit('afterexit', null, sig) + /* istanbul ignore next */ + if (isWin && sig === 'SIGHUP') { + // "SIGHUP" throws an `ENOSYS` error on Windows, + // so use a supported signal instead + sig = 'SIGINT' + } + /* istanbul ignore next */ + process.kill(process.pid, sig) + } + } + }) + + module.exports.signals = function () { + return signals + } + + var loaded = false + + var load = function load () { + if (loaded || !processOk(global.process)) { + return + } + loaded = true + + // This is the number of onSignalExit's that are in play. + // It's important so that we can count the correct number of + // listeners on signals, and don't wait for the other one to + // handle it instead of us. + emitter.count += 1 + + signals = signals.filter(function (sig) { + try { + process.on(sig, sigListeners[sig]) + return true + } catch (er) { + return false + } + }) + + process.emit = processEmit + process.reallyExit = processReallyExit + } + module.exports.load = load + + var originalProcessReallyExit = process.reallyExit + var processReallyExit = function processReallyExit (code) { + /* istanbul ignore if */ + if (!processOk(global.process)) { + return + } + process.exitCode = code || /* istanbul ignore next */ 0 + emit('exit', process.exitCode, null) + /* istanbul ignore next */ + emit('afterexit', process.exitCode, null) + /* istanbul ignore next */ + originalProcessReallyExit.call(process, process.exitCode) + } + + var originalProcessEmit = process.emit + var processEmit = function processEmit (ev, arg) { + if (ev === 'exit' && processOk(global.process)) { + /* istanbul ignore else */ + if (arg !== undefined) { + process.exitCode = arg + } + var ret = originalProcessEmit.apply(this, arguments) + /* istanbul ignore next */ + emit('exit', process.exitCode, null) + /* istanbul ignore next */ + emit('afterexit', process.exitCode, null) + /* istanbul ignore next */ + return ret + } else { + return originalProcessEmit.apply(this, arguments) + } + } +} diff --git a/files_find/Files Find/bin/src/node_modules/signal-exit/package.json b/files_find/Files Find/bin/src/node_modules/signal-exit/package.json new file mode 100644 index 0000000..e1a0031 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/signal-exit/package.json @@ -0,0 +1,38 @@ +{ + "name": "signal-exit", + "version": "3.0.7", + "description": "when you want to fire an event no matter how a process exits.", + "main": "index.js", + "scripts": { + "test": "tap", + "snap": "tap", + "preversion": "npm test", + "postversion": "npm publish", + "prepublishOnly": "git push origin --follow-tags" + }, + "files": [ + "index.js", + "signals.js" + ], + "repository": { + "type": "git", + "url": "https://github.com/tapjs/signal-exit.git" + }, + "keywords": [ + "signal", + "exit" + ], + "author": "Ben Coe ", + "license": "ISC", + "bugs": { + "url": "https://github.com/tapjs/signal-exit/issues" + }, + "homepage": "https://github.com/tapjs/signal-exit", + "devDependencies": { + "chai": "^3.5.0", + "coveralls": "^3.1.1", + "nyc": "^15.1.0", + "standard-version": "^9.3.1", + "tap": "^15.1.1" + } +} diff --git a/files_find/Files Find/bin/src/node_modules/signal-exit/signals.js b/files_find/Files Find/bin/src/node_modules/signal-exit/signals.js new file mode 100644 index 0000000..3bd67a8 --- /dev/null +++ b/files_find/Files Find/bin/src/node_modules/signal-exit/signals.js @@ -0,0 +1,53 @@ +// This is not the set of all possible signals. +// +// It IS, however, the set of all signals that trigger +// an exit on either Linux or BSD systems. Linux is a +// superset of the signal names supported on BSD, and +// the unknown signals just fail to register, so we can +// catch that easily enough. +// +// Don't bother with SIGKILL. It's uncatchable, which +// means that we can't fire any callbacks anyway. +// +// If a user does happen to register a handler on a non- +// fatal signal like SIGWINCH or something, and then +// exit, it'll end up firing `process.emit('exit')`, so +// the handler will be fired anyway. +// +// SIGBUS, SIGFPE, SIGSEGV and SIGILL, when not raised +// artificially, inherently leave the process in a +// state from which it is not safe to try and enter JS +// listeners. +module.exports = [ + 'SIGABRT', + 'SIGALRM', + 'SIGHUP', + 'SIGINT', + 'SIGTERM' +] + +if (process.platform !== 'win32') { + module.exports.push( + 'SIGVTALRM', + 'SIGXCPU', + 'SIGXFSZ', + 'SIGUSR2', + 'SIGTRAP', + 'SIGSYS', + 'SIGQUIT', + 'SIGIOT' + // should detect profiler and enable/disable accordingly. + // see #21 + // 'SIGPROF' + ) +} + +if (process.platform === 'linux') { + module.exports.push( + 'SIGIO', + 'SIGPOLL', + 'SIGPWR', + 'SIGSTKFLT', + 'SIGUNUSED' + ) +} diff --git a/for_each_start/For Each Start/bin/processor.exe b/for_each_start/For Each Start/bin/processor.exe deleted file mode 100644 index 1b11e6a..0000000 Binary files a/for_each_start/For Each Start/bin/processor.exe and /dev/null differ