From 80be69f01b46ecf371bbf877c3258bd1923c561c Mon Sep 17 00:00:00 2001 From: Fabien Franzen Date: Sat, 22 Jan 2011 09:33:45 -0800 Subject: [PATCH 01/28] Fixed _keys duplicates during _load() (which came apparent in forEach loops). --- lib/dirty/dirty.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index 31fe598..e3fd671 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -104,9 +104,9 @@ Dirty.prototype._load = function() { delete self._docs[row.key]; } else { if (!(row.key in self._docs)) { + self._keys.push(row.key); length++; } - self._keys.push(row.key); self._docs[row.key] = row.val; } return ''; From 1ed10dc801c8730cd3f6a5ad82fb130d736043a6 Mon Sep 17 00:00:00 2001 From: Fabien Franzen Date: Sun, 27 Feb 2011 23:53:58 -0800 Subject: [PATCH 02/28] Fixed duplicate keys (issue 17) --- lib/dirty/dirty.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index e3fd671..ac30bc2 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -30,7 +30,7 @@ Dirty.prototype.set = function(key, val, cb) { this._keys.splice(this._keys.indexOf(key), 1) delete this._docs[key]; } else { - if (!this._keys[key]) { + if (this._keys.indexOf(key) === -1) { this._keys.push(key); } this._docs[key] = val; From 0fb2660449138d45c6b21233a3cb311a3d4d534e Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Thu, 14 Apr 2011 15:40:24 +0530 Subject: [PATCH 03/28] fixing existing tests for current version of node --- lib/dirty/dirty.js | 2 +- test/simple/test-dirty.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index ac30bc2..eb6a173 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -77,7 +77,7 @@ Dirty.prototype._load = function() { this._readStream .on('error', function(err) { - if (err.errno == process.binding('net').ENOENT) { + if (err.code == 'ENOENT') { self.emit('load', 0); return; } diff --git a/test/simple/test-dirty.js b/test/simple/test-dirty.js index 9f462b7..5b643ab 100644 --- a/test/simple/test-dirty.js +++ b/test/simple/test-dirty.js @@ -181,7 +181,7 @@ test(function _load() { assert.equal(event, 'load'); assert.equal(length, 0); }); - readStreamEmit.error({errno: process.binding('net').ENOENT}) + readStreamEmit.error({code: 'ENOENT'}) })(); })(); }); From a3a6d264934a22bdcb7df19c7929c0d1ca56111b Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Thu, 14 Apr 2011 15:43:26 +0530 Subject: [PATCH 04/28] removing keys array. --- lib/dirty/dirty.js | 11 +---------- test/system/test-load.js | 6 +++++- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index eb6a173..01de627 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -13,7 +13,6 @@ var Dirty = exports.Dirty = function(path) { this.writeBundle = 1000; this._docs = {}; - this._keys = []; this._queue = []; this._readStream = null; this._writeStream = null; @@ -27,12 +26,8 @@ module.exports = Dirty; Dirty.prototype.set = function(key, val, cb) { if (val === undefined) { - this._keys.splice(this._keys.indexOf(key), 1) delete this._docs[key]; } else { - if (this._keys.indexOf(key) === -1) { - this._keys.push(key); - } this._docs[key] = val; } @@ -54,14 +49,11 @@ Dirty.prototype.rm = function(key, cb) { }; Dirty.prototype.forEach = function(fn) { - - for (var i = 0; i < this._keys.length; i++) { - var key = this._keys[i]; + for (var key in this._docs) { if (fn(key, this._docs[key]) === false) { break; } } - }; Dirty.prototype._load = function() { @@ -104,7 +96,6 @@ Dirty.prototype._load = function() { delete self._docs[row.key]; } else { if (!(row.key in self._docs)) { - self._keys.push(row.key); length++; } self._docs[row.key] = row.val; diff --git a/test/system/test-load.js b/test/system/test-load.js index 21dd88f..1bcbee4 100644 --- a/test/system/test-load.js +++ b/test/system/test-load.js @@ -19,7 +19,11 @@ db.on('drain', function() { assert.strictEqual(db2.get(1), 'A'); assert.strictEqual(db2.get(2), 'B'); assert.strictEqual(db2.get(3), undefined); - assert.strictEqual(db2._keys.length, 2); + var count = 0; + for (var k in db2._docs) { + count++; + } + assert.strictEqual(count, 2); assert.ok(!('3' in db2._docs)); }); }); From 527e69d37b0140a0c56886d404f70046e0ef3923 Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Sat, 16 Apr 2011 12:42:17 +0530 Subject: [PATCH 05/28] added compacting ability --- lib/dirty/dirty.js | 67 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index 01de627..329c121 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -127,7 +127,7 @@ Dirty.prototype._load = function() { }; Dirty.prototype._maybeFlush = function() { - if (this.flushing || !this.path || !this._queue.length) { + if (this.flushing || !this.path || !this._queue.length || this.compacting) { return; } @@ -178,3 +178,68 @@ Dirty.prototype._flush = function() { this._queue = []; }; + +Dirty.prototype.compact = function(cb) { + if (this.compacting) return; + this.compacting = true; + var self = this; + this.on('compacted', function(){ + self._queue_backup = []; + self._endCompacting(cb); + }); + this.on('compactingFailed', function(){ + self._queue = self._queue_backup.concat(self._queue); + self._endCompacting(cb); + }) + if (this.flushing) { + this._writeStream.once('drain', function(){ + self._startCompacting(); + }) + } else { + this._startCompacting(); + } +} + +Dirty.prototype._endCompacting = function(cb) { + this.compacting = false; + if (cb) cb(); + this._maybeFlush(); +} + +Dirty.prototype.__defineGetter__("_compactPath", function() { + return this.path + ".compact"; +}) + +Dirty.prototype._startCompacting = function() { + var self = this; + this._queue_backup = this.queue; + this._queue = []; + var ws = fs.createWriteStream(this._compactPath, { + encoding: 'utf-8', + flags: 'w' + }); + var bundleLength = 0; + var bundleStr = ''; + var writeToStream = function() { + ws.write(bundleStr); + bundleStr = ''; + bundleLength = 0; + } + for (var k in this._docs) { + bundleStr += JSON.stringify({key: k, val: this._docs[k]})+'\n'; + bundleLength++; + if (bundleLength >= this.writeBundle) { + writeToStream(); + } + }; + writeToStream(); + ws.on("error", function(){ + self.emit('compactingFailed'); + }); + ws.on('drain', function(){ + fs.rename(self._compactPath, self.path, function(err){ + if (err) self.emit('compactingFailed'); + else self.emit('compacted'); + }) + }); +} From 63d851ad8a2ad17a76f4d828bafb35dc97c4d858 Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Sun, 17 Apr 2011 10:35:16 +0530 Subject: [PATCH 06/28] tests for compacting behaviour --- lib/dirty/dirty.js | 79 +++++++++------- test/simple/test-dirty.js | 194 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 232 insertions(+), 41 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index 329c121..361d585 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -128,6 +128,7 @@ Dirty.prototype._load = function() { Dirty.prototype._maybeFlush = function() { if (this.flushing || !this.path || !this._queue.length || this.compacting) { + return; } @@ -179,45 +180,66 @@ Dirty.prototype._flush = function() { this._queue = []; }; +Dirty.prototype.__defineGetter__("_compactPath", function() { + return this.path + ".compact"; +}); + Dirty.prototype.compact = function(cb) { if (this.compacting) return; - this.compacting = true; var self = this; - this.on('compacted', function(){ - self._queue_backup = []; - self._endCompacting(cb); - }); - this.on('compactingFailed', function(){ - self._queue = self._queue_backup.concat(self._queue); - self._endCompacting(cb); - }) + this.compacting = true; + if (this.flushing) { this._writeStream.once('drain', function(){ self._startCompacting(); - }) - } else { + }); + } else { this._startCompacting(); } -} - -Dirty.prototype._endCompacting = function(cb) { - this.compacting = false; - if (cb) cb(); - this._maybeFlush(); -} - -Dirty.prototype.__defineGetter__("_compactPath", function() { - return this.path + ".compact"; -}) +}; Dirty.prototype._startCompacting = function() { var self = this; - this._queue_backup = this.queue; + this._queueBackup = this._queue; this._queue = []; var ws = fs.createWriteStream(this._compactPath, { encoding: 'utf-8', flags: 'w' }); + ws.on("error", function(){ + self.emit('compactingFailed'); + }); + ws.on('drain', function(){ + self._moveCompactedDataOverOriginal(); + }); + this._writeCompactedData(ws); +}; + +Dirty.prototype._moveCompactedDataOverOriginal = function() { + var self = this; + fs.rename(this._compactPath, self.path, function(err){ + if (err) self.emit('compactingFailed'); + else self.emit('compacted'); + }); +} + +Dirty.prototype.on('compacted', function(){ + this._queueBackup = []; + this._endCompacting(); +}); + +Dirty.prototype.on('compactingError', function(){ + this._queue = this._queueBackup.concat(this._queue); + this._endCompacting(); +}); + +Dirty.prototype._endCompacting = function(cb) { + this.compacting = false; + if (cb) cb(); + this._maybeFlush(); +}; + +Dirty.prototype._writeCompactedData = function(ws) { var bundleLength = 0; var bundleStr = ''; var writeToStream = function() { @@ -233,13 +255,4 @@ Dirty.prototype._startCompacting = function() { } }; writeToStream(); - ws.on("error", function(){ - self.emit('compactingFailed'); - }); - ws.on('drain', function(){ - fs.rename(self._compactPath, self.path, function(err){ - if (err) self.emit('compactingFailed'); - else self.emit('compacted'); - }) - }); -} +}; diff --git a/test/simple/test-dirty.js b/test/simple/test-dirty.js index 5b643ab..78cef6a 100644 --- a/test/simple/test-dirty.js +++ b/test/simple/test-dirty.js @@ -1,9 +1,11 @@ require('../common'); +fsStub = global.GENTLY.stub('fs'); var Dirty = require('dirty'), EventEmitter = require('events').EventEmitter, dirtyLoad = Dirty.prototype._load, gently, dirty; +process.setMaxListeners(20); (function testConstructor() { var gently = new Gently(); @@ -220,29 +222,33 @@ test(function set() { }); test(function _maybeFlush() { + function setup() { + dirty.compacting = false; + dirty.flushing = false; + dirty.path = '/foo/bar'; + dirty._queue = [1]; + } + (function testNothingToFlush() { gently.expect(dirty, '_flush', 0); dirty._maybeFlush(); })(); (function testFlush() { - dirty.flushing = false; - dirty.path = '/foo/bar'; - dirty._queue = [1]; - + setup(); gently.expect(dirty, '_flush'); dirty._maybeFlush(); })(); (function testOneFlushAtATime() { + setup(); dirty.flushing = true; - gently.expect(dirty, '_flush', 0); dirty._maybeFlush(); })(); (function testNoFlushingWithoutPath() { - dirty.flushing = false; + setup(); dirty.path = null; gently.expect(dirty, '_flush', 0); @@ -250,13 +256,19 @@ test(function _maybeFlush() { })(); (function testNoFlushingWithoutQueue() { - dirty.flushing = false; - dirty.path = '/foo/bar'; + setup(); dirty._queue = []; gently.expect(dirty, '_flush', 0); dirty._maybeFlush(); })(); + + (function testNoFlushingWhileCompacting() { + setup(); + dirty.compacting = true; + gently.expect(dirty, '_flush', 0); + dirty._maybeFlush(); + })(); }); test(function _flush() { @@ -300,6 +312,172 @@ test(function _flush() { assert.deepEqual(dirty._queue, []); }); +test(function compactPath(){ + (function testIsPathAppendedWithCompact(){ + dirty.path = 'foo/bar.baz'; + assert.strictEqual('foo/bar.baz.compact', dirty._compactPath); + })(); +}); + +test(function compact() { + var setup = function() { + dirty.compacting = false; + dirty.flushing = false; + dirty._writeStream = {}; + }; + + (function testSetsCompactingAndStartsCompacting(){ + setup(); + gently.expect(dirty,'_startCompacting'); + dirty.compact(); + assert.strictEqual(true, dirty.compacting); + })(); + + (function testDoesntDoAnythingWhenAlreadyCompacting() { + setup(); + dirty.compacting = true; + gently.expect(dirty, '_startCompacting', 0); + dirty.compact(); + })(); + + (function whenFlushing(oldSetup) { + var setup = function() { + oldSetup(); + dirty.flushing = true; + }; + (function testDoesntStart() { + setup(); + gently.expect(dirty._writeStream, "once"); + gently.expect(dirty, '_startCompacting', 0); + dirty.compact(); + })(); + + (function testSetsCompactingToTrue(){ + setup(); + gently.expect(dirty._writeStream, "once"); + dirty.compact(); + assert.strictEqual(true, dirty.compacting); + })(); + + (function testSignsUpForASingleDrainOnWritestream() { + setup(); + gently.expect(dirty._writeStream, "once", function(a,b){ + assert.strictEqual("drain", a); + }) + dirty.compact(); + })(); + + (function testStartsCompactingOnceDrainIsEmitted(){ + setup(); + gently.expect(dirty._writeStream, "once", function(a,b){ + gently.expect(dirty, "_startCompacting"); + b(); + }); + dirty.compact(); + })(); + })(setup); +}); + +test(function _startCompacting(){ + var ws = {}; + function expectWriteStream(){ + dirty.path = 'foo/bar.baz'; + gently.expect(fsStub, "createWriteStream", function(path, obj){ + assert.strictEqual("foo/bar.baz.compact", path); + assert.strictEqual('utf-8', obj.encoding); + assert.strictEqual('w', obj.flags); + return ws; + }); + }; + function setup(){ + expectWriteStream(); + gently.expect(ws, "on", 2); + gently.expect(dirty, '_writeCompactedData'); + }; + (function testBacksUpAndEmptiesTheQueue(){ + setup(); + dirty._queue = [1,2,3]; + dirty._queueBackup = ["hi", "hello"]; + dirty._startCompacting(); + assert.deepEqual([], dirty._queue); + assert.deepEqual([1,2,3], dirty._queueBackup); + })(); + + (function testCreatesAWriteStreamToACompactFile(){ + setup(); + dirty._startCompacting(); + })(); + + (function testEmitsCompactingFailedOnWriteError(){ + expectWriteStream(); + gently.expect(ws, 'on', function(type, cb){ + assert.strictEqual('error', type); + gently.expect(dirty, 'emit', function(evt){ + assert.strictEqual('compactingFailed', evt); + }); + cb(); + gently.expect(ws, 'on'); + gently.expect(dirty, '_writeCompactedData'); + }); + dirty._startCompacting(); + })(); + + (function testRenamesCompactFileToOriginalOnDrain(){ + expectWriteStream(); + gently.expect(ws, 'on'); + gently.expect(ws, 'on', function(type, cb){ + assert.strictEqual('drain', type); + gently.expect(dirty, "_moveCompactedDataOverOriginal"); + cb(); + gently.expect(dirty,'_writeCompactedData'); + }) + dirty._startCompacting(); + })(); + + (function testPassesWriteStreamToWriteCompactedData(){ + expectWriteStream(); + gently.expect(ws, 'on',2); + gently.expect(dirty, '_writeCompactedData', function(obj){ + assert.strictEqual(ws, obj); + }); + dirty._startCompacting(); + })(); +}); + +test(function _moveCompactedDataOverOriginal() { + (function testRenamesCompactedFileToOriginal(){ + dirty.path = 'foo/bar.baz' + gently.expect(fsStub, 'rename', function(src, dst){ + assert.strictEqual('foo/bar.baz.compact', src); + assert.strictEqual('foo/bar.baz', dst); + }); + dirty._moveCompactedDataOverOriginal(); + })(); + + (function testEmitsCompactingFailedIfRenameErrorsOut(){ + gently.expect(fsStub, 'rename', function(a,b,cb){ + gently.expect(dirty, 'emit', function(evt){ + assert.strictEqual('compactingFailed', evt); + }); + cb(new Error('')); + }); + dirty._moveCompactedDataOverOriginal(); + })(); + + (function testEmitsCompactedIfRenameErrorsOut(){ + gently.expect(fsStub, 'rename', function(a,b,cb){ + gently.expect(dirty, 'emit', function(evt){ + assert.strictEqual('compacted', evt); + }); + cb(); + }); + dirty._moveCompactedDataOverOriginal(); + })(); +}); + +// test(function ) + + test(function rm() { var KEY = 'foo', CB = function() {}; gently.expect(dirty, 'set', function (key, val, cb) { From 4799ef4cffa379f87af080657a6665aa2a1bbabd Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Sun, 17 Apr 2011 23:30:41 +0530 Subject: [PATCH 07/28] new test for compaction --- lib/dirty/dirty.js | 41 ++++++------ test/simple/test-dirty.js | 129 +++++++++++++++++++++++++------------- 2 files changed, 107 insertions(+), 63 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index 361d585..c51c2d9 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -109,21 +109,24 @@ Dirty.prototype._load = function() { } self.emit('load', length); }); + this._recreateWriteStream(); +}; - this._writeStream = fs.createWriteStream(this.path, { - encoding: 'utf-8', - flags: 'a' - }); - - this._writeStream.on('drain', function() { - self.flushing = false; +Dirty.prototype._recreateWriteStream = function(){ + var self = this; + this._writeStream = fs.createWriteStream(this.path, { + encoding: 'utf-8', + flags: 'a' + }); - if (!self._queue.length) { - self.emit('drain'); - } else { - self._maybeFlush(); - } - }); + this._writeStream.on('drain', function() { + self.flushing = false; + if (!self._queue.length) { + self.emit('drain'); + } else { + self._maybeFlush(); + } + }); }; Dirty.prototype._maybeFlush = function() { @@ -207,7 +210,7 @@ Dirty.prototype._startCompacting = function() { flags: 'w' }); ws.on("error", function(){ - self.emit('compactingFailed'); + self.emit('compactingError'); }); ws.on('drain', function(){ self._moveCompactedDataOverOriginal(); @@ -217,14 +220,14 @@ Dirty.prototype._startCompacting = function() { Dirty.prototype._moveCompactedDataOverOriginal = function() { var self = this; - fs.rename(this._compactPath, self.path, function(err){ - if (err) self.emit('compactingFailed'); + fs.rename(this._compactPath, this.path, function(err){ + self._recreateWriteStream(); + if (err) self.emit('compactingError'); else self.emit('compacted'); }); } Dirty.prototype.on('compacted', function(){ - this._queueBackup = []; this._endCompacting(); }); @@ -233,9 +236,9 @@ Dirty.prototype.on('compactingError', function(){ this._endCompacting(); }); -Dirty.prototype._endCompacting = function(cb) { +Dirty.prototype._endCompacting = function() { + this._queueBackup = []; this.compacting = false; - if (cb) cb(); this._maybeFlush(); }; diff --git a/test/simple/test-dirty.js b/test/simple/test-dirty.js index 78cef6a..21ed791 100644 --- a/test/simple/test-dirty.js +++ b/test/simple/test-dirty.js @@ -1,5 +1,4 @@ require('../common'); -fsStub = global.GENTLY.stub('fs'); var Dirty = require('dirty'), EventEmitter = require('events').EventEmitter, dirtyLoad = Dirty.prototype._load, @@ -54,7 +53,6 @@ test(function _load() { (function testWithPath() { var PATH = dirty.path = '/dirty.db', READ_STREAM = {}, - WRITE_STREAM = {}, readStreamEmit = {}; gently.expect(HIJACKED.fs, 'createReadStream', function (path, options) { @@ -71,44 +69,11 @@ test(function _load() { readStreamEmit[event] = cb; return this; }); - - gently.expect(HIJACKED.fs, 'createWriteStream', function (path, options) { - assert.equal(path, PATH); - assert.equal(options.flags, 'a'); - assert.equal(options.encoding, 'utf-8'); - - return WRITE_STREAM; - }); - - gently.expect(WRITE_STREAM, 'on', function (event, cb) { - assert.strictEqual(event, 'drain'); - - (function testQueueEmpty() { - dirty._queue = []; - dirty.flushing = true; - - gently.expect(dirty, 'emit', function (event) { - assert.strictEqual(event, 'drain'); - }); - - cb(); - assert.strictEqual(dirty.flushing, false); - })(); - - (function testQueueNotEmpty() { - dirty._queue = [1]; - dirty.flushing = true; - - gently.expect(dirty, '_maybeFlush'); - - cb(); - assert.strictEqual(dirty.flushing, false); - })(); - }); + + gently.expect(dirty, '_recreateWriteStream'); dirtyLoad.call(dirty); - assert.strictEqual(dirty._writeStream, WRITE_STREAM); assert.strictEqual(dirty._readStream, READ_STREAM); (function testReading() { @@ -188,6 +153,48 @@ test(function _load() { })(); }); +test(function _recreateWriteStream(){ + var WRITE_STREAM = {}; + var PATH = 'foo/bar.baz'; + dirty.path = PATH; + + gently.expect(HIJACKED.fs, 'createWriteStream', function (path, options) { + assert.equal(path, PATH); + assert.equal(options.flags, 'a'); + assert.equal(options.encoding, 'utf-8'); + + return WRITE_STREAM; + }); + + gently.expect(WRITE_STREAM, 'on', function (event, cb) { + assert.strictEqual(event, 'drain'); + + (function testQueueEmpty() { + dirty._queue = []; + dirty.flushing = true; + + gently.expect(dirty, 'emit', function (event) { + assert.strictEqual(event, 'drain'); + }); + + cb(); + assert.strictEqual(dirty.flushing, false); + })(); + + (function testQueueNotEmpty() { + dirty._queue = [1]; + dirty.flushing = true; + + gently.expect(dirty, '_maybeFlush'); + + cb(); + assert.strictEqual(dirty.flushing, false); + })(); + }); + dirty._recreateWriteStream(); + assert.strictEqual(dirty._writeStream, WRITE_STREAM); +}); + test(function get() { var KEY = 'example', VAL = {}; dirty._docs[KEY] = VAL; @@ -382,7 +389,7 @@ test(function _startCompacting(){ var ws = {}; function expectWriteStream(){ dirty.path = 'foo/bar.baz'; - gently.expect(fsStub, "createWriteStream", function(path, obj){ + gently.expect(HIJACKED.fs, "createWriteStream", function(path, obj){ assert.strictEqual("foo/bar.baz.compact", path); assert.strictEqual('utf-8', obj.encoding); assert.strictEqual('w', obj.flags); @@ -413,7 +420,7 @@ test(function _startCompacting(){ gently.expect(ws, 'on', function(type, cb){ assert.strictEqual('error', type); gently.expect(dirty, 'emit', function(evt){ - assert.strictEqual('compactingFailed', evt); + assert.strictEqual('compactingError', evt); }); cb(); gently.expect(ws, 'on'); @@ -447,7 +454,7 @@ test(function _startCompacting(){ test(function _moveCompactedDataOverOriginal() { (function testRenamesCompactedFileToOriginal(){ dirty.path = 'foo/bar.baz' - gently.expect(fsStub, 'rename', function(src, dst){ + gently.expect(HIJACKED.fs, 'rename', function(src, dst){ assert.strictEqual('foo/bar.baz.compact', src); assert.strictEqual('foo/bar.baz', dst); }); @@ -455,9 +462,10 @@ test(function _moveCompactedDataOverOriginal() { })(); (function testEmitsCompactingFailedIfRenameErrorsOut(){ - gently.expect(fsStub, 'rename', function(a,b,cb){ + gently.expect(HIJACKED.fs, 'rename', function(a,b,cb){ + gently.expect(dirty, '_recreateWriteStream'); gently.expect(dirty, 'emit', function(evt){ - assert.strictEqual('compactingFailed', evt); + assert.strictEqual('compactingError', evt); }); cb(new Error('')); }); @@ -465,7 +473,8 @@ test(function _moveCompactedDataOverOriginal() { })(); (function testEmitsCompactedIfRenameErrorsOut(){ - gently.expect(fsStub, 'rename', function(a,b,cb){ + gently.expect(HIJACKED.fs, 'rename', function(a,b,cb){ + gently.expect(dirty, '_recreateWriteStream'); gently.expect(dirty, 'emit', function(evt){ assert.strictEqual('compacted', evt); }); @@ -475,8 +484,40 @@ test(function _moveCompactedDataOverOriginal() { })(); }); -// test(function ) +var _onCompactingComplete = function (evt){ + (function testEmptiesTheBackupQueue(){ + dirty._queueBackup = [1,2,3]; + gently.expect(dirty, "_maybeFlush"); + evt(); + assert.deepEqual([],dirty._queueBackup); + })(); + + (function testSetsCompactingToFalse(){ + dirty._queueBackup = [1,2,3]; + gently.expect(dirty, "_maybeFlush"); + evt(); + assert.strictEqual(false, dirty.compacting); + })(); +} + +test(function _onCompactedResetsTheState(){ + _onCompactingComplete(function(){ + dirty.emit("compacted"); + }); +}); +test(function _onCompactingErrorResetsTheStateToBeforeCompacting(){ + (function testPrependsTheBackupQueueToTheQueue(){ + dirty._queueBackup = ["tap", "2R"]; + dirty._queue = ["destroy", "one", "land"]; + dirty.emit('compactingError'); + assert.deepEqual(["tap", "2R", "destroy", "one", "land"], dirty._queue); + })(); + + _onCompactingComplete(function(){ + dirty.emit('compactingError'); + }); +}); test(function rm() { var KEY = 'foo', CB = function() {}; From 4e8652a0746b4cebf9d570bd85d15c1d39bd9509 Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Mon, 18 Apr 2011 01:12:58 +0530 Subject: [PATCH 08/28] added a compacting filter --- lib/dirty/dirty.js | 11 ++++++++++- test/system/test-compact.js | 31 +++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 test/system/test-compact.js diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index c51c2d9..95beaa3 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -16,6 +16,7 @@ var Dirty = exports.Dirty = function(path) { this._queue = []; this._readStream = null; this._writeStream = null; + this._compactingFilters = []; this._load(); }; @@ -251,7 +252,11 @@ Dirty.prototype._writeCompactedData = function(ws) { bundleLength = 0; } for (var k in this._docs) { - bundleStr += JSON.stringify({key: k, val: this._docs[k]})+'\n'; + var doc = this._docs[k]; + if (this._compactingFilters.every(function(filterFn){ + return filterFn(k, doc); + })) continue; + bundleStr += JSON.stringify({key: k, val: doc})+'\n'; bundleLength++; if (bundleLength >= this.writeBundle) { writeToStream(); @@ -259,3 +264,7 @@ Dirty.prototype._writeCompactedData = function(ws) { }; writeToStream(); }; + +Dirty.prototype.addCompactingFilter = function(filter) { + this._compactingFilters.push(filter); +} diff --git a/test/system/test-compact.js b/test/system/test-compact.js new file mode 100644 index 0000000..4841432 --- /dev/null +++ b/test/system/test-compact.js @@ -0,0 +1,31 @@ +require('../common'); +var DB_FILE = TEST_TMP+'/flush.dirty'; + db = require('dirty')(DB_FILE), + fs = require('fs'); + +db.addCompactingFilter(function(key, val){ + return /magic/.test(val); +}); + +db.on('load', function(){ + db.set('red', 'lightning Bolt'); + db.set('black', 'dark ritual'); + db.set('blue', 'ancestral recall'); + db.set('white', 'healing salve'); + db.set('green', 'llanowar Elves'); + db.set('purple', 'some magic'); + db.on('drain', function(){ + db.set('green', 'giant growth'); + db.compact(); + db.on('compacted', function(){ + assert.strictEqual( + fs.readFileSync(DB_FILE, 'utf-8'), + JSON.stringify({key: 'red', 'val': 'lightning Bolt'})+'\n'+ + JSON.stringify({key: 'black', 'val': 'dark ritual'})+'\n'+ + JSON.stringify({key: 'blue', 'val': 'ancestral recall'})+'\n'+ + JSON.stringify({key: 'white', 'val': 'healing salve'})+'\n'+ + JSON.stringify({key: 'green', 'val': 'giant growth'})+'\n' + ); + }) + }) +}) \ No newline at end of file From ec17cc61bff8b937ba06f473f803e5ae0a4dac20 Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Wed, 20 Apr 2011 17:26:07 +0530 Subject: [PATCH 09/28] adding custom indexes --- README.md | 27 +++++++++ lib/dirty/dirty.js | 79 ++++++++++++++++++++------ test/simple/test-dirty.js | 107 +++++++++++++++++++++++++++--------- test/system/test-compact.js | 13 +++-- test/system/test-indexes.js | 37 +++++++++++++ 5 files changed, 217 insertions(+), 46 deletions(-) create mode 100644 test/system/test-indexes.js diff --git a/README.md b/README.md index dcd3baf..ab6c1d8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +#EDITS + +This is a fork of node-dirty with the following abilities added. + +* A Compacting function +* Custom Indexes + # node-dirty ## Purpose @@ -103,6 +110,26 @@ key once, even if it had been overwritten. Emitted whenever all records have been written to disk. +### dirty.compact() + +Compacts the database and gets rid of all redundant rows. It creates a new file to write to and then overwrites the existing file if it successfully writes it out. Use this when your system has some idle cycles. This should make load much faster. + +### dirty.addCompactingFilter(filter) + +Used while compacting the database and gets RID of any row that's matched by the filter (filter returns true). This is useful if you want to use the compacting run to clean up the database of stale data (like old database sessions). These filters are not persisted. You have to re-add them everytime the app starts. +filter is function(key, value); + +### dirty event: 'compacted' + +Emitted once compacting is complete if you start a compact run and succeeds. + +### dirty event: 'compactingError' + +Emitted once compacting is complete if you start a compact run and it fails. When this happens the in memory store will be inconsistent with the database on file. The memory store will no longer contain any rows that were filtered out by the compacting filter. But these rows will still be in the database. Ideally, since the filters are for removing stale rows that aren't harmful, this shouldn't matter. + + + + ## License node-dirty is licensed under the MIT license. diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index 95beaa3..6a6d510 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -17,6 +17,7 @@ var Dirty = exports.Dirty = function(path) { this._readStream = null; this._writeStream = null; this._compactingFilters = []; + this._indexFns = {}; this._load(); }; @@ -26,19 +27,23 @@ Dirty.Dirty = Dirty; module.exports = Dirty; Dirty.prototype.set = function(key, val, cb) { - if (val === undefined) { - delete this._docs[key]; - } else { - this._docs[key] = val; - } + this._updateDocs(key, val); + if (!cb) { + this._queue.push(key); + } else { + this._queue.push([key, cb]); + } - if (!cb) { - this._queue.push(key); - } else { - this._queue.push([key, cb]); - } + this._maybeFlush(); +}; - this._maybeFlush(); +Dirty.prototype._updateDocs = function(key, val) { + this._updateIndexes(key, val); + if (val === undefined) { + delete this._docs[key]; + } else { + this._docs[key] = val; + } }; Dirty.prototype.get = function(key) { @@ -89,18 +94,17 @@ Dirty.prototype._load = function() { self.emit('error', new Error('Could not load corrupted row: '+rowStr)); return ''; } - + if (row.val === undefined) { if (row.key in self._docs) { length--; } - delete self._docs[row.key]; } else { if (!(row.key in self._docs)) { length++; } - self._docs[row.key] = row.val; } + self._updateDocs(row.key, row.val); return ''; }); }) @@ -253,9 +257,12 @@ Dirty.prototype._writeCompactedData = function(ws) { } for (var k in this._docs) { var doc = this._docs[k]; - if (this._compactingFilters.every(function(filterFn){ + if (this._compactingFilters.some(function(filterFn){ return filterFn(k, doc); - })) continue; + })) { + this._updateDocs(k, undefined); + continue; + } bundleStr += JSON.stringify({key: k, val: doc})+'\n'; bundleLength++; if (bundleLength >= this.writeBundle) { @@ -267,4 +274,42 @@ Dirty.prototype._writeCompactedData = function(ws) { Dirty.prototype.addCompactingFilter = function(filter) { this._compactingFilters.push(filter); -} +}; + +Dirty.prototype.addIndex = function(name, indexFn) { + this._indexFns[name] = {indexFn: indexFn, keys: {}}; +}; + +Dirty.prototype._updateIndexes = function(key, newVal) { + for (idxKey in this._indexFns) { + var indexFn = this._indexFns[idxKey].indexFn; + var keys = this._indexFns[idxKey].keys; + if (this._docs[key]) { + var oldIdxKey = indexFn(key, this._docs[key]); + } + if (newVal) { + var newIdxKey = indexFn(key, newVal); + }; + if ((oldIdxKey) && (oldIdxKey === newIdxKey)) { + continue; + }; + if (oldIdxKey) { + oldCol = keys[oldIdxKey]; + oldCol.splice(oldCol.indexOf(key), 1); + if (oldCol.length == 0) delete keys[oldIdxKey]; + } + if (newIdxKey) { + if (!keys[newIdxKey]) keys[newIdxKey] = []; + keys[newIdxKey].push(key); + } + }; +}; + +Dirty.prototype.find = function(name, value) { + var self = this; + var validKeys = this._indexFns[name].keys[value]; + return !validKeys ? [] : + this._indexFns[name].keys[value].map(function(k){ + return {key: k, val: self._docs[k]}; + }); +}; diff --git a/test/simple/test-dirty.js b/test/simple/test-dirty.js index 21ed791..21b4605 100644 --- a/test/simple/test-dirty.js +++ b/test/simple/test-dirty.js @@ -319,6 +319,36 @@ test(function _flush() { assert.deepEqual(dirty._queue, []); }); + +test(function rm() { + var KEY = 'foo', CB = function() {}; + gently.expect(dirty, 'set', function (key, val, cb) { + assert.strictEqual(key, KEY); + assert.strictEqual(val, undefined); + assert.strictEqual(cb, CB); + }); + dirty.rm(KEY, CB); +}); + +test(function forEach() { + for (var i = 1; i <= 4; i++) { + dirty.set(i, {}); + }; + + var i = 0; + dirty.forEach(function(key, doc) { + i++; + assert.equal(key, i); + assert.strictEqual(doc, dirty._docs[i]); + + if (i == 3) { + return false; + } + }); + + assert.equal(i, 3); +}); + test(function compactPath(){ (function testIsPathAppendedWithCompact(){ dirty.path = 'foo/bar.baz'; @@ -519,31 +549,58 @@ test(function _onCompactingErrorResetsTheStateToBeforeCompacting(){ }); }); -test(function rm() { - var KEY = 'foo', CB = function() {}; - gently.expect(dirty, 'set', function (key, val, cb) { - assert.strictEqual(key, KEY); - assert.strictEqual(val, undefined); - assert.strictEqual(cb, CB); - }); - dirty.rm(KEY, CB); -}); - -test(function forEach() { - for (var i = 1; i <= 4; i++) { - dirty.set(i, {}); - }; - - var i = 0; - dirty.forEach(function(key, doc) { - i++; - assert.equal(key, i); - assert.strictEqual(doc, dirty._docs[i]); +test(function Indexes(){ + (function setup() { + dirty.addIndex('race', function(k,v){ + return v.race; + }); + dirty.addIndex('damageType', function(k,v){ + return v.damageType; + }); + })(); + + var eva = {race: 'cra', damageType: 'ranged', sex: 'F'}; + var ama = {race: 'sadida', damageType: 'ranged', sex: 'F'}; + var yug = {race: 'eliatrope', set: 'M'}; - if (i == 3) { - return false; - } - }); + (function testSearchingForANonExistingElementReturnsNothing(){ + assert.deepEqual([], dirty.find('race', 'cra')); + })(); + + (function testAddingASingleElementMakesItSearchableByIndex(){ + dirty.set('Evangelyne', eva); + assert.deepEqual([{key: 'Evangelyne', val: eva}], dirty.find('race', 'cra')); + })(); + + (function testSearchingForAnItemWhichMatchesNOIndexesReturnsNothing(){ + assert.deepEqual([], dirty.find('race', 'sadida')); + })(); + + (function testSearchingForAnItemThatIsDeletedReturnsNothing(){ + dirty.rm('Evangelyne'); + assert.deepEqual([], dirty.find('race', 'cra')); + })(); + + (function testSearchingForSomethingThatMatchesMoreThanOneItemReturnsThemAll(){ + dirty.set('Evangelyne', eva); + dirty.set('Amalia', eva); + assert.deepEqual([{key: 'Evangelyne', val: eva}, {key: 'Amalia', val: eva}], dirty.find('race', 'cra')); + assert.deepEqual([{key: 'Evangelyne', val: eva}, {key: 'Amalia', val: eva}], dirty.find('damageType', 'ranged')); + })(); + + (function testSearchingAfterModificationReturnsTheSameItemIfTheIndexValueHasntChanged(){ + dirty.set('Amalia', ama); + assert.deepEqual([{key: 'Evangelyne', val: eva}, {key: 'Amalia', val: ama}], dirty.find('damageType', 'ranged')); + })(); - assert.equal(i, 3); + (function testSearchingAfterModificationReturnsUnderTheNewIndex(){ + assert.deepEqual([{key: 'Evangelyne', val: eva}], dirty.find('race', 'cra')); + assert.deepEqual([{key: 'Amalia', val: ama}], dirty.find('race', 'sadida')); + })(); + + (function testAnItemThatIsUndefinedByAnIndexFunctionIsNotIndexed(){ + dirty.set('Yugo', yug); + assert.deepEqual({'ranged': ['Evangelyne', 'Amalia']}, dirty._indexFns['damageType'].keys) + })(); }); + diff --git a/test/system/test-compact.js b/test/system/test-compact.js index 4841432..6d8a0a7 100644 --- a/test/system/test-compact.js +++ b/test/system/test-compact.js @@ -7,6 +7,10 @@ db.addCompactingFilter(function(key, val){ return /magic/.test(val); }); +db.addCompactingFilter(function(key, val){ + return key == 'flan'; +}); + db.on('load', function(){ db.set('red', 'lightning Bolt'); db.set('black', 'dark ritual'); @@ -14,6 +18,7 @@ db.on('load', function(){ db.set('white', 'healing salve'); db.set('green', 'llanowar Elves'); db.set('purple', 'some magic'); + db.set('flan', 'is good'); db.on('drain', function(){ db.set('green', 'giant growth'); db.compact(); @@ -24,8 +29,8 @@ db.on('load', function(){ JSON.stringify({key: 'black', 'val': 'dark ritual'})+'\n'+ JSON.stringify({key: 'blue', 'val': 'ancestral recall'})+'\n'+ JSON.stringify({key: 'white', 'val': 'healing salve'})+'\n'+ - JSON.stringify({key: 'green', 'val': 'giant growth'})+'\n' + JSON.stringify({key: 'green', 'val': 'giant growth'})+'\n' ); - }) - }) -}) \ No newline at end of file + }); + }); +}); \ No newline at end of file diff --git a/test/system/test-indexes.js b/test/system/test-indexes.js new file mode 100644 index 0000000..90a6f60 --- /dev/null +++ b/test/system/test-indexes.js @@ -0,0 +1,37 @@ +require('../common'); +var DB_FILE = TEST_TMP+'/load.dirty'; +db = require('dirty')(DB_FILE), +fs = require('fs'), +loaded = false; + +db.addIndex('start', function(k,v){ + return v[0]; +}); +db.addIndex('end', function(k, v) { + return v.slice(-1); +}); + +db.addCompactingFilter(function(k,v){ + return !(/\w1/.test(k)); +}); + + +db.on('load', function(){ + db.set("e1", 'hello'); + db.set("g1", 'buten tag'); + db.set("e2", 'hi'); + db.set("m1", 'salamat peg'); + db.set("j1", 'konnichiwa'); + assert.deepEqual([], db.find('start', 'g')); + assert.deepEqual([{key: 'e1', val: 'hello'},{key: 'e2', val: 'hi'}], db.find('start', 'h')); + db.on('drain', function() { + db.rm("j1"); + db.set("g1", 'guten tag'); + assert.deepEqual([{key: 'g1', val: 'guten tag'},{key: 'm1', val: 'salamat peg'}], db.find('end','g')); + assert.deepEqual([], db.find('start', 'j')); + db.compact() + db.on('compacted', function(){ + assert.deepEqual([{key: 'e1', val: 'hello'}], db.find('start', 'h')); + }); + }); +}); From 164ba1e7fbbd5003807e6636998bc3746c070f54 Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Thu, 21 Apr 2011 19:53:44 +0530 Subject: [PATCH 10/28] minor refactoring of indexs --- lib/dirty/dirty.js | 66 +++++++++++++++++++++++---------------- test/simple/test-dirty.js | 5 --- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index 6a6d510..f8450a7 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -276,40 +276,52 @@ Dirty.prototype.addCompactingFilter = function(filter) { this._compactingFilters.push(filter); }; -Dirty.prototype.addIndex = function(name, indexFn) { - this._indexFns[name] = {indexFn: indexFn, keys: {}}; +Dirty.prototype.addIndex = function(index, indexFn) { + this._indexFns[index] = {indexFn: indexFn, keyMap: {}}; }; -Dirty.prototype._updateIndexes = function(key, newVal) { - for (idxKey in this._indexFns) { - var indexFn = this._indexFns[idxKey].indexFn; - var keys = this._indexFns[idxKey].keys; - if (this._docs[key]) { - var oldIdxKey = indexFn(key, this._docs[key]); +Dirty.prototype._deleteKeyFromIndexedKeys = function(keyMap, indexValue, key) { + var keys = keyMap[indexValue]; + keys.splice(keys.indexOf(key), 1); + if (keys.length === 0) delete keyMap[indexValue]; +}; + +Dirty.prototype._addKeyToIndexedKeys = function(keyMap, indexValue, key) { + var keys = keyMap[indexValue] || []; + keys.push(key); + keyMap[indexValue] = keys; +}; + +Dirty.prototype._updateIndex = function(index, key, newVal) { + var indexFn = this._indexFns[index].indexFn; + var keyMap = this._indexFns[index].keyMap; + if (key in this._docs) { + var oldIndexValue = indexFn(key, this._docs[key]); + if (newVal != undefined) { + var newIndexValue = indexFn(key, newVal); + if (oldIndexValue === newIndexValue) return; + this._deleteKeyFromIndexedKeys(keyMap, oldIndexValue, key); + this._addKeyToIndexedKeys(keyMap, newIndexValue, key); + } else this._deleteKeyFromIndexedKeys(keyMap, oldIndexValue, key); + } else { + if (newVal != undefined) { + var newIndexValue = indexFn(key, newVal); + this._addKeyToIndexedKeys(keyMap, newIndexValue, key); } - if (newVal) { - var newIdxKey = indexFn(key, newVal); - }; - if ((oldIdxKey) && (oldIdxKey === newIdxKey)) { - continue; - }; - if (oldIdxKey) { - oldCol = keys[oldIdxKey]; - oldCol.splice(oldCol.indexOf(key), 1); - if (oldCol.length == 0) delete keys[oldIdxKey]; - } - if (newIdxKey) { - if (!keys[newIdxKey]) keys[newIdxKey] = []; - keys[newIdxKey].push(key); - } + } +}; + +Dirty.prototype._updateIndexes = function(key, newVal) { + for (index in this._indexFns) { + this._updateIndex(index, key, newVal); }; }; -Dirty.prototype.find = function(name, value) { +Dirty.prototype.find = function(index, value) { var self = this; - var validKeys = this._indexFns[name].keys[value]; + var validKeys = this._indexFns[index].keyMap[value]; return !validKeys ? [] : - this._indexFns[name].keys[value].map(function(k){ - return {key: k, val: self._docs[k]}; + validKeys.map(function(k){ + return {key: k, val: self._docs[k]}; }); }; diff --git a/test/simple/test-dirty.js b/test/simple/test-dirty.js index 21b4605..a09bf0d 100644 --- a/test/simple/test-dirty.js +++ b/test/simple/test-dirty.js @@ -597,10 +597,5 @@ test(function Indexes(){ assert.deepEqual([{key: 'Evangelyne', val: eva}], dirty.find('race', 'cra')); assert.deepEqual([{key: 'Amalia', val: ama}], dirty.find('race', 'sadida')); })(); - - (function testAnItemThatIsUndefinedByAnIndexFunctionIsNotIndexed(){ - dirty.set('Yugo', yug); - assert.deepEqual({'ranged': ['Evangelyne', 'Amalia']}, dirty._indexFns['damageType'].keys) - })(); }); From ee5205f270fe3fc8f69fedb6b2c6a0e9d5b67456 Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Thu, 21 Apr 2011 19:59:55 +0530 Subject: [PATCH 11/28] documentation for indexes --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index ab6c1d8..ded068f 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,20 @@ Emitted once compacting is complete if you start a compact run and succeeds. Emitted once compacting is complete if you start a compact run and it fails. When this happens the in memory store will be inconsistent with the database on file. The memory store will no longer contain any rows that were filtered out by the compacting filter. But these rows will still be in the database. Ideally, since the filters are for removing stale rows that aren't harmful, this shouldn't matter. +### dirty.addIndex(index, indexFn) + +Use this to add an index named index. indexFn is a function(key, val) that returns the indexed value. For example + + dirty.addIndex('eyeColor', function(k, v){ + return v.eyes; + }); + +You can add as many indexes as you want, but beware this adds to every add/delete/update operation an O(k) operation where k is the number of values which match a given index. If your index is not well distributed, with large databases you might face an issue. Don't worry about this most of the time. + + +### dirty.find(index, value) + +This returns all documents with the given value for the index. From 93c8d08b6425c69d36bdc2d780514b674bdf8ac5 Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Fri, 22 Apr 2011 00:32:33 +0530 Subject: [PATCH 12/28] added length and redundantLength --- lib/dirty/dirty.js | 40 ++++++++++++-------- test/simple/test-dirty.js | 74 +++++++++++++++++++++++++++++++++++++ test/system/test-compact.js | 15 +++++++- 3 files changed, 111 insertions(+), 18 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index f8450a7..b1a3393 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -18,7 +18,8 @@ var Dirty = exports.Dirty = function(path) { this._writeStream = null; this._compactingFilters = []; this._indexFns = {}; - + this._length = 0; + this._redundantLength = 0; this._load(); }; @@ -33,15 +34,20 @@ Dirty.prototype.set = function(key, val, cb) { } else { this._queue.push([key, cb]); } - this._maybeFlush(); }; -Dirty.prototype._updateDocs = function(key, val) { +Dirty.prototype._updateDocs = function(key, val, skipRedundantRows) { this._updateIndexes(key, val); + if (key in this._docs) { + this._length--; + if (!skipRedundantRows) this._redundantLength++; + } if (val === undefined) { + if (!skipRedundantRows) this._redundantLength++; delete this._docs[key]; } else { + this._length++; this._docs[key] = val; } }; @@ -66,8 +72,7 @@ Dirty.prototype._load = function() { if (!this.path) { return; } - - var self = this, buffer = '', length = 0; + var self = this, buffer = ''; this._readStream = fs.createReadStream(this.path, { encoding: 'utf-8', flags: 'r' @@ -95,15 +100,6 @@ Dirty.prototype._load = function() { return ''; } - if (row.val === undefined) { - if (row.key in self._docs) { - length--; - } - } else { - if (!(row.key in self._docs)) { - length++; - } - } self._updateDocs(row.key, row.val); return ''; }); @@ -112,7 +108,7 @@ Dirty.prototype._load = function() { if (buffer.length) { self.emit('error', new Error('Corrupted row at the end of the db: '+buffer)); } - self.emit('load', length); + self.emit('load', self._length); }); this._recreateWriteStream(); }; @@ -192,6 +188,14 @@ Dirty.prototype.__defineGetter__("_compactPath", function() { return this.path + ".compact"; }); +Dirty.prototype.__defineGetter__('length', function(){ + return this._length; +}); + +Dirty.prototype.__defineGetter__('redundantLength', function(){ + return this._redundantLength; +}); + Dirty.prototype.compact = function(cb) { if (this.compacting) return; var self = this; @@ -210,6 +214,8 @@ Dirty.prototype._startCompacting = function() { var self = this; this._queueBackup = this._queue; this._queue = []; + this._redundantLengthBackup = this._redundantLength; + this._redundantLength = 0; var ws = fs.createWriteStream(this._compactPath, { encoding: 'utf-8', flags: 'w' @@ -238,11 +244,13 @@ Dirty.prototype.on('compacted', function(){ Dirty.prototype.on('compactingError', function(){ this._queue = this._queueBackup.concat(this._queue); + this._redundantLength += this._redundantLengthBackup; this._endCompacting(); }); Dirty.prototype._endCompacting = function() { this._queueBackup = []; + this._redundantLengthBackup = 0; this.compacting = false; this._maybeFlush(); }; @@ -260,7 +268,7 @@ Dirty.prototype._writeCompactedData = function(ws) { if (this._compactingFilters.some(function(filterFn){ return filterFn(k, doc); })) { - this._updateDocs(k, undefined); + this._updateDocs(k, undefined, true); continue; } bundleStr += JSON.stringify({key: k, val: doc})+'\n'; diff --git a/test/simple/test-dirty.js b/test/simple/test-dirty.js index a09bf0d..b89c7c7 100644 --- a/test/simple/test-dirty.js +++ b/test/simple/test-dirty.js @@ -599,3 +599,77 @@ test(function Indexes(){ })(); }); +test(function length(){ + (function setup(){ + dirty.forEach(function(k,v){ + dirty.rm(k); + }) + assert.equal(0, dirty.length); + })(); + + (function testAddingAnItemIncreasesLength(){ + dirty.set('steam',['fire', 'ice']); + assert.equal(1, dirty.length); + })(); + + (function testModifyingAnItemDoesntChangeLength(){ + dirty.set('steam', ['fire', 'water']); + assert.equal(1, dirty.length); + })(); + + (function testRemovingAnItemReducesLength(){ + dirty.rm('steam'); + assert.equal(0, dirty.length); + })(); + + (function testReAddingADeletedItemIncreasesLength(){ + dirty.set('steam', ['fire', 'water']); + assert.equal(1, dirty.length); + })(); + + (function testReAddingAnExistingItemDoesNotIncreaseLength(){ + dirty.set('steam', ['fire', 'water']); + assert.equal(1, dirty.length); + })(); +}) + +test(function redundantLength(){ + var original_length; + (function setup(){ + dirty.forEach(function(k,v){ + dirty.rm(k); + }) + original_length = dirty.redundantLength; + })(); + + (function testAddingAnItemDoesNotIncreaseRedundantLength(){ + dirty.set('arcane','spread'); + assert.equal(original_length, dirty.redundantLength); + })(); + + (function testReAddingAnExistingItemIncreasesRedundantLength(){ + dirty.set('arcane','spread'); + assert.equal(original_length + 1, dirty.redundantLength); + })(); + + (function testModifyingAnItemIncreasesRedundantLength(){ + dirty.set('arcane', 'beam'); + assert.equal(original_length + 2, dirty.redundantLength); + })(); + + (function testRemovingAnExistingItemIncreasesRedundantLengthByTwo(){ + dirty.rm('arcane'); + assert.equal(original_length + 4, dirty.redundantLength); + })(); + + (function testRemovingANonExistingItemIncreasesRedundantLength(){ + dirty.rm('arcane'); + assert.equal(original_length + 5, dirty.redundantLength); + })(); + + (function testReAddingADeletedItemDoesNotIncreasRedundantLength(){ + dirty.set('arcane', 'beam'); + assert.equal(original_length + 5, dirty.redundantLength); + })(); +}) + diff --git a/test/system/test-compact.js b/test/system/test-compact.js index 6d8a0a7..da34007 100644 --- a/test/system/test-compact.js +++ b/test/system/test-compact.js @@ -19,18 +19,29 @@ db.on('load', function(){ db.set('green', 'llanowar Elves'); db.set('purple', 'some magic'); db.set('flan', 'is good'); + assert.equal(7, db.length); + assert.equal(0, db.redundantLength); db.on('drain', function(){ + db.set('green', 'llanowar Elves'); db.set('green', 'giant growth'); + db.rm('red'); + db.rm('red'); + assert.equal(5, db.redundantLength); + db.set('red', 'lightning bolt') + assert.equal(7, db.length); + assert.equal(5, db.redundantLength); db.compact(); db.on('compacted', function(){ assert.strictEqual( fs.readFileSync(DB_FILE, 'utf-8'), - JSON.stringify({key: 'red', 'val': 'lightning Bolt'})+'\n'+ JSON.stringify({key: 'black', 'val': 'dark ritual'})+'\n'+ JSON.stringify({key: 'blue', 'val': 'ancestral recall'})+'\n'+ JSON.stringify({key: 'white', 'val': 'healing salve'})+'\n'+ - JSON.stringify({key: 'green', 'val': 'giant growth'})+'\n' + JSON.stringify({key: 'green', 'val': 'giant growth'})+'\n'+ + JSON.stringify({key: 'red', 'val': 'lightning bolt'})+'\n' ); + assert.equal(5, db.length); + assert.equal(0, db.redundantLength); }); }); }); \ No newline at end of file From 0e6fad495d25b8b3fa99395db23dd8ad72fc0037 Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Fri, 22 Apr 2011 00:32:55 +0530 Subject: [PATCH 13/28] documentation change --- README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ded068f..183adeb 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,15 @@ You can add as many indexes as you want, but beware this adds to every add/delet ### dirty.find(index, value) -This returns all documents with the given value for the index. +This returns all documents with the given value for the index. + +### dirty.length + +This is a count of the number of documents. If compacting fails, this can become incorrect (since it will not be aware of filtered rows. It will reflect the memory store not the disk store.) + +### dirty.redundantLength + +This is a count of the number of redundant rows. You can use this to decide when to compact.˝ From f5169c4fd1266552caf51d66af7240cae74b741f Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Fri, 22 Apr 2011 02:49:27 +0530 Subject: [PATCH 14/28] fixed bug which prevented multiple instances of Dirty --- lib/dirty/dirty.js | 23 +++++++------- test/system/test-compact.js | 61 ++++++++++++++++++++----------------- 2 files changed, 44 insertions(+), 40 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index b1a3393..0ec7e33 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -3,7 +3,7 @@ if (global.GENTLY) require = GENTLY.hijack(require); var fs = require('fs'), sys = require('sys'), EventEmitter = require('events').EventEmitter; - + var Dirty = exports.Dirty = function(path) { if (!(this instanceof Dirty)) return new Dirty(path); @@ -21,12 +21,21 @@ var Dirty = exports.Dirty = function(path) { this._length = 0; this._redundantLength = 0; this._load(); + var self = this; + this.on('compacted', function(){ + self._endCompacting(); + }); + this.on('compactingError', function(){ + self._queue = self._queueBackup.concat(self._queue); + self._redundantLength += self._redundantLengthBackup; + self._endCompacting(); + }); }; sys.inherits(Dirty, EventEmitter); Dirty.Dirty = Dirty; module.exports = Dirty; - + Dirty.prototype.set = function(key, val, cb) { this._updateDocs(key, val); if (!cb) { @@ -238,16 +247,6 @@ Dirty.prototype._moveCompactedDataOverOriginal = function() { }); } -Dirty.prototype.on('compacted', function(){ - this._endCompacting(); -}); - -Dirty.prototype.on('compactingError', function(){ - this._queue = this._queueBackup.concat(this._queue); - this._redundantLength += this._redundantLengthBackup; - this._endCompacting(); -}); - Dirty.prototype._endCompacting = function() { this._queueBackup = []; this._redundantLengthBackup = 0; diff --git a/test/system/test-compact.js b/test/system/test-compact.js index da34007..6f6362d 100644 --- a/test/system/test-compact.js +++ b/test/system/test-compact.js @@ -3,14 +3,6 @@ var DB_FILE = TEST_TMP+'/flush.dirty'; db = require('dirty')(DB_FILE), fs = require('fs'); -db.addCompactingFilter(function(key, val){ - return /magic/.test(val); -}); - -db.addCompactingFilter(function(key, val){ - return key == 'flan'; -}); - db.on('load', function(){ db.set('red', 'lightning Bolt'); db.set('black', 'dark ritual'); @@ -22,26 +14,39 @@ db.on('load', function(){ assert.equal(7, db.length); assert.equal(0, db.redundantLength); db.on('drain', function(){ - db.set('green', 'llanowar Elves'); - db.set('green', 'giant growth'); - db.rm('red'); - db.rm('red'); - assert.equal(5, db.redundantLength); - db.set('red', 'lightning bolt') - assert.equal(7, db.length); - assert.equal(5, db.redundantLength); - db.compact(); - db.on('compacted', function(){ - assert.strictEqual( - fs.readFileSync(DB_FILE, 'utf-8'), - JSON.stringify({key: 'black', 'val': 'dark ritual'})+'\n'+ - JSON.stringify({key: 'blue', 'val': 'ancestral recall'})+'\n'+ - JSON.stringify({key: 'white', 'val': 'healing salve'})+'\n'+ - JSON.stringify({key: 'green', 'val': 'giant growth'})+'\n'+ - JSON.stringify({key: 'red', 'val': 'lightning bolt'})+'\n' - ); - assert.equal(5, db.length); - assert.equal(0, db.redundantLength); + var db2 = require('dirty')(DB_FILE); + + db2.addCompactingFilter(function(key, val){ + return /magic/.test(val); }); + + db2.addCompactingFilter(function(key, val){ + return key == 'flan'; + }); + + + db2.on('load', function(){ + db2.set('green', 'llanowar Elves'); + db2.set('green', 'giant growth'); + db2.rm('red'); + db2.rm('red'); + assert.equal(5, db2.redundantLength); + db2.set('red', 'lightning bolt') + assert.equal(7, db2.length); + assert.equal(5, db2.redundantLength); + db2.compact(); + db2.on('compacted', function(){ + assert.strictEqual( + fs.readFileSync(DB_FILE, 'utf-8'), + JSON.stringify({key: 'black', 'val': 'dark ritual'})+'\n'+ + JSON.stringify({key: 'blue', 'val': 'ancestral recall'})+'\n'+ + JSON.stringify({key: 'white', 'val': 'healing salve'})+'\n'+ + JSON.stringify({key: 'green', 'val': 'giant growth'})+'\n'+ + JSON.stringify({key: 'red', 'val': 'lightning bolt'})+'\n' + ); + assert.equal(5, db2.length); + assert.equal(0, db2.redundantLength); + }); + }) }); }); \ No newline at end of file From e50883707d8d1054e3b862efd462ccd92fa8e06f Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Mon, 25 Apr 2011 22:17:57 +0530 Subject: [PATCH 15/28] changed get to return a clone --- lib/dirty/dirty.js | 17 ++++++++++++++++- test/simple/test-dirty.js | 2 +- test/system/test-for-each.js | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index 0ec7e33..75ba83c 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -62,7 +62,22 @@ Dirty.prototype._updateDocs = function(key, val, skipRedundantRows) { }; Dirty.prototype.get = function(key) { - return this._docs[key]; + return this._clone(this._docs[key]); +}; + +Dirty.prototype._clone = function(obj) { + if (Object.prototype.toString.call(obj) === '[object Array]') { + return obj.slice(); + } + + if (Object.prototype.toString.call(obj) !== '[object Object]') { + return obj; + } + var retval = {}; + for (k in obj) { + retval[k] = obj[k]; + } + return retval; }; Dirty.prototype.rm = function(key, cb) { diff --git a/test/simple/test-dirty.js b/test/simple/test-dirty.js index b89c7c7..4e96a8c 100644 --- a/test/simple/test-dirty.js +++ b/test/simple/test-dirty.js @@ -199,7 +199,7 @@ test(function get() { var KEY = 'example', VAL = {}; dirty._docs[KEY] = VAL; - assert.strictEqual(dirty.get(KEY), VAL); + assert.deepEqual(dirty.get(KEY), VAL); }); test(function set() { diff --git a/test/system/test-for-each.js b/test/system/test-for-each.js index 7212e31..26f0f9c 100644 --- a/test/system/test-for-each.js +++ b/test/system/test-for-each.js @@ -9,7 +9,7 @@ var i = 0; db.forEach(function(key, doc) { i++; assert.equal(key, i); - assert.strictEqual(doc, db.get(key)); + assert.deepEqual(doc, db.get(key)); }); assert.equal(i, 3); From 34c985deded70118cb7a769665189e890cff6e9c Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Mon, 25 Apr 2011 22:32:54 +0530 Subject: [PATCH 16/28] tests to keep clone --- test/simple/test-dirty.js | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/test/simple/test-dirty.js b/test/simple/test-dirty.js index 4e96a8c..d075d4a 100644 --- a/test/simple/test-dirty.js +++ b/test/simple/test-dirty.js @@ -200,6 +200,30 @@ test(function get() { dirty._docs[KEY] = VAL; assert.deepEqual(dirty.get(KEY), VAL); + + (function testReturnsNumbersAsTheyAre(){ + dirty._docs[KEY] = 5; + assert.strictEqual(5, dirty.get(KEY)); + })(); + + (function testReturnsStringsAsTheyAre(){ + dirty._docs[KEY] = "theont"; + assert.strictEqual("theont", dirty.get(KEY)); + })(); + + (function testReturnsACloneOfArrays(){ + VAL = [1,2,3]; + dirty._docs[KEY] = VAL; + assert.notStrictEqual(VAL, dirty.get(KEY)); + assert.deepEqual(VAL, dirty.get(KEY)); + })(); + + (function testReturnsACloneOfObjects(){ + VAL = {foo: 'bar'}; + dirty._docs[KEY] = VAL; + assert.notStrictEqual(VAL, dirty.get(KEY)); + assert.deepEqual(VAL, dirty.get(KEY)); + })(); }); test(function set() { From 65af575ad24be4d5902c1e6888329ff0bc6ff967 Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Sun, 18 Sep 2011 17:26:31 +0530 Subject: [PATCH 17/28] added a function to find all the values a particular index has --- lib/dirty/dirty.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index 75ba83c..8ec5d64 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -347,3 +347,7 @@ Dirty.prototype.find = function(index, value) { return {key: k, val: self._docs[k]}; }); }; + +Dirty.prototype.indexValues = function(index) { + return _(this._indexFns[index].keyMap).keys(); +}; \ No newline at end of file From efe891a742de42fe913c09d9f37cdbd468fdc485 Mon Sep 17 00:00:00 2001 From: Vishnu Iyengar Date: Sun, 18 Sep 2011 17:37:21 +0530 Subject: [PATCH 18/28] bumping version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 20cb8dc..8045f26 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name" : "dirty" , "description" : "A tiny & fast key value store with append-only disk log. Ideal for apps with < 1 million records." -, "version": "0.9.1" +, "version": "0.9.2" , "dependencies": {"gently": ">=0.8.0"} , "main" : "./lib/dirty" } From ca53e83f622a7dc6f1c9ab3d5fff03370d4c1197 Mon Sep 17 00:00:00 2001 From: Vishnu S Iyengar Date: Sun, 12 May 2013 21:07:41 -0700 Subject: [PATCH 19/28] updated dirty to the lastest version from the original repo --- lib/dirty/dirty.js | 69 +++++++++++------- test/system/test-compact.js | 52 ------------- test/system/test-indexes.js | 37 ---------- test/test-api.js | 31 ++++++++ test/test-compact.js | 142 ++++++++++++++++++++++++++++++++++++ test/test-indexes.js | 73 ++++++++++++++++++ 6 files changed, 290 insertions(+), 114 deletions(-) delete mode 100644 test/system/test-compact.js delete mode 100644 test/system/test-indexes.js create mode 100644 test/test-compact.js create mode 100644 test/test-indexes.js diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index 8e7122c..8a3bdd2 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -214,7 +214,6 @@ Dirty.prototype._flush = function() { bundleStr = '', key, cbs = []; - this.flushing = true; function callbacks(err, cbs) { @@ -286,14 +285,13 @@ Dirty.prototype.__defineGetter__('redundantLength', function(){ Dirty.prototype.compact = function(cb) { if (this.compacting) return; var self = this; - this.compacting = true; - if (this.flushing) { - this._writeStream.once('drain', function(){ - self._startCompacting(); - }); + this.once('drain', function(){ + this.compact(cb); + }); } else { - this._startCompacting(); + this.compacting = true; + this._startCompacting(); } }; @@ -310,9 +308,6 @@ Dirty.prototype._startCompacting = function() { ws.on("error", function(){ self.emit('compactingError'); }); - ws.on('drain', function(){ - self._moveCompactedDataOverOriginal(); - }); this._writeCompactedData(ws); }; @@ -333,27 +328,51 @@ Dirty.prototype._endCompacting = function() { }; Dirty.prototype._writeCompactedData = function(ws) { - var bundleLength = 0; - var bundleStr = ''; + var keys = []; + var self = this; + for (var k in this._docs) { keys.push(k) }; + var writeToStream = function() { - ws.write(bundleStr); - bundleStr = ''; - bundleLength = 0; + if (keys.length === 0) { + ws.once('finish', function(){ + self._writeStream.once('finish', function(){ + self._moveCompactedDataOverOriginal(); + }); + self._writeStream.end(); + }) + ws.end(); + return; + } + var bundleStr = buildBundle(); + var isDrained = ws.write(bundleStr); + if (isDrained) { + process.nextTick(writeToStream); + } else { + ws.once('drain', writeToStream); + } } - for (var k in this._docs) { - var doc = this._docs[k]; - if (this._compactingFilters.some(function(filterFn){ - return filterFn(k, doc); + + var buildBundle = function() { + var bundleLength = 0, + bundleStr = ''; + for (var i=0; i< keys.length; i++) { + var doc = self._docs[keys[i]]; + if (self._compactingFilters.some(function(filterFn){ + return filterFn(keys[i], doc); })) { - this._updateDocs(k, undefined, true); + self._updateDocs(keys[i], undefined, true); continue; } - bundleStr += JSON.stringify({key: k, val: doc})+'\n'; + bundleStr += JSON.stringify({key: keys[i], val: doc})+'\n'; bundleLength++; - if (bundleLength >= this.writeBundle) { - writeToStream(); + if (bundleLength >= self.writeBundle) { + keys = keys.slice(i+1); + return bundleStr; } - }; + } + keys = []; + return bundleStr; + } writeToStream(); }; @@ -397,7 +416,7 @@ Dirty.prototype._updateIndex = function(index, key, newVal) { }; Dirty.prototype._updateIndexes = function(key, newVal) { - for (index in this._indexFns) { + for (var index in this._indexFns) { this._updateIndex(index, key, newVal); }; }; diff --git a/test/system/test-compact.js b/test/system/test-compact.js deleted file mode 100644 index 6f6362d..0000000 --- a/test/system/test-compact.js +++ /dev/null @@ -1,52 +0,0 @@ -require('../common'); -var DB_FILE = TEST_TMP+'/flush.dirty'; - db = require('dirty')(DB_FILE), - fs = require('fs'); - -db.on('load', function(){ - db.set('red', 'lightning Bolt'); - db.set('black', 'dark ritual'); - db.set('blue', 'ancestral recall'); - db.set('white', 'healing salve'); - db.set('green', 'llanowar Elves'); - db.set('purple', 'some magic'); - db.set('flan', 'is good'); - assert.equal(7, db.length); - assert.equal(0, db.redundantLength); - db.on('drain', function(){ - var db2 = require('dirty')(DB_FILE); - - db2.addCompactingFilter(function(key, val){ - return /magic/.test(val); - }); - - db2.addCompactingFilter(function(key, val){ - return key == 'flan'; - }); - - - db2.on('load', function(){ - db2.set('green', 'llanowar Elves'); - db2.set('green', 'giant growth'); - db2.rm('red'); - db2.rm('red'); - assert.equal(5, db2.redundantLength); - db2.set('red', 'lightning bolt') - assert.equal(7, db2.length); - assert.equal(5, db2.redundantLength); - db2.compact(); - db2.on('compacted', function(){ - assert.strictEqual( - fs.readFileSync(DB_FILE, 'utf-8'), - JSON.stringify({key: 'black', 'val': 'dark ritual'})+'\n'+ - JSON.stringify({key: 'blue', 'val': 'ancestral recall'})+'\n'+ - JSON.stringify({key: 'white', 'val': 'healing salve'})+'\n'+ - JSON.stringify({key: 'green', 'val': 'giant growth'})+'\n'+ - JSON.stringify({key: 'red', 'val': 'lightning bolt'})+'\n' - ); - assert.equal(5, db2.length); - assert.equal(0, db2.redundantLength); - }); - }) - }); -}); \ No newline at end of file diff --git a/test/system/test-indexes.js b/test/system/test-indexes.js deleted file mode 100644 index 90a6f60..0000000 --- a/test/system/test-indexes.js +++ /dev/null @@ -1,37 +0,0 @@ -require('../common'); -var DB_FILE = TEST_TMP+'/load.dirty'; -db = require('dirty')(DB_FILE), -fs = require('fs'), -loaded = false; - -db.addIndex('start', function(k,v){ - return v[0]; -}); -db.addIndex('end', function(k, v) { - return v.slice(-1); -}); - -db.addCompactingFilter(function(k,v){ - return !(/\w1/.test(k)); -}); - - -db.on('load', function(){ - db.set("e1", 'hello'); - db.set("g1", 'buten tag'); - db.set("e2", 'hi'); - db.set("m1", 'salamat peg'); - db.set("j1", 'konnichiwa'); - assert.deepEqual([], db.find('start', 'g')); - assert.deepEqual([{key: 'e1', val: 'hello'},{key: 'e2', val: 'hi'}], db.find('start', 'h')); - db.on('drain', function() { - db.rm("j1"); - db.set("g1", 'guten tag'); - assert.deepEqual([{key: 'g1', val: 'guten tag'},{key: 'm1', val: 'salamat peg'}], db.find('end','g')); - assert.deepEqual([], db.find('start', 'j')); - db.compact() - db.on('compacted', function(){ - assert.deepEqual([{key: 'e1', val: 'hello'}], db.find('start', 'h')); - }); - }); -}); diff --git a/test/test-api.js b/test/test-api.js index e9ea8ef..bda97c4 100644 --- a/test/test-api.js +++ b/test/test-api.js @@ -131,7 +131,38 @@ function dirtyAPITests(file) { }); + describe('clone behaviour', function(done){ + after(cleanup) + var db = dirty(file); + + it('will return numbers as they are', function(done){ + db.set('key_n', 5); + assert.strictEqual(5, db.get('key_n')); + done(); + }); + + it('will return strings as they are', function(done){ + db.set('key_s', "jello"); + assert.strictEqual("jello", db.get('key_s')); + done(); + }); + + it('will return a clone of arrays', function(done){ + var val = [1,"hello", 4]; + db.set('key_a', val); + assert.deepEqual(val, db.get('key_a')); + done(); + }); + + it('will return a clone of objects', function(done){ + var val = {foo: 'bar'}; + db.set('key_o', val); + assert.deepEqual(val, db.get('key_o')); + done(); + }); + }); }); + } dirtyAPITests(''); diff --git a/test/test-compact.js b/test/test-compact.js new file mode 100644 index 0000000..7350ef1 --- /dev/null +++ b/test/test-compact.js @@ -0,0 +1,142 @@ +var config = require('./config'), + path = require('path'), + fs = require('fs'), + dirty = require(config.LIB_DIRTY), + assert = require('assert'), + db; +var exists = (fs.exists) ? fs.exists : path.exists; +var file = config.TMP_PATH + '/compacttest.dirty'; +describe('compacting', function(){ + beforeEach(function(done){ + db = dirty(file); + db.once('load', function(){ + db.set('red', 'lightning Bolt'); + db.set('black', 'dark ritual'); + db.set('blue', 'ancestral recall'); + db.set('white', 'healing salve'); + db.set('green', 'llanowar Elves'); + db.once('drain', function(){ + done(); + }) + }); + }); + + afterEach(function (done) { + exists(file, function(doesExist) { + if (doesExist) { + fs.unlinkSync(file); + } + done(); + }); + }); + + var assertPristine = function() { + assert.strictEqual(0, db.redundantLength); + assert.strictEqual( + fs.readFileSync(file, 'utf-8'), + JSON.stringify({key: 'black', 'val': 'dark ritual'})+'\n'+ + JSON.stringify({key: 'blue', 'val': 'ancestral recall'})+'\n'+ + JSON.stringify({key: 'white', 'val': 'healing salve'})+'\n'+ + JSON.stringify({key: 'green', 'val': 'giant growth'})+'\n'+ + JSON.stringify({key: 'red', 'val': 'lightning bolt'})+'\n' + ); + } + + it('should accurately report length as number of rows', function(done){ + assert.strictEqual(5, db.length); + done(); + }); + + it('should report redundantLength as empty', function(){ + assert.strictEqual(0, db.redundantLength); + }); + + it('should accurately display redundant length', function(done){ + db.set('green', 'llanowar Elves'); + db.set('green', 'giant growth'); + db.rm('red'); + db.rm('red'); + assert.strictEqual(5, db.redundantLength); + done(); + }); + + it('should compact out redundant rows', function(done){ + db.set('green', 'llanowar Elves'); + db.set('green', 'giant growth'); + db.rm('red'); + db.set('red', 'lightning bolt') + db.once('drain', function(){ + db.compact(); + db.on('compacted', function(){ + assertPristine(); + done(); + }) + }); + }); + + it('should compact correctly with a small bundle write limit', function(done){ + db.set('green', 'giant growth'); + db.rm('red'); + db.set('red', 'lightning bolt') + db.writeBundle = 2; + db.once('drain', function(){ + db.compact(); + db.once('compacted', function(){ + assertPristine(); + done(); + }) + }); + }); + + it('should not compact while flushing', function(done){ + var didFlush = false; + db.set('green', 'giant growth'); + db.rm('red'); + db.once('drain', function(){ + assert.ok(!db.compacting); + didFlush = true; + }); + db.set('red', 'lightning bolt'); + db.compact(); + db.once('compacted', function(){ + assertPristine(); + assert.ok(didFlush); + done(); + }) + }); + + it('should not flush while compacting', function(done){ + db.compact(); + var didCompact = false; + var listener = function(){ + db._events['compacted'].splice(0,1); + assert.ok(!db.flushing); + didCompact = true; + }; + db._events['compacted'] = [listener, db._events['compacted']]; + db.once('drain', function(){ + assert.ok(didCompact); + done(); + }) + db.set('red', 'lightning bolt'); + }); + + it('should get rid of rows while compacting', function(done){ + db.addCompactingFilter(function(key, val){ + return /magic/.test(val); + }); + db.set('purple', "it's magic"); + db.set('green', 'giant growth'); + db.rm('red'); + db.set('red', 'lightning bolt') + assert.strictEqual(6, db.length); + db.on('drain', function(){ + db.on('compacted', function(){ + assertPristine(); + assert.strictEqual(5, db.length); + done(); + }); + db.compact(); + }); + }); +}); diff --git a/test/test-indexes.js b/test/test-indexes.js new file mode 100644 index 0000000..08324a6 --- /dev/null +++ b/test/test-indexes.js @@ -0,0 +1,73 @@ +var config = require('./config'), + path = require('path'), + fs = require('fs'), + dirty = require(config.LIB_DIRTY), + assert = require('assert'), + db; +var exists = (fs.exists) ? fs.exists : path.exists; +var file = config.TMP_PATH + '/compacttest.indexes'; +describe('indexes', function(){ + var eva = {race: 'cra', damageType: 'ranged', sex: 'F'}; + var ama = {race: 'sadida', damageType: 'ranged', sex: 'F'}; + var yug = {race: 'eliatrope', set: 'M'}; + + beforeEach(function(done){ + db = dirty(file); + db.once('load', function(){ + db.addIndex('race', function(k,v){ + return v.race; + }); + db.addIndex('damageType', function(k,v){ + return v.damageType; + }); + db.set('Evangeline', eva); + db.set('Amalia', ama); + db.set('Yugo', yug); + db.once('drain', function(){ + done(); + }) + }); + }); + + afterEach(function (done) { + exists(file, function(doesExist) { + if (doesExist) { + fs.unlinkSync(file); + } + done(); + }); + }); + + describe('.find', function() { + it('returns nothing when searching for a non-existing element', function(){ + assert.deepEqual([], db.find('race', 'erutrof')); + }); + + it('returns the lone element if it exists', function(){ + assert.deepEqual([{key: 'Evangeline', val: eva}], db.find('race', 'cra')); + }); + + it('returns nothing if the item is deleted', function(){ + db.rm('Evangeline'); + assert.deepEqual([], db.find('race', 'cra')); + }); + + it('returns all items that match an index value', function(){ + var ranged = db.find('damageType', 'ranged'); + assert.deepEqual([{key: 'Evangeline', val: eva}, {key: 'Amalia', val: ama}], ranged); + }); + + it('returns an item under a new index if it is modified', function(){ + var newEva = db.get('Evangeline'); + newEva.race = 'cra-n'; + db.set('Evangeline', newEva); + assert.deepEqual([], db.find('race', 'cra')); + assert.deepEqual([{key: 'Evangeline', val: newEva}], db.find('race', 'cra-n')); + }); + + it.skip('does not find items for which the index function returns `undefined`', function(){ + assert.deepEqual([], db.find('damageType', undefined)); + }); + }); +}); + From c711387dba323d1c71a3c01a6f874a11a8a6883d Mon Sep 17 00:00:00 2001 From: Vishnu S Iyengar Date: Sun, 12 May 2013 21:21:10 -0700 Subject: [PATCH 20/28] removing unnecessary line --- lib/dirty/dirty.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index 8a3bdd2..b3cc4cc 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -1,5 +1,3 @@ -if (global.GENTLY) require = GENTLY.hijack(require); - var fs = require('fs'), util = require('util'), EventEmitter = require('events').EventEmitter; From a69cc645e59cb2a9dc56ce374fe50343a181613b Mon Sep 17 00:00:00 2001 From: Vishnu S Iyengar Date: Sun, 12 May 2013 21:24:13 -0700 Subject: [PATCH 21/28] bumping version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index e090cf5..c984164 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "dirty", "description": "A tiny & fast key value store with append-only disk log. Ideal for apps with < 1 million records.", - "version": "0.9.8-pre", + "version": "0.9.9", "dependencies": {}, "main": "./lib/dirty", "devDependencies": { From 8d7988e7cc55d4a0961b21f7166c6b30e6e94294 Mon Sep 17 00:00:00 2001 From: Vishnu S Iyengar Date: Sun, 12 May 2013 22:20:36 -0700 Subject: [PATCH 22/28] find returns clones now --- lib/dirty/dirty.js | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index b3cc4cc..c7a8a5f 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -424,7 +424,7 @@ Dirty.prototype.find = function(index, value) { var validKeys = this._indexFns[index].keyMap[value]; return !validKeys ? [] : validKeys.map(function(k){ - return {key: k, val: self._docs[k]}; + return {key: k, val: self.get(k)}; }); }; diff --git a/package.json b/package.json index c984164..bf680ba 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "dirty", "description": "A tiny & fast key value store with append-only disk log. Ideal for apps with < 1 million records.", - "version": "0.9.9", + "version": "0.9.10", "dependencies": {}, "main": "./lib/dirty", "devDependencies": { From a357b1bb3227533a883770ddaef3eaee09fcc8ba Mon Sep 17 00:00:00 2001 From: Vishnu S Iyengar Date: Mon, 13 May 2013 01:17:15 -0700 Subject: [PATCH 23/28] fixing indexvalues and test --- lib/dirty/dirty.js | 6 +++++- package.json | 2 +- test/test-indexes.js | 5 +++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index c7a8a5f..bd74cec 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -429,5 +429,9 @@ Dirty.prototype.find = function(index, value) { }; Dirty.prototype.indexValues = function(index) { - return _(this._indexFns[index].keyMap).keys(); + var keys = []; + for (var k in this._indexFns[index].keyMap){ + keys.push(k); + } + return keys; }; diff --git a/package.json b/package.json index bf680ba..4fe6240 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "dirty", "description": "A tiny & fast key value store with append-only disk log. Ideal for apps with < 1 million records.", - "version": "0.9.10", + "version": "0.9.11", "dependencies": {}, "main": "./lib/dirty", "devDependencies": { diff --git a/test/test-indexes.js b/test/test-indexes.js index 08324a6..71c58d1 100644 --- a/test/test-indexes.js +++ b/test/test-indexes.js @@ -68,6 +68,11 @@ describe('indexes', function(){ it.skip('does not find items for which the index function returns `undefined`', function(){ assert.deepEqual([], db.find('damageType', undefined)); }); + + it('lists out all the values the index has taken', function(){ + assert.deepEqual(['cra', 'sadida', 'eliatrope'], db.indexValues('race')); + assert.deepEqual(['ranged', "undefined"], db.indexValues('damageType')); + }); }); }); From 18c522ad19936ce12e1d27fd709f37659f715b8e Mon Sep 17 00:00:00 2001 From: Vishnu S Iyengar Date: Thu, 23 May 2013 01:36:27 -0700 Subject: [PATCH 24/28] indexes dont index empty values --- lib/dirty/dirty.js | 8 +++++--- package.json | 2 +- test/test-indexes.js | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index bd74cec..d3857cd 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -389,9 +389,11 @@ Dirty.prototype._deleteKeyFromIndexedKeys = function(keyMap, indexValue, key) { }; Dirty.prototype._addKeyToIndexedKeys = function(keyMap, indexValue, key) { - var keys = keyMap[indexValue] || []; - keys.push(key); - keyMap[indexValue] = keys; + if (indexValue !== undefined) { + var keys = keyMap[indexValue] || []; + keys.push(key); + keyMap[indexValue] = keys; + } }; Dirty.prototype._updateIndex = function(index, key, newVal) { diff --git a/package.json b/package.json index 4fe6240..6f08524 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "dirty", "description": "A tiny & fast key value store with append-only disk log. Ideal for apps with < 1 million records.", - "version": "0.9.11", + "version": "0.9.12", "dependencies": {}, "main": "./lib/dirty", "devDependencies": { diff --git a/test/test-indexes.js b/test/test-indexes.js index 71c58d1..ef749db 100644 --- a/test/test-indexes.js +++ b/test/test-indexes.js @@ -71,7 +71,7 @@ describe('indexes', function(){ it('lists out all the values the index has taken', function(){ assert.deepEqual(['cra', 'sadida', 'eliatrope'], db.indexValues('race')); - assert.deepEqual(['ranged', "undefined"], db.indexValues('damageType')); + assert.deepEqual(['ranged'], db.indexValues('damageType')); }); }); }); From cde7e5a60604622e9abd0c0e98eb01852fa852c1 Mon Sep 17 00:00:00 2001 From: Vishnu S Iyengar Date: Thu, 23 May 2013 16:42:24 -0700 Subject: [PATCH 25/28] moved to multiple index values per index --- README.md | 6 ++-- lib/dirty/dirty.js | 42 +++++++++++++----------- lib/dirty/set.js | 52 +++++++++++++++++++++++++++++ package.json | 2 +- test/config.js | 6 ++-- test/test-indexes.js | 25 ++++++++++++-- test/test-set.js | 78 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 183 insertions(+), 28 deletions(-) create mode 100644 lib/dirty/set.js create mode 100644 test/test-set.js diff --git a/README.md b/README.md index 6f631ba..b63f31f 100644 --- a/README.md +++ b/README.md @@ -133,10 +133,10 @@ Emitted once compacting is complete if you start a compact run and it fails. Whe ### dirty.addIndex(index, indexFn) -Use this to add an index named index. indexFn is a function(key, val) that returns the indexed value. For example +Use this to add an index named index. indexFn is a function(key, val) that returns all the index values of that record. For example - dirty.addIndex('eyeColor', function(k, v){ - return v.eyes; + dirty.addIndex('identifyingColor', function(k, v){ + return [v.eyeColor, v.hairColor, v.skinColor]; }); You can add as many indexes as you want, but beware this adds to every add/delete/update operation an O(k) operation where k is the number of values which match a given index. If your index is not well distributed, with large databases you might face an issue. Don't worry about this most of the time. diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index d3857cd..d4fa4f3 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -1,6 +1,7 @@ var fs = require('fs'), util = require('util'), - EventEmitter = require('events').EventEmitter; + EventEmitter = require('events').EventEmitter, + Set = require('./set'); /** * Constructor function @@ -382,35 +383,38 @@ Dirty.prototype.addIndex = function(index, indexFn) { this._indexFns[index] = {indexFn: indexFn, keyMap: {}}; }; -Dirty.prototype._deleteKeyFromIndexedKeys = function(keyMap, indexValue, key) { +Dirty.prototype._deleteKeyFromIndexedKeys = function(keyMap, indexValues, key) { + indexValues.forEach(function(indexValue){ var keys = keyMap[indexValue]; - keys.splice(keys.indexOf(key), 1); - if (keys.length === 0) delete keyMap[indexValue]; + keys.remove(key) + if (keys.empty()) delete keyMap[indexValue]; + }); }; -Dirty.prototype._addKeyToIndexedKeys = function(keyMap, indexValue, key) { - if (indexValue !== undefined) { - var keys = keyMap[indexValue] || []; - keys.push(key); - keyMap[indexValue] = keys; - } +Dirty.prototype._addKeyToIndexedKeys = function(keyMap, indexValues, key) { + indexValues.forEach(function(indexValue){ + var keys = keyMap[indexValue] || new Set(); + keys.push(key); + keyMap[indexValue] = keys; + }) }; Dirty.prototype._updateIndex = function(index, key, newVal) { var indexFn = this._indexFns[index].indexFn; var keyMap = this._indexFns[index].keyMap; if (key in this._docs) { - var oldIndexValue = indexFn(key, this._docs[key]); + var oldIndexValues = indexFn(key, this._docs[key]); if (newVal != undefined) { - var newIndexValue = indexFn(key, newVal); - if (oldIndexValue === newIndexValue) return; - this._deleteKeyFromIndexedKeys(keyMap, oldIndexValue, key); - this._addKeyToIndexedKeys(keyMap, newIndexValue, key); - } else this._deleteKeyFromIndexedKeys(keyMap, oldIndexValue, key); + var newIndexValues = indexFn(key, newVal); + var indexesToDeleteFrom = new Set(oldIndexValues).difference(new Set(newIndexValues)).toArray(); + var indexesToAddTo = new Set(newIndexValues).difference(new Set(oldIndexValues)).toArray(); + this._deleteKeyFromIndexedKeys(keyMap, indexesToDeleteFrom, key); + this._addKeyToIndexedKeys(keyMap, indexesToAddTo, key); + } else this._deleteKeyFromIndexedKeys(keyMap, oldIndexValues, key); } else { if (newVal != undefined) { - var newIndexValue = indexFn(key, newVal); - this._addKeyToIndexedKeys(keyMap, newIndexValue, key); + var newIndexValues = indexFn(key, newVal); + this._addKeyToIndexedKeys(keyMap, newIndexValues, key); } } }; @@ -425,7 +429,7 @@ Dirty.prototype.find = function(index, value) { var self = this; var validKeys = this._indexFns[index].keyMap[value]; return !validKeys ? [] : - validKeys.map(function(k){ + validKeys.toArray().map(function(k){ return {key: k, val: self.get(k)}; }); }; diff --git a/lib/dirty/set.js b/lib/dirty/set.js new file mode 100644 index 0000000..dd64057 --- /dev/null +++ b/lib/dirty/set.js @@ -0,0 +1,52 @@ +function Set(initial) { + this._items = {}; + this._length = 0; + (initial || []).forEach(function(item){ + this.add(item); + }, this); +} + +Set.prototype.add = function(item) { + if (!this.contains(item)) this._length++; + this._items[item] = true; + return this; +} + +Set.prototype.push = Set.prototype.add; + +Set.prototype.empty = function() { + return this._length === 0; +} + +Set.prototype.contains = function(item) { + return this._items.hasOwnProperty(item); +} + +Set.prototype.remove = function(item) { + if (this.contains(item)) this._length--; + delete this._items[item]; + return this; +} + +Set.prototype.toArray = function() { + var retVal = []; + for (var item in this._items) { + if (this.contains(item)) retVal.push(item); + } + return retVal; +} + +Set.prototype.difference = function(other, returnArray) { + var result = returnArray ? [] : new Set(); + for (var item in this._items) { + if (this.contains(item) && !other.contains(item)) result.push(item); + } + return result; +} + +Set.prototype.__defineGetter__("length", function(){ + return this._length; +}); + + +module.exports = Set; \ No newline at end of file diff --git a/package.json b/package.json index 6f08524..0cccb6f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "dirty", "description": "A tiny & fast key value store with append-only disk log. Ideal for apps with < 1 million records.", - "version": "0.9.12", + "version": "0.9.13", "dependencies": {}, "main": "./lib/dirty", "devDependencies": { diff --git a/test/config.js b/test/config.js index fd3976e..3dbaf47 100644 --- a/test/config.js +++ b/test/config.js @@ -3,12 +3,14 @@ var path = require('path'), rimraf = require('rimraf'); var TMP_PATH = path.join(__dirname, 'tmp'), - LIB_DIRTY = path.join(__dirname, '../lib/dirty'); + LIB_DIRTY = path.join(__dirname, '../lib/dirty'), + LIB_SET = path.join(__dirname, '../lib/dirty/set'); rimraf.sync(TMP_PATH); fs.mkdirSync(TMP_PATH); module.exports = { TMP_PATH: TMP_PATH, - LIB_DIRTY: LIB_DIRTY + LIB_DIRTY: LIB_DIRTY, + LIB_SET: LIB_SET }; diff --git a/test/test-indexes.js b/test/test-indexes.js index ef749db..a136c5a 100644 --- a/test/test-indexes.js +++ b/test/test-indexes.js @@ -15,10 +15,16 @@ describe('indexes', function(){ db = dirty(file); db.once('load', function(){ db.addIndex('race', function(k,v){ - return v.race; + return [v.race]; }); db.addIndex('damageType', function(k,v){ - return v.damageType; + return v.damageType ? [v.damageType] : []; + }); + var characters = "abcdefghijklmnopqrstuvwxyz".split(''); + db.addIndex('character', function(k, v){ + return characters.filter(function(c){ + return new RegExp(c).exec(k) + }) }); db.set('Evangeline', eva); db.set('Amalia', ama); @@ -65,10 +71,23 @@ describe('indexes', function(){ assert.deepEqual([{key: 'Evangeline', val: newEva}], db.find('race', 'cra-n')); }); - it.skip('does not find items for which the index function returns `undefined`', function(){ + it('does not find items for which the index function returns `undefined`', function(){ assert.deepEqual([], db.find('damageType', undefined)); }); + it('allows the same item to be known via multiple indexes', function(){ + var compare = function(a,b){ + return (a.key < b.key) ? -1 : 1; + } + assert.deepEqual([{key: 'Yugo', val: yug}], db.find('character', 'u')); + assert.deepEqual([{key: 'Amalia', val: ama},{key: 'Evangeline', val: eva}], db.find('character', 'i').sort(compare)); + assert.deepEqual([], db.find('character', 'y')); + db.rm('Evangeline'); + db.set('Evangelyne', eva); + assert.deepEqual([{key: 'Amalia', val: ama}], db.find('character', 'i')); + assert.deepEqual([{key: 'Evangelyne', val: eva}], db.find('character', 'y')); + }); + it('lists out all the values the index has taken', function(){ assert.deepEqual(['cra', 'sadida', 'eliatrope'], db.indexValues('race')); assert.deepEqual(['ranged'], db.indexValues('damageType')); diff --git a/test/test-set.js b/test/test-set.js new file mode 100644 index 0000000..eaae8d2 --- /dev/null +++ b/test/test-set.js @@ -0,0 +1,78 @@ +var config = require('./config'), + fs = require('fs'), + assert = require('assert') + Set = require(config.LIB_SET); + +describe('test-sets', function() { + it('should be possible to create an empty set', function(){ + assert.ok(new Set().empty()); + }) + + it('should be possible to initialize a set from an array', function(){ + var set = new Set(['foo','bar']); + assert.ok(!set.empty()); + assert.equal(2, set.length); + }); + + it('should not add duplicate elements', function(){ + var set = new Set(['foo', 'bar', 'foo']); + set.add('foo'); + set.add('bar'); + assert.equal(2, set.length); + }); + + it('should report the existence of elements', function(){ + var set = new Set(['foo', 'bar']); + assert.ok(set.contains('bar')); + assert.ok(!set.contains('baz')); + }); + + it('should let you remove elements', function(){ + var set = new Set(['foo', 'bar']); + set.remove('foo'); + assert.equal(1, set.length); + assert.ok(!set.contains('foo')); + }); + + it('should provide an array of all items in the set', function(){ + var set = new Set(['foo', 'bar', 'foo']); + assert.deepEqual(['bar', 'foo'], set.toArray().sort()); + }); + + + it('should not get confused by properties on the prototype of Object', function(){ + Object.prototype.baz = true; + var set = new Set(['foo', 'bar']); + var assertHasBaz = function() { + assert.equal(3, set.length); + assert.ok(set.contains('baz')); + assert.deepEqual(['bar', 'baz', 'foo'], set.toArray().sort()); + } + var assertHasNoBaz = function() { + assert.equal(2, set.length); + assert.ok(!set.contains('baz')); + assert.deepEqual(['bar', 'foo'], set.toArray().sort()); + } + assertHasNoBaz(); + set.add('baz'); + assertHasBaz(); + set.remove('baz'); + assertHasNoBaz(); + set.remove('baz'); + assertHasNoBaz(); + }); + + it('should return all items that are not present in the given set', function(){ + var set1 = new Set(['foo', 'bar', 'baz']); + var set2 = new Set(['baz', 'qux', 'quux']); + assert.deepEqual(['bar', 'foo'], set1.difference(set2).toArray().sort()); + assert.deepEqual(['quux', 'qux'], set2.difference(set1).toArray().sort()); + }); + + it('should return all items that are not present in the given set as an array if passed a second parameter (true)', function(){ + var set1 = new Set(['foo', 'bar', 'baz']); + var set2 = new Set(['baz', 'qux', 'quux']); + assert.deepEqual(['bar', 'foo'], set1.difference(set2, true).sort()); + assert.deepEqual(['quux', 'qux'], set2.difference(set1, true).sort()); + }); +}); From f1b9e300858c9339bca18807c0e135e78555ebdd Mon Sep 17 00:00:00 2001 From: Vishnu S Iyengar Date: Fri, 24 May 2013 10:16:55 -0700 Subject: [PATCH 26/28] fixing bad test breaking other tests --- test/test-set.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test-set.js b/test/test-set.js index eaae8d2..daaf77e 100644 --- a/test/test-set.js +++ b/test/test-set.js @@ -41,6 +41,7 @@ describe('test-sets', function() { it('should not get confused by properties on the prototype of Object', function(){ + try { Object.prototype.baz = true; var set = new Set(['foo', 'bar']); var assertHasBaz = function() { @@ -60,6 +61,11 @@ describe('test-sets', function() { assertHasNoBaz(); set.remove('baz'); assertHasNoBaz(); + } catch (e) { + throw e + } finally { + delete Object.prototype.baz; + } }); it('should return all items that are not present in the given set', function(){ From 357b24017a049f03c6084538a54869646228d02f Mon Sep 17 00:00:00 2001 From: Vishnu S Iyengar Date: Fri, 24 May 2013 10:17:26 -0700 Subject: [PATCH 27/28] get returns a deep clone which should allow fancy datastructures and indexing --- lib/dirty/dirty.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/dirty/dirty.js b/lib/dirty/dirty.js index d4fa4f3..13a7431 100644 --- a/lib/dirty/dirty.js +++ b/lib/dirty/dirty.js @@ -86,7 +86,7 @@ Dirty.prototype._clone = function(obj) { } var retval = {}; for (var k in obj) { - retval[k] = obj[k]; + retval[k] = this._clone(obj[k]); } return retval; }; From 16f473a18c26bfb13d1315f3ce2b149366cfb0ce Mon Sep 17 00:00:00 2001 From: Vishnu S Iyengar Date: Fri, 24 May 2013 10:17:47 -0700 Subject: [PATCH 28/28] bumping version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 0cccb6f..a5ef2da 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "dirty", "description": "A tiny & fast key value store with append-only disk log. Ideal for apps with < 1 million records.", - "version": "0.9.13", + "version": "0.9.14", "dependencies": {}, "main": "./lib/dirty", "devDependencies": {