From 14cc39ba4ab00a62531b218ad38f5e6cad4110c8 Mon Sep 17 00:00:00 2001 From: luin Date: Tue, 3 May 2016 00:28:40 +0800 Subject: [PATCH 1/6] feat: add dropBufferSupport option to improve the performance --- API.md | 2 ++ README.md | 7 +++---- lib/commander.js | 27 +++++++++++++++++++++++++-- lib/redis.js | 6 ++++++ lib/redis/parser.js | 8 +++++++- 5 files changed, 43 insertions(+), 7 deletions(-) diff --git a/API.md b/API.md index ab0898e2..de42c188 100644 --- a/API.md +++ b/API.md @@ -59,6 +59,8 @@ Creates a Redis instance | [options.connectionName] | string | null | Connection name. | | [options.db] | number | 0 | Database index to use. | | [options.password] | string | null | If set, client will send AUTH command with the value of this option when connected. | +| [options.parser] | string | null | Either "hiredis" or "javascript". If not set, "hiredis" parser will be used if it's installed (`npm install hiredis`), otherwise "javascript" parser will be used. | +| [options.dropBufferSupport] | boolean | false | Drop the buffer support for better performance. This option is recommanded to be enabled when "hiredis" parser is used. Refer to /~https://github.com/luin/ioredis/wiki/Improve-Performance for more details. | | [options.enableReadyCheck] | boolean | true | When a connection is established to the Redis server, the server might still be loading the database from disk. While loading, the server not respond to any commands. To work around this, when this option is `true`, ioredis will check the status of the Redis server, and when the Redis server is able to process commands, a `ready` event will be emitted. | | [options.enableOfflineQueue] | boolean | true | By default, if there is no active connection to the Redis server, commands are added to a queue and are executed once the connection is "ready" (when `enableReadyCheck` is `true`, "ready" means the Redis server has loaded the database from disk, otherwise means the connection to the Redis server has been established). If this option is false, when execute the command when the connection isn't ready, an error will be returned. | | [options.connectTimeout] | number | 10000 | The milliseconds before a timeout occurs during the initial connection to the Redis server. | diff --git a/README.md b/README.md index f777ad52..f0a8ecd6 100644 --- a/README.md +++ b/README.md @@ -807,10 +807,9 @@ var cluster = new Redis.Cluster([ }); ``` -## Native Parser -If [hiredis](/~https://github.com/redis/hiredis-node) is installed (by `npm install hiredis`), -ioredis will use it by default. Otherwise, a pure JavaScript parser will be used. -Typically, there's not much difference between them in terms of performance. +## Improve Performance +ioredis supports two parsers, "hiredis" and "javascript". Refer to /~https://github.com/luin/ioredis/wiki/Improve-Performance +for details about the differences between them in terms of performance.
diff --git a/lib/commander.js b/lib/commander.js index e5f9c1c5..701ab843 100644 --- a/lib/commander.js +++ b/lib/commander.js @@ -3,6 +3,11 @@ var _ = require('lodash'); var Command = require('./command'); var Script = require('./script'); +var Promise = require('bluebird'); + +var DROP_BUFFER_SUPPORT_ERROR = '*Buffer methods are not available ' + + 'because "dropBufferSupport" option is enabled.' + + 'Refer to /~https://github.com/luin/ioredis/wiki/Improve-Performance for more details.'; /** * Commander @@ -106,7 +111,16 @@ function generateFunction(_commandName, _encoding) { args[i - firstArgIndex] = arguments[i]; } - var options = { replyEncoding: _encoding }; + var options; + if (this.options.dropBufferSupport) { + if (!_encoding) { + return Promise.reject(new Error(DROP_BUFFER_SUPPORT_ERROR)).nodeify(callback); + } + options = { replyEncoding: null }; + } else { + options = { replyEncoding: _encoding }; + } + if (this.options.showFriendlyErrorStack) { options.errorStack = new Error().stack; } @@ -133,7 +147,16 @@ function generateScriptingFunction(_script, _encoding) { args[i] = arguments[i]; } - var options = { replyEncoding: _encoding }; + var options; + if (this.options.dropBufferSupport) { + if (!_encoding) { + return Promise.reject(new Error(DROP_BUFFER_SUPPORT_ERROR)).nodeify(callback); + } + options = { replyEncoding: null }; + } else { + options = { replyEncoding: _encoding }; + } + if (this.options.showFriendlyErrorStack) { options.errorStack = new Error().stack; } diff --git a/lib/redis.js b/lib/redis.js index b810807f..d1197603 100644 --- a/lib/redis.js +++ b/lib/redis.js @@ -36,6 +36,11 @@ var ScanStream = require('./scan_stream'); * @param {number} [options.db=0] - Database index to use. * @param {string} [options.password=null] - If set, client will send AUTH command * with the value of this option when connected. + * @param {string} [options.parser=null] - Either "hiredis" or "javascript". If not set, "hiredis" parser + * will be used if it's installed (`npm install hiredis`), otherwise "javascript" parser will be used. + * @param {boolean} [options.dropBufferSupport=false] - Drop the buffer support for better performance. + * This option is recommanded to be enabled when "hiredis" parser is used. + * Refer to /~https://github.com/luin/ioredis/wiki/Improve-Performance for more details. * @param {boolean} [options.enableReadyCheck=true] - When a connection is established to * the Redis server, the server might still be loading the database from disk. * While loading, the server not respond to any commands. @@ -166,6 +171,7 @@ Redis.defaultOptions = { db: 0, // Others parser: null, + dropBufferSupport: false, enableOfflineQueue: true, enableReadyCheck: true, autoResubscribe: true, diff --git a/lib/redis/parser.js b/lib/redis/parser.js index 6137cc4c..4cd11de9 100644 --- a/lib/redis/parser.js +++ b/lib/redis/parser.js @@ -20,7 +20,7 @@ exports.initParser = function () { this.replyParser = new Parser({ name: this.options.parser, stringNumbers: this.options.stringNumbers, - returnBuffers: true, + returnBuffers: !this.options.dropBufferSupport, returnError: function (err) { _this.returnError(new ReplyError(err.message)); }, @@ -33,6 +33,12 @@ exports.initParser = function () { _this.disconnect(true); } }); + + if (this.replyParser.name === 'hiredis' && !this.options.dropBufferSupport) { + console.warn('ioredis is using hiredis parser, however "dropBufferSupport" is disabled. ' + + 'It\'s highly recommanded to enable this option. ' + + 'Refer to /~https://github.com/luin/ioredis/wiki/Improve-Performance for more details.'); + } }; exports.returnError = function (err) { From 19cd6cd1eb4103099c4e517b7fbdfe24170c4e72 Mon Sep 17 00:00:00 2001 From: luin Date: Tue, 3 May 2016 10:53:03 +0800 Subject: [PATCH 2/6] test: add test for dropBufferSupport option --- lib/redis/parser.js | 2 +- test/functional/drop_buffer_support.js | 85 ++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 test/functional/drop_buffer_support.js diff --git a/lib/redis/parser.js b/lib/redis/parser.js index 4cd11de9..2068c42f 100644 --- a/lib/redis/parser.js +++ b/lib/redis/parser.js @@ -35,7 +35,7 @@ exports.initParser = function () { }); if (this.replyParser.name === 'hiredis' && !this.options.dropBufferSupport) { - console.warn('ioredis is using hiredis parser, however "dropBufferSupport" is disabled. ' + + console.warn('[WARN] ioredis is using hiredis parser, however "dropBufferSupport" is disabled. ' + 'It\'s highly recommanded to enable this option. ' + 'Refer to /~https://github.com/luin/ioredis/wiki/Improve-Performance for more details.'); } diff --git a/test/functional/drop_buffer_support.js b/test/functional/drop_buffer_support.js new file mode 100644 index 00000000..6469320e --- /dev/null +++ b/test/functional/drop_buffer_support.js @@ -0,0 +1,85 @@ +'use strict'; + +describe.only('dropBufferSupport', function () { + it('should be disabled by default', function () { + var redis = new Redis({ lazyConnect: true }); + expect(redis.options).to.have.property('dropBufferSupport', false); + }); + + it('should reject the buffer commands', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.getBuffer('foo', function (err) { + expect(err.message).to.match(/Buffer methods are not available/); + + redis.callBuffer('get', 'foo', function (err) { + expect(err.message).to.match(/Buffer methods are not available/); + redis.disconnect(); + done(); + }); + }); + }); + + it('should reject the custom buffer commands', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.defineCommand('geteval', { + numberOfKeys: 0, + lua: 'return "string"' + }); + redis.getevalBuffer(function (err) { + expect(err.message).to.match(/Buffer methods are not available/); + redis.disconnect(); + done(); + }); + }); + + it('should set the returnBuffers option to false', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.once('ready', function () { + expect(redis.replyParser.options.return_buffers).to.eql(false); + redis.disconnect(); + done(); + }); + }); + + it('should return strings correctly', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.set('foo', new Buffer('bar'), function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('OK'); + redis.get('foo', function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('bar'); + redis.disconnect(); + done(); + }); + }); + }); + + it('should return strings for custom commands', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.defineCommand('geteval', { + numberOfKeys: 0, + lua: 'return "string"' + }); + redis.geteval(function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('string'); + redis.disconnect(); + done(); + }); + }); + + it('should return strings correctly', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.set('foo', new Buffer('bar'), function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('OK'); + redis.get('foo', function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('bar'); + redis.disconnect(); + done(); + }); + }); + }); +}); From 68cfd587a36e4f8346fbebdf3a0f7e8ca3bf2d46 Mon Sep 17 00:00:00 2001 From: luin Date: Tue, 3 May 2016 12:21:01 +0800 Subject: [PATCH 3/6] perf(benchmark): update benchmark to test dropBufferSupport --- benchmarks/single_node.js | 217 +++++++++----------------------------- 1 file changed, 47 insertions(+), 170 deletions(-) diff --git a/benchmarks/single_node.js b/benchmarks/single_node.js index baf8b8c6..fe1c4f8e 100644 --- a/benchmarks/single_node.js +++ b/benchmarks/single_node.js @@ -1,13 +1,10 @@ 'use strict'; var childProcess = require('child_process'); -var nodeRedis = require('redis'); -var IORedis = require('../'); -var ndredis, ioredis; +var Redis = require('../'); console.log('=========================='); -console.log('ioredis: ' + require('../package.json').version); -console.log('node_redis: ' + require('redis/package.json').version); +console.log('redis: ' + require('../package.json').version); var os = require('os'); console.log('CPU: ' + os.cpus().length); console.log('OS: ' + os.platform() + ' ' + os.arch()); @@ -15,207 +12,87 @@ console.log('node version: ' + process.version); console.log('current commit: ' + childProcess.execSync('git rev-parse --short HEAD')); console.log('=========================='); +var redisJD, redisJ, redisBD, redisB; var waitReady = function (next) { - var pending = 2; - ndredis.on('ready', function () { + var pending = 4; + function check() { if (!--pending) { next(); } - }); + } + redisJD = new Redis({ parser: 'javascript', dropBufferSupport: true }); + redisJ = new Redis({ parser: 'javascript', dropBufferSupport: false }); + redisBD = new Redis({ parser: 'hiredis', dropBufferSupport: true }); + redisB = new Redis({ parser: 'hiredis', dropBufferSupport: false }); + redisJD.on('ready', check); + redisJ.on('ready', check); + redisBD.on('ready', check); + redisB.on('ready', check); +}; - ioredis.on('ready', function () { - if (!--pending) { - next(); - } - }); +var quit = function () { + redisJD.quit(); + redisJ.quit(); + redisBD.quit(); + redisB.quit(); }; -suite('simple set', function () { +suite('SET foo bar', function () { set('mintime', 5000); set('concurrency', 300); before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); waitReady(start); }); - bench('ioredis', function (next) { - ioredis.set('foo', 'bar', next); - }); - - bench('node_redis', function (next) { - ndredis.set('foo', 'bar', next); + bench('javascript parser + dropBufferSupport: true', function (next) { + redisJD.set('foo', 'bar', next); }); - after(function () { - ndredis.quit(); - ioredis.quit(); - }); -}); - -suite('simple get', function () { - set('mintime', 5000); - set('concurrency', 300); - before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - ndredis.set('foo', 'bar', start); - }); + bench('javascript parser', function (next) { + redisJ.setBuffer('foo', 'bar', next); }); - bench('ioredis', function (next) { - ioredis.get('foo', next); + bench('hiredis parser + dropBufferSupport: true', function (next) { + redisBD.set('foo', 'bar', next); }); - bench('node_redis', function (next) { - ndredis.get('foo', next); + bench('hiredis parser', function (next) { + redisB.setBuffer('foo', 'bar', next); }); - after(function () { - ndredis.quit(); - ioredis.quit(); - }); + after(quit); }); -suite('simple get with pipeline', function () { +suite('LRANGE foo 0 99', function () { set('mintime', 5000); set('concurrency', 300); before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - ndredis.set('foo', 'bar', start); - }); - }); - - bench('ioredis', function (next) { - var pipeline = ioredis.pipeline(); - for (var i = 0; i < 10; ++i) { - pipeline.get('foo'); - } - pipeline.exec(next); - }); - - bench('node_redis', function (next) { - var pending = 0; - for (var i = 0; i < 10; ++i) { - pending += 1; - ndredis.get('foo', check); + var redis = new Redis(); + var item = []; + for (var i = 0; i < 100; ++i) { + item.push((Math.random() * 100000 | 0) + 'str'); } - function check() { - if (!--pending) { - next(); - } - } - }); - - after(function () { - ndredis.quit(); - ioredis.quit(); - }); -}); - -suite('lrange 100', function () { - set('mintime', 5000); - set('concurrency', 300); - before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - var item = []; - for (var i = 0; i < 100; ++i) { - item.push((Math.random() * 100000 | 0) + 'str'); - } - ndredis.del('foo'); - ndredis.lpush('foo', item, start); - }); - }); - - bench('ioredis', function (next) { - ioredis.lrange('foo', 0, 99, next); - }); - - bench('node_redis', function (next) { - ndredis.lrange('foo', 0, 99, next); - }); - - after(function () { - ndredis.quit(); - ioredis.quit(); - }); -}); - -suite('publish', function () { - set('mintime', 5000); - set('concurrency', 300); - - before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - start(); + redis.del('foo'); + redis.lpush('foo', item, function () { + waitReady(start); }); }); - bench('ioredis', function (next) { - ioredis.publish('foo', 'bar', next); - }); - - bench('node_redis', function (next) { - ndredis.publish('foo', 'bar', next); + bench('javascript parser + dropBufferSupport: true', function (next) { + redisJD.lrange('foo', 0, 99, next); }); - after(function () { - ndredis.quit(); - ioredis.quit(); - }); -}); - -suite('subscribe', function () { - set('mintime', 5000); - set('concurrency', 300); - - var ndpublisher = null; - var iopublisher = null; - var ndsubscriber = null; - var iosubscriber = null; - - before(function (start) { - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - ndsubscriber = ndredis; - ndsubscriber.subscribe('foo'); - iosubscriber = ioredis; - iosubscriber.subscribe('foo'); - - ndredis = nodeRedis.createClient(); - ioredis = new IORedis(); - waitReady(function () { - ndpublisher = ndredis; - iopublisher = ioredis; - start(); - }); - }); + bench('javascript parser', function (next) { + redisJ.lrangeBuffer('foo', 0, 99, next); }); - bench('ioredis', function (next) { - iosubscriber.removeAllListeners('message'); - ndsubscriber.removeAllListeners('message'); - iosubscriber.on('message', next); - iopublisher.publish('foo', 'bar'); + bench('hiredis parser + dropBufferSupport: true', function (next) { + redisBD.lrange('foo', 0, 99, next); }); - bench('node_redis', function (next) { - iosubscriber.removeAllListeners('message'); - ndsubscriber.removeAllListeners('message'); - ndsubscriber.on('message', next); - ndpublisher.publish('foo', 'bar'); + bench('hiredis parser', function (next) { + redisB.lrangeBuffer('foo', 0, 99, next); }); - after(function () { - ndredis.quit(); - ioredis.quit(); - }); + after(quit); }); From b335cd2a46d0c6d04783bd57cf3bd88a27c7819e Mon Sep 17 00:00:00 2001 From: luin Date: Tue, 3 May 2016 12:21:19 +0800 Subject: [PATCH 4/6] test: not test private method --- test/functional/drop_buffer_support.js | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/test/functional/drop_buffer_support.js b/test/functional/drop_buffer_support.js index 6469320e..2d4c2022 100644 --- a/test/functional/drop_buffer_support.js +++ b/test/functional/drop_buffer_support.js @@ -1,6 +1,6 @@ 'use strict'; -describe.only('dropBufferSupport', function () { +describe('dropBufferSupport', function () { it('should be disabled by default', function () { var redis = new Redis({ lazyConnect: true }); expect(redis.options).to.have.property('dropBufferSupport', false); @@ -32,15 +32,6 @@ describe.only('dropBufferSupport', function () { }); }); - it('should set the returnBuffers option to false', function (done) { - var redis = new Redis({ dropBufferSupport: true }); - redis.once('ready', function () { - expect(redis.replyParser.options.return_buffers).to.eql(false); - redis.disconnect(); - done(); - }); - }); - it('should return strings correctly', function (done) { var redis = new Redis({ dropBufferSupport: true }); redis.set('foo', new Buffer('bar'), function (err, res) { From beca075a5182cd6095090d3f2a25d40f0a218b11 Mon Sep 17 00:00:00 2001 From: luin Date: Tue, 3 May 2016 20:50:28 +0800 Subject: [PATCH 5/6] test: fix the duplicated test --- test/functional/drop_buffer_support.js | 92 +++++++++++++------------- 1 file changed, 47 insertions(+), 45 deletions(-) diff --git a/test/functional/drop_buffer_support.js b/test/functional/drop_buffer_support.js index 2d4c2022..fcc741e2 100644 --- a/test/functional/drop_buffer_support.js +++ b/test/functional/drop_buffer_support.js @@ -6,34 +6,8 @@ describe('dropBufferSupport', function () { expect(redis.options).to.have.property('dropBufferSupport', false); }); - it('should reject the buffer commands', function (done) { - var redis = new Redis({ dropBufferSupport: true }); - redis.getBuffer('foo', function (err) { - expect(err.message).to.match(/Buffer methods are not available/); - - redis.callBuffer('get', 'foo', function (err) { - expect(err.message).to.match(/Buffer methods are not available/); - redis.disconnect(); - done(); - }); - }); - }); - - it('should reject the custom buffer commands', function (done) { - var redis = new Redis({ dropBufferSupport: true }); - redis.defineCommand('geteval', { - numberOfKeys: 0, - lua: 'return "string"' - }); - redis.getevalBuffer(function (err) { - expect(err.message).to.match(/Buffer methods are not available/); - redis.disconnect(); - done(); - }); - }); - it('should return strings correctly', function (done) { - var redis = new Redis({ dropBufferSupport: true }); + var redis = new Redis({ dropBufferSupport: false }); redis.set('foo', new Buffer('bar'), function (err, res) { expect(err).to.eql(null); expect(res).to.eql('OK'); @@ -46,28 +20,56 @@ describe('dropBufferSupport', function () { }); }); - it('should return strings for custom commands', function (done) { - var redis = new Redis({ dropBufferSupport: true }); - redis.defineCommand('geteval', { - numberOfKeys: 0, - lua: 'return "string"' + context('enabled', function () { + it('should reject the buffer commands', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.getBuffer('foo', function (err) { + expect(err.message).to.match(/Buffer methods are not available/); + + redis.callBuffer('get', 'foo', function (err) { + expect(err.message).to.match(/Buffer methods are not available/); + redis.disconnect(); + done(); + }); + }); }); - redis.geteval(function (err, res) { - expect(err).to.eql(null); - expect(res).to.eql('string'); - redis.disconnect(); - done(); + + it('should reject the custom buffer commands', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.defineCommand('geteval', { + numberOfKeys: 0, + lua: 'return "string"' + }); + redis.getevalBuffer(function (err) { + expect(err.message).to.match(/Buffer methods are not available/); + redis.disconnect(); + done(); + }); }); - }); - it('should return strings correctly', function (done) { - var redis = new Redis({ dropBufferSupport: true }); - redis.set('foo', new Buffer('bar'), function (err, res) { - expect(err).to.eql(null); - expect(res).to.eql('OK'); - redis.get('foo', function (err, res) { + it('should return strings correctly', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.set('foo', new Buffer('bar'), function (err, res) { expect(err).to.eql(null); - expect(res).to.eql('bar'); + expect(res).to.eql('OK'); + redis.get('foo', function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('bar'); + redis.disconnect(); + done(); + }); + }); + }); + + it('should return strings for custom commands', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + redis.defineCommand('geteval', { + numberOfKeys: 0, + lua: 'return "string"' + }); + redis.geteval(function (err, res) { + expect(err).to.eql(null); + expect(res).to.eql('string'); redis.disconnect(); done(); }); From d7f6dcec87d38cb0bd8bec7ca85b1bf50f87e17c Mon Sep 17 00:00:00 2001 From: luin Date: Tue, 3 May 2016 20:53:53 +0800 Subject: [PATCH 6/6] test: add test for dropBufferSupport and pipeline --- test/functional/drop_buffer_support.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/functional/drop_buffer_support.js b/test/functional/drop_buffer_support.js index fcc741e2..9f16fd0f 100644 --- a/test/functional/drop_buffer_support.js +++ b/test/functional/drop_buffer_support.js @@ -74,5 +74,18 @@ describe('dropBufferSupport', function () { done(); }); }); + + it('should work with pipeline', function (done) { + var redis = new Redis({ dropBufferSupport: true }); + var pipeline = redis.pipeline(); + pipeline.set('foo', 'bar'); + pipeline.get(new Buffer('foo')); + pipeline.exec(function (err, res) { + expect(err).to.eql(null); + expect(res[0][1]).to.eql('OK'); + expect(res[1][1]).to.eql('bar'); + done(); + }); + }); }); });