From a7488068b2ebfbe58219ad17912a829e024c080a Mon Sep 17 00:00:00 2001 From: Wyatt Preul Date: Mon, 12 Dec 2016 14:56:16 -0600 Subject: [PATCH] joyent/node-manta#296 add client encryption support --- .gitignore | 1 + README.md | 2 +- bin/mget | 79 ++++- bin/mput | 56 ++- docs/man/mget.md | 35 +- docs/man/mput.md | 77 ++++ lib/client.js | 90 ++++- lib/create_client.js | 11 +- lib/cse.js | 641 ++++++++++++++++++++++++++++++++++ lib/index.js | 5 +- lib/parse_etm_stream.js | 89 +++++ test/client.test.js | 296 +++++++++++++++- test/cse.test.js | 432 +++++++++++++++++++++++ test/parse_etm_stream.test.js | 134 +++++++ 14 files changed, 1923 insertions(+), 25 deletions(-) create mode 100644 lib/cse.js create mode 100644 lib/parse_etm_stream.js create mode 100644 test/cse.test.js create mode 100644 test/parse_etm_stream.test.js diff --git a/.gitignore b/.gitignore index fc23e5a..14b0f00 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ smf/manifests/bapi.xml /test/node.paths /npm-debug.log /share/manta.completion +.DS_Store diff --git a/README.md b/README.md index e581700..05605c4 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ A full set of commands for interacting with Manta is in `bin`. # More documentation Docs can be found here: -[http://apidocs.joyent.com/manta/](http://apidocs.joyent.com/manta/) +[https://apidocs.joyent.com/manta/](https://apidocs.joyent.com/manta/) # Testing diff --git a/bin/mget b/bin/mget index 2a7b979..338b64b 100755 --- a/bin/mget +++ b/bin/mget @@ -8,6 +8,7 @@ var fs = require('fs'); var http = require('http'); var path = require('path-platform'); var url = require('url'); +var util = require('util'); var bunyan = require('bunyan'); var dashdash = require('dashdash'); @@ -25,8 +26,46 @@ var LOG = bunyan.createLogger({ stream: process.stderr }); + +var ENCRYPT_AUTH_MODES = ['MandatoryAuthentication', 'OptionalAuthentication']; + +dashdash.addOptionType({ + name: 'encryptAuthMode', + takesArg: true, + helpArg: 'MODE', + parseArg: function parseEncryptAuthMode(option, optstr, arg) { + manta.encryptValidateAuthMode(arg); + return (arg); + } +}); + var OPTIONS_PARSER = dashdash.createParser({ options: manta.DEFAULT_CLI_OPTIONS.concat([ + { + group: 'Client-side encryption options' + }, + { + names: ['decrypt', 'e'], + type: 'bool', + help: 'Expect the object to be encrypted, and decrypt it.' + }, + { + names: ['encrypt-key'], + type: 'string', + env: 'MANTA_ENCRYPT_KEY', + helpArg: 'KEY', + help: 'Base64 secret key for decrypting remote encrypted objects.' + }, + { + names: ['encrypt-auth-mode'], + type: 'encryptAuthMode', + env: 'MANTA_ENCRYPT_AUTH_MODE', + helpArg: 'MODE', + help: 'Whether decryption will enforce authentication ' + + '("MandatoryAuthentication", the default) or allow some ' + + 'operations, e.g. range requests, which cannot enforce ' + + 'authentication ("OptionalAuthentication")' + }, { group: NAME + ' options' }, @@ -55,6 +94,11 @@ var OPTIONS_PARSER = dashdash.createParser({ names: ['remote-name', 'O'], type: 'bool', help: 'write output to a file using remote object name as filename' + }, + { + names: ['include'], + type: 'bool', + help: 'Include the HTTP-header in the output.' } ]) }); @@ -114,16 +158,37 @@ function parseOptions() { opts.headers[tmp[0]] = tmp[1].trim(); }); + if (opts.decrypt) { + if (!opts.encrypt_key) { + manta.cli_usage(OPTIONS_PARSER, 'decryption requires ' + + '--encrypt-key or the MANTA_ENCRYPT_KEY environment variable'); + } + var base64Key = new Buffer(opts.encrypt_key, 'base64'); + opts.encrypt = { + getKey: function (keyId, cb) { + cb(null, base64Key); + }, + authMode: opts.encrypt_auth_mode + }; + } + return (opts); } -function printEntry(obj) { - console.log('%j', obj); +// The same as `printEntry` from minfo. +function printHeaders(res) { + console.log('HTTP/%s %s %s', + res.httpVersion, + res.statusCode, + http.STATUS_CODES[res.statusCode]); + Object.keys(res.headers).forEach(function (k) { + console.log('%s: %s', k, res.headers[k]); + }); + console.log(); } - ///--- Mainline (function main() { @@ -143,6 +208,10 @@ function printEntry(obj) { client.get(p, function (err, stream, res) { ifError(err); + if (opts.include) { + printHeaders(res); + } + var bar; var src = stream; if (opts.progress || drawProgressBar) { @@ -162,6 +231,10 @@ function printEntry(obj) { src = stream.pipe(bar.stream()); } + src.on('error', function (srcErr) { + ifError(srcErr); + }); + src.pipe(out); src.on('end', function () { diff --git a/bin/mput b/bin/mput index 36fd08c..60f5515 100755 --- a/bin/mput +++ b/bin/mput @@ -29,6 +29,45 @@ var LOG = bunyan.createLogger({ var OPTIONS_PARSER = dashdash.createParser({ options: manta.DEFAULT_CLI_OPTIONS.concat([ + { + group: 'Client-side encryption options' + }, + { + names: ['encrypt', 'e'], + type: 'bool', + help: 'Encrypt the file before storing it in Manta, using ' + + '--encrypt-* options.' + }, + { + names: ['encrypt-cipher'], + type: 'string', + env: 'MANTA_ENCRYPT_CIPHER', + helpArg: 'CIPHER', + help: 'Algorithm to use for encrypting file.' + }, + { + names: ['encrypt-hmac'], + type: 'string', + env: 'MANTA_ENCRYPT_HMAC', + helpArg: 'HMAC', + help: 'HMAC type to use when encrypting a file. Types are ' + + 'HmacMD5, HmacSHA1, HmacSHA256, and HmacSHA512' + }, + { + names: ['encrypt-key'], + type: 'string', + env: 'MANTA_ENCRYPT_KEY', + helpArg: 'KEY', + help: 'Base64 secret key for encrypting content.' + }, + { + names: ['encrypt-key-id'], + type: 'string', + env: 'MANTA_ENCRYPT_KEY_ID', + helpArg: 'KEY_ID', + help: 'Key identifier. This can be used for locating the key ' + + 'for decryption.' + }, { group: NAME + ' options' }, @@ -146,6 +185,21 @@ function parseOptions() { opts['role-tag'] = opts['role-tag'][0].split(/\s*,\s*/); } + if (opts.encrypt) { + if (!opts.encrypt_key || !opts.encrypt_key_id || !opts.encrypt_cipher) { + manta.cli_usage(OPTIONS_PARSER, 'encrypt requires --encrypt-key, ' + + '--encrypt-key-id, and --encrypt-cipher or related ' + + 'environment variables'); + } + + opts.encrypt = { + key: (new Buffer(opts.encrypt_key, 'base64').toString()), + keyId: opts.encrypt_key_id, + cipher: opts.encrypt_cipher, + hmacType: opts.encrypt_hmac + }; + } + return (opts); } @@ -244,7 +298,7 @@ function printEntry(obj) { }; var fstream = fs.createReadStream(options.file, f_opts); fstream.pause(); - fstream.on('open', function () { + fstream.once('open', function () { put(fstream, stats, cb); }); } diff --git a/docs/man/mget.md b/docs/man/mget.md index d3500d6..4437087 100644 --- a/docs/man/mget.md +++ b/docs/man/mget.md @@ -45,6 +45,16 @@ OPTIONS `-a, --account login` Authenticate as account (login name). +`--encrypt-key=KEY` + Base64 encoded key used to encrypt the file. Will decrypt the stored object + when it's encrypted. If this value is missing then the original encrypted + object is returned. + +`--encrypt-auth-mode=MODE` + Determines if the decrypted object size is strictly enforced to be the same + as the original (pre-encrypted) object size. Defaults to + "MandatoryAuthentication", "OptionalAuthentication" disables strict mode. + `-H, --headers` Print HTTP headers on stderr. @@ -60,9 +70,6 @@ OPTIONS Authenticate using the SSH key described by FINGERPRINT. The key must either be in `~/.ssh` or loaded in the SSH agent via `ssh-add`. -`--role=ROLE,ROLE,...` - Specify which roles to assume for the request. - `-o, --output file` Write output to <file> instead of stdout. @@ -73,6 +80,9 @@ OPTIONS `-q, --quiet` Do not display a progress meter. +`--role=ROLE,ROLE,...` + Specify which roles to assume for the request. + `--user user` Authenticate as user under account. @@ -84,11 +94,12 @@ OPTIONS ENVIRONMENT ----------- -`MANTA_USER` - In place of `-a, --account` -`MANTA_SUBUSER` - In place of `--user`. +`MANTA_ENCRYPT_KEY` + In place of `--encrypt-key` + +`MANTA_ENCRYPT_AUTH_MODE` + In place of `--encrypt-auth-mode` `MANTA_KEY_ID` In place of `-k, --key`. @@ -96,12 +107,18 @@ ENVIRONMENT `MANTA_ROLE` In place of `--role`. -`MANTA_URL` - In place of `-u, --url`. +`MANTA_SUBUSER` + In place of `--user`. `MANTA_TLS_INSECURE` In place of `-i, --insecure`. +`MANTA_URL` + In place of `-u, --url`. + +`MANTA_USER` + In place of `-a, --account` + The shortcut `~~` is equivalent to `/:login` where `:login` is the account login name. diff --git a/docs/man/mput.md b/docs/man/mput.md index abf24b8..90d39c6 100644 --- a/docs/man/mput.md +++ b/docs/man/mput.md @@ -53,6 +53,22 @@ OPTIONS `-c, --copies file` Create COPIES copies as a replication factor (default 2). +`-e, --encrypt` + Encrypt the file using the provided key, key ID, and algorithm. Any encrypted + file will be stored with a m-encrypt-type header set to client/1. + +`--encrypt-cipher=CIPHER` + Encryption algorithm to use. + +`--encrypt-hmac=HMAC` + HMAC algorithm to use for none authentication ciphers (default HmacSHA256). + +`--encrypt-key=KEY` + Base64 encoded key used to encrypt the file. + +`--encrypt-key-id=ID` + Value to uniquely identify the key used for encryption. + `-f, --file file` Create contents of object from file. @@ -96,6 +112,15 @@ OPTIONS ENVIRONMENT ----------- +`MANTA_ENCRYPT_CIPHER` + In place of `--encrypt-cipher` + +`MANTA_ENCRYPT_KEY` + In place of `--encrypt-key` + +`MANTA_ENCRYPT_KEY_ID` + In place of `--encrypt-key-id` + `MANTA_USER` In place of `-a, --account` @@ -117,6 +142,58 @@ ENVIRONMENT The shortcut `~~` is equivalent to `/:login` where `:login` is the account login name. +CIPHER STRINGS +-------------- + +Below is a list of supported cipher strings to use in the `--encrypt-cipher` +option. Each uses a 16 byte block size and a 16 byte initialization vector. +The strings are case-insensitive as well. + +`AES128/GCM/NoPadding` + 16 byte key size + +`AES192/GCM/NoPadding` + 24 byte key size + +`AES256/GCM/NoPadding` + 32 byte key size + +`AES128/CTR/NoPadding` + 16 byte key size + +`AES192/CTR/NoPadding` + 24 byte key size + +`AES256/CTR/NoPadding` + 32 byte key size + +`AES128/CBC/PKCS5Padding` + 16 byte key size + +`AES192/CBC/PKCS5Padding` + 24 byte key size + +`AES256/CBC/PKCS5Padding` + 32 byte key size + +HMAC STRINGS +------------ + +Below is a list of supported HMAC strings to use when using not using a GCM +algorithm. + +`HmacMD5` + MD5 digest of 16 bytes + +`HmacSHA1` + SHA1 digest of 20 bytes + +`HmacSHA256` + SHA256 digest of 32 bytes + +`HmacSHA512` + SHA512 digest of 64 bytes + DIAGNOSTICS ----------- diff --git a/lib/client.js b/lib/client.js index 8de4b24..c0588ce 100644 --- a/lib/client.js +++ b/lib/client.js @@ -23,6 +23,7 @@ var Watershed = require('watershed').Watershed; var LOMStream = require('lomstream').LOMStream; var auth = require('smartdc-auth'); +var cse = require('./cse'); var jobshare = require('./jobshare'); var Queue = require('./queue'); var trackmarker = require('./trackmarker'); @@ -165,13 +166,29 @@ function createOptions(opts, userOpts) { assert.object(opts, 'options'); assert.string(opts.path, 'options.path'); assert.object(userOpts, 'userOptions'); - + assert.ok(!userOpts.encrypt || typeof (userOpts.encrypt) === 'object', + 'options.encrypt must be false, an object, or not set'); + + // userEncrypt defaults to false if client.encrypt is false + var userEncrypt = (opts.encrypt === false) || (userOpts.encrypt === false) ? + false : (userOpts.encrypt || {}); + var optsEncrypt = opts.encrypt || {}; + + var encrypt = (userEncrypt === false) ? false : { + cipher: userEncrypt.cipher || optsEncrypt.cipher, + keyId: userEncrypt.keyId || optsEncrypt.keyId, + key: userEncrypt.key || optsEncrypt.key, + getKey: userEncrypt.getKey || optsEncrypt.getKey, + hmacType: userEncrypt.hmacType || optsEncrypt.hmacType, + authMode: userEncrypt.authMode || optsEncrypt.authMode + }; var id = opts.req_id || libuuid.v4(); var options = { headers: normalizeHeaders(userOpts.headers), id: id, path: opts.path.replace(/\/$/, ''), - query: clone(userOpts.query || {}) + query: clone(userOpts.query || {}), + encrypt: encrypt }; if (userOpts.role) @@ -492,6 +509,10 @@ function resultToInfoCb(_path, cb) { * - user : optional user to create jobs under * - subuser: optional subuser under the user * - role: optional array of roles that are active for requests + * - encrypt: optional, when false then client-side encryption is disabled. + * When set to an object it can contain defaults for the client-side + * encryption cipher, keyId, key, getKey, and hmacType options provided + * to get()/put() * * Throws TypeError's if you pass bad arguments. */ @@ -510,7 +531,8 @@ function MantaClient(options) { assert.optionalFunc(options.sign, 'options.sign'); assert.ok(options.url || options.socketPath, 'one of options.url or options.socketPath is required'); - + assert.ok(!options.encrypt || typeof (options.encrypt) === 'object', + 'options.encrypt must be false, an object, or not set'); EventEmitter.call(this); var self = this; @@ -522,6 +544,22 @@ function MantaClient(options) { this.user = options.user; this.subuser = options.subuser; + this.encrypt = (options.encrypt === false) ? false : + (options.encrypt || {}); + + if (this.encrypt) { + assert.optionalString(this.encrypt.cipher, 'options.encrypt.cipher'); + assert.optionalFunc(this.encrypt.getKey, 'options.encrypt.getKey'); + assert.optionalString(this.encrypt.hmacType, + 'options.encrypt.hmacType'); + assert.optionalString(this.encrypt.key, 'options.encrypt.key'); + assert.optionalString(this.encrypt.keyId, 'options.encrypt.keyId'); + assert.optionalString(this.encrypt.authMode, + 'options.encrypt.authMode'); + if (this.encrypt.authMode) { + cse.validateAuthMode(this.encrypt.authMode); + } + } if (options.role) { options.headers = options.headers || {}; @@ -784,7 +822,8 @@ MantaClient.prototype.get = function get(p, opts, cb) { var length = false; var options = createOptions({ accept: opts.accept || '*/*', - path: _path + path: _path, + encrypt: this.encrypt }, opts); var log = this.log.child({ path: _path, @@ -827,7 +866,16 @@ MantaClient.prototype.get = function get(p, opts, cb) { res.pause(); - cb(null, stream, res); + // Not encrypted, return original file stream + if (!options.encrypt || !options.encrypt.getKey || + !cse.isSupported(res.headers)) { + + cb(null, stream, res); + } else { + cse.decrypt({getKey: options.encrypt.getKey, + authMode: options.encrypt.authMode, + isRangeRequest: !!options.headers.range}, stream, res, cb); + } if (length === false && res.headers['content-length'] && @@ -1808,6 +1856,8 @@ MantaClient.prototype.mkdirp = function mkdirp(dir, opts, cb) { * - cb: callback of the form f(err) */ MantaClient.prototype.put = function put(p, input, opts, cb) { + var self = this; + assert.string(p, 'path'); assert.stream(input, 'input'); if (typeof (opts) === 'function') { @@ -1831,7 +1881,8 @@ MantaClient.prototype.put = function put(p, input, opts, cb) { mime.lookup(_path)), contentLength: opts.size, expect: '100-continue', - path: _path + path: _path, + encrypt: this.encrypt }, opts); var log = this.log.child({ path: _path, @@ -1843,17 +1894,40 @@ MantaClient.prototype.put = function put(p, input, opts, cb) { parseInt(opts.copies, 10); } - if (options.headers['content-length'] === undefined) + if (options.headers['content-length'] === undefined) { options.headers['transfer-encoding'] = 'chunked'; + } options._original_path = p; // needed for mkdirp case - log.debug(options, 'put: entered'); + if (options.encrypt && options.encrypt.cipher) { + options.headers['e-content-type'] = options.headers['content-type']; + options.headers['content-type'] = 'application/octet-stream'; + var encOptions = { + cipher: options.encrypt.cipher, + key: options.encrypt.key, + keyId: options.encrypt.keyId, + hmacType: options.encrypt.hmacType, + contentLength: options.contentLength, + headers: options.headers + }; + cse.encrypt(encOptions, input, function (err, encrypted) { + if (err) { + cb(err); + return; + } + + doPut(self, log, options, encrypted, cb, opts.mkdirs); + }); + return; + } + doPut(this, log, options, input, cb, opts.mkdirs); }; function doPut(self, log, options, input, cb, allowretry) { + log.debug(options, 'put: entered'); self.signRequest({ headers: options.headers }, function onSignRequest(err) { diff --git a/lib/create_client.js b/lib/create_client.js index 65efe8d..c488bcd 100644 --- a/lib/create_client.js +++ b/lib/create_client.js @@ -100,6 +100,7 @@ var DEFAULT_OPTIONS = [ function cloneOptions(options) { assert.object(options, 'options'); + var encrypt = (options.encrypt === false) ? false : (options.encrypt || {}); return ({ agent: options.agent, @@ -112,7 +113,15 @@ function cloneOptions(options) { user: options.user, subuser: options.subuser, role: options.role, - url: options.url + url: options.url, + encrypt: (encrypt === false) ? false : { + getKey: encrypt.getKey, + keyId: encrypt.keyId, + key: encrypt.key, + cipher: encrypt.cipher, + hmacType: encrypt.hmacType, + authMode: encrypt.authMode + } }); } diff --git a/lib/cse.js b/lib/cse.js new file mode 100644 index 0000000..a65324a --- /dev/null +++ b/lib/cse.js @@ -0,0 +1,641 @@ +// Copyright 2017 Joyent, Inc. + +/* + * Client side encryption module that implements RFD 71: + * https://github.com/joyent/rfd/tree/master/rfd/0071 + * + * Exports decrypt(), encrypt(), and isSupported() functions for use + * by the Manta client module to encrypt/decrypt get/put requests to Manta. + */ + +var crypto = require('crypto'); +var assert = require('assert-plus'); +var ParseEtMStream = require('./parse_etm_stream'); +var PassThrough = require('stream').PassThrough; +var util = require('util'); +var verror = require('verror'); + +var VError = verror.VError; + +var VERSION = 1; +var CIPHERS = { + 'AES128/GCM/NOPADDING': { + string: 'aes-128-gcm', + blockBytes: 16, + ivBytes: 16, + keyBytes: 16, + tagBytes: 16 + }, + 'AES192/GCM/NOPADDING': { + string: 'aes-192-gcm', + blockBytes: 16, + ivBytes: 16, + keyBytes: 24, + tagBytes: 16 + }, + 'AES256/GCM/NOPADDING': { + string: 'aes-256-gcm', + blockBytes: 16, + ivBytes: 16, + keyBytes: 32, + tagBytes: 16 + }, + 'AES128/CTR/NOPADDING': { + string: 'aes-128-ctr', + blockBytes: 16, + ivBytes: 16, + keyBytes: 16 + }, + 'AES192/CTR/NOPADDING': { + string: 'aes-192-ctr', + blockBytes: 16, + ivBytes: 16, + keyBytes: 24 + }, + 'AES256/CTR/NOPADDING': { + string: 'aes-256-ctr', + blockBytes: 16, + ivBytes: 16, + keyBytes: 32 + }, + 'AES128/CBC/PKCS5PADDING': { + string: 'aes-128-cbc', + isPadded: true, + blockBytes: 16, + ivBytes: 16, + keyBytes: 16 + }, + 'AES192/CBC/PKCS5PADDING': { + string: 'aes-192-cbc', + isPadded: true, + blockBytes: 16, + ivBytes: 16, + keyBytes: 24 + }, + 'AES256/CBC/PKCS5PADDING': { + string: 'aes-256-cbc', + isPadded: true, + blockBytes: 16, + ivBytes: 16, + keyBytes: 32 + } +}; + +// GCM encryption modes are only supported in node v1.0 and greater. +var NODE_MAJOR = parseInt(process.versions.node.split('.')[0], 10); +if (NODE_MAJOR === 0) { + delete CIPHERS['AES128/GCM/NOPADDING']; + delete CIPHERS['AES192/GCM/NOPADDING']; + delete CIPHERS['AES256/GCM/NOPADDING']; +} + +var HMACS = [ + { + type: 'HmacMD5', + algorithm: 'md5', + bytes: 16 + }, + { + type: 'HmacSHA1', + algorithm: 'sha1', + bytes: 20 + }, + { + type: 'HmacSHA256', + algorithm: 'sha256', + bytes: 32 + }, + { + type: 'HmacSHA512', + algorithm: 'sha512', + bytes: 64 + } +]; +var REQUIRED_HEADERS = [ + 'm-encrypt-key-id', + 'm-encrypt-iv', + 'm-encrypt-cipher', + 'm-encrypt-type' +]; +/*JSSTYLED*/ +var METADATA_PATTERN = /^e\-.*/i; + + +var ENCRYPT_AUTH_MODES = ['MandatoryAuthentication', 'OptionalAuthentication']; +var ENCRYPT_AUTH_MODES_UPPER = ['MANDATORYAUTHENTICATION', + 'OPTIONALAUTHENTICATION']; + +// Case insensitive validation of auth mode +function validateAuthMode(authMode) { + if (ENCRYPT_AUTH_MODES_UPPER.indexOf(authMode.toUpperCase()) === -1) { + throw new Error(util.format( + 'invalid authentication mode: "%s" (must be one of "%s")', + authMode, ENCRYPT_AUTH_MODES.join('", "'))); + } +} +exports.validateAuthMode = validateAuthMode; + + +/** + * Decrypt an encrypted stream and verify the integrity of the decrypted payload + * The encrypted m-encrypt-metadata header is also decrypted and removed. + * + * Parameters: + * - options: getKey function used to retrieve key value. Signature for getKey + * is getKey(keyId, cb) the cb function should return (err, key) + * - encrypted: readable stream of encrypted data + * - res: raw HTTP response from manta request, used for reading headers + * - cb: callback of the form f(err, output, res) + */ +exports.decrypt = function decrypt(options, encrypted, res, cb) { + assert.object(options, 'options'); + assert.object(res, 'res'); + assert.object(res.headers, 'res.headers'); + assert.stream(encrypted, 'encrypted'); + assert.func(options.getKey, 'options.getKey'); + + var isRangeRequest = options.isRangeRequest !== undefined ? + options.isRangeRequest : false; + var isMandatoryAuthentication = true; + if (options.authMode !== undefined) { + validateAuthMode(options.authMode); + isMandatoryAuthentication = ( + options.authMode === 'MandatoryAuthentication'); + } + + var invalidHeaders = validateHeaders(res.headers); + if (invalidHeaders) { + cb(new Error('Headers are missing or invalid: ' + + invalidHeaders), null, res); + return; + } + + var algorithm = getAlgorithm(res.headers['m-encrypt-cipher']); + if (!algorithm) { + cb(new Error('Unsupported cipher algorithm: ' + + res.headers['m-encrypt-cipher']), null, res); + return; + } + + var hmacType = null; + if (!algorithm.tagBytes) { + hmacType = getHmacType(res.headers['m-encrypt-hmac-type']); + if (util.isError(hmacType)) { + cb(hmacType, null, res); + return; + } + } + + options.getKey(res.headers['m-encrypt-key-id'], function (err, key) { + if (err) { + cb(new VError(err, 'failed executing options.getKey'), null, res); + return; + } + + var iv = new Buffer(res.headers['m-encrypt-iv'], 'base64'); + var decipher = crypto.createDecipheriv(algorithm.string, key, iv); + var hmac = null; + + if (!algorithm.tagBytes) { + hmac = crypto.createHmac(hmacType.algorithm, key); + hmac.update(iv); + } + + var parseEtMStream = new ParseEtMStream(hmacType, + res.headers['content-length'], algorithm.tagBytes); + var output = new PassThrough(); + var passThrough = new PassThrough(); + var byteLength = 0; + + // Only used for AEAD ciphers + var bufferedForAuth = new Buffer(''); + var isTagSet = false; + function handleParseTag(authTag) { + decipher.setAuthTag(authTag); + isTagSet = true; + if (bufferedForAuth.length) { + var decrypted = decipher.update(bufferedForAuth); + + if (decrypted.length) { + passThrough.write(decrypted); + bufferedForAuth = new Buffer(''); + } + } + } + + // Write cipher data to decipher and pass to passThrough stream + function handleEtmData(data) { + // AEAD requires the auth tag when authenticating the request + if (algorithm.tagBytes && isMandatoryAuthentication && !isTagSet) { + bufferedForAuth = Buffer.concat([bufferedForAuth, data]); + return; + } + + var decrypted = decipher.update(data); + if (decrypted.length) { + passThrough.write(decrypted); + } + } + + function handlePassThroughData(data) { + byteLength += Buffer.byteLength(data); + + if (hmac) { + hmac.update(data); + } + } + + function handleEtmFinish() { + // final called for AEAD after setting the auth tag + if (!algorithm.tagBytes) { + passThrough.write(decipher.final()); + } + + // Indicate that we are done writing to passThrough + passThrough.end(); + } + + function handleEncryptedError(streamErr) { + parseEtMStream.removeListener('tag', handleParseTag); + parseEtMStream.removeListener('data', handleEtmData); + parseEtMStream.removeListener('finish', handleEtmFinish); + decipher.removeListener('error', handleDecipherError); + passThrough.removeListener('end', handlePassThroughEnd); + passThrough.removeListener('data', handlePassThroughData); + + output.emit('error', new VError(streamErr, + 'failed to read encrypted data')); + } + + function handleDecipherError(decipherErr) { + passThrough.removeListener('data', handlePassThroughData); + passThrough.removeListener('end', handlePassThroughEnd); + output.emit('error', new VError(decipherErr, + 'failed to write to decipher')); + } + + function handlePassThroughEnd(data) { + if (!algorithm.tagBytes && !isRangeRequest) { + var digest = hmac.digest(); + if (digest.compare(parseEtMStream.digest()) !== 0) { + output.emit('error', new Error('cipher hmac doesn\'t ' + + 'match stored hmac value')); + return; + } + } else if (algorithm.tagBytes && isMandatoryAuthentication) { + try { + decipher.final(); + } catch (authErr) { + output.emit('error', authErr); + } + } + + var origLength = res.headers['m-encrypt-plaintext-content-length']; + if (!isRangeRequest && origLength && + byteLength !== parseInt(origLength, 10)) { + + output.emit('error', new Error( + 'decrypted file size doesn\'t match original copy')); + return; + } + + // Overwrite the content-length with the decrypted byte length + res.headers['content-length'] = byteLength; + } + + var decErr = decryptMetadata(algorithm, hmacType, res.headers, key); + if (decErr) { + cb(decErr, null, res); + return; + } + + encrypted.once('error', handleEncryptedError); + parseEtMStream.on('data', handleEtmData); + parseEtMStream.once('finish', handleEtmFinish); + passThrough.on('data', handlePassThroughData); + passThrough.once('end', handlePassThroughEnd); + decipher.once('error', handleDecipherError); + + cb(null, output, res); + + if (algorithm.tagBytes && !isRangeRequest) { + parseEtMStream.once('tag', handleParseTag); + } + + encrypted.pipe(parseEtMStream); + passThrough.pipe(output); + }); +}; + + +/** + * Encrypt a readable stream and any e-header headers. + * + * Parameters: + * - options: + * - cipher: (string) encryption algorithm to use, refer to RFD 71 for list + * - key: (string) raw encryption key value + * - keyId: (string) identifier for the key, will be saved with object + * - hmacType: (string) type of hmac algorithm to use + * - contentLength: (number) original size of input stream in bytes + * - headers: (object) raw request headers, *will be mutated* + * - "e-header" headers are encrypted and deleted + * - input: readable stream to encrypt + * - cb: callback of the form f(err, output) + */ +exports.encrypt = function encrypt(options, input, cb) { + assert.object(options, 'options'); + assert.stream(input, 'input'); + assert.string(options.cipher, 'options.cipher'); + assert.string(options.key, 'options.key'); + assert.string(options.keyId, 'options.keyId'); + assert.optionalString(options.hmacType, 'options.hmacType'); + assert.optionalNumber(options.contentLength, 'options.contentLength'); + assert.object(options.headers, 'options.headers'); + + var algorithm = getAlgorithm(options.cipher); + if (!algorithm) { + throw new Error('Unsupported cipher algorithm: ' + options.cipher); + } + + assert.ok(Buffer.byteLength(options.key) === algorithm.keyBytes, + 'key size must be ' + algorithm.keyBytes + ' bytes'); + + var hmacType = getHmacType(options.hmacType || 'HmacSHA256'); + if (util.isError(hmacType)) { + throw hmacType; + } + + var iv = crypto.randomBytes(algorithm.ivBytes); + var cipher = crypto.createCipheriv(algorithm.string, options.key, iv); + var hmac = null; + var output = new PassThrough(); + + // only calculate hmac when not using AEAD cipher + if (!algorithm.tagBytes) { + options.headers['m-encrypt-hmac-type'] = hmacType.type; + hmac = crypto.createHmac(hmacType.algorithm, options.key); + hmac.update(iv); + input.on('data', handleInputData); + } else { + options.headers['m-encrypt-aead-tag-length'] = algorithm.tagBytes; + } + + function handleInputData(data) { + hmac.update(data); + } + + cipher.once('error', function (err) { + input.removeListener('data', handleInputData); + output.emit('error', new VError(err, 'failed reading cipher')); + }); + + cipher.once('end', function (data) { + // when the algorithm is an AEAD one, write the auth tag + if (algorithm.tagBytes) { + var authTag = cipher.getAuthTag(); + if (!authTag) { + output.emit('error', new Error('Failed to get auth tag')); + } else { + output.write(authTag); + } + } else { + var digest = hmac.digest(); + assert.ok(Buffer.byteLength(digest) === hmacType.bytes, + 'hmac digest not expected size. expected bytes: ' + + hmacType.bytes + ', actual bytes: ' + Buffer.byteLength(digest)); + + // Append the digest to the end of the payload + output.write(digest); + } + }); + + var originalContentLength = options.contentLength || + options.headers['content-length']; + + // If not chunked encoding, calculate content-length with hmac/auth bytes + if (originalContentLength) { + calculateContentLength(originalContentLength, options.headers, + algorithm, hmacType.bytes); + } + + options.headers['m-encrypt-type'] = 'client/' + VERSION; + options.headers['m-encrypt-key-id'] = options.keyId; + options.headers['m-encrypt-iv'] = new Buffer(iv).toString('base64'); + options.headers['m-encrypt-cipher'] = options.cipher; + + encryptMetadata(algorithm, hmacType, options.headers, options.key); + input.pipe(cipher).pipe(output); + cb(null, output); +}; + + +/** + * Determines if the response is encrypted and can be decrypted by this module + * + * Parameters: + * - headers: (object) raw response headers + * Returns: + * boolean indicating if the response is encrypted and if this module can + * decrypt the response. + */ +exports.isSupported = function isSupported(headers) { + var encTypes = headers['m-encrypt-type'] ? + headers['m-encrypt-type'].split('/') : []; + + assert.ok(encTypes.length === 0 || encTypes.length === 2, 'm-encrypt-type' + + ' header must have a single / separator'); + + return (encTypes[0] === 'client' && isSupportedVersion(encTypes[1])); +}; + + +function isSupportedVersion(version) { + if (!/\d/.test(version)) { + return (false); + } + + var major = parseInt(version, 10); + + return (major === VERSION); +} + + +function validateHeaders(headers) { + var missingHeaders = []; + REQUIRED_HEADERS.forEach(function (header) { + if (headers[header] === undefined || headers[header] === null) { + missingHeaders.push(header); + } + }); + + return (missingHeaders.length ? missingHeaders : null); +} + + +function decryptMetadata(algorithm, hmacType, headers, key) { + if (!headers['m-encrypt-metadata']) { + return (false); + } + + var decipher = crypto.createDecipheriv(algorithm.string, key, + new Buffer(headers['m-encrypt-metadata-iv'], 'base64')); + var encrypted = new Buffer(headers['m-encrypt-metadata'], 'base64'); + var hmac = null; + + if (algorithm.tagBytes) { + var offset = Buffer.byteLength(encrypted) - algorithm.tagBytes; + var authTag = encrypted.slice(offset); + + encrypted = encrypted.slice(0, offset); + decipher.setAuthTag(authTag); + } else { + hmac = crypto.createHmac(hmacType.algorithm, key); + hmac.update(encrypted); + if (headers['m-encrypt-metadata-hmac'] !== hmac.digest('base64')) { + return (new Error('m-encrypt-metadata-hmac doesn\'t match')); + } + } + + var decrypted = decipher.update(encrypted); + decrypted = Buffer.concat([decrypted, decipher.final()]); + var deserializedHeaders = deserializeHeaders(decrypted.toString()); + var headerKeys = Object.keys(deserializedHeaders) || []; + for (var i = 0, il = headerKeys.length; i < il; ++i) { + var headerKey = headerKeys[i]; + var value = deserializedHeaders[headerKey]; + headers[headerKey] = value; + } + + delete headers['m-encrypt-metadata-iv']; + delete headers['m-encrypt-metadata-hmac']; + delete headers['m-encrypt-metadata']; + delete headers['m-encrypt-metadata-aead-tag-length']; + + if (headers['e-content-type']) { + headers['content-type'] = headers['e-content-type']; + delete headers['e-content-type']; + } + + // style dictates we must always return a value, false indicates success + return (false); +} + + +function encryptMetadata(algorithm, hmacType, headers, key) { + var iv = crypto.randomBytes(algorithm.ivBytes); + headers['m-encrypt-metadata-iv'] = new Buffer(iv).toString('base64'); + var cipher = crypto.createCipheriv(algorithm.string, key, iv); + var hmac = crypto.createHmac(hmacType.algorithm, key); + + var keysToEncrypt = []; + var headersToEncrypt = {}; + Object.keys(headers).forEach(function (headerKey) { + if (METADATA_PATTERN.test(headerKey)) { + keysToEncrypt.push(headerKey); + headersToEncrypt[headerKey] = headers[headerKey]; + } + }); + + var serializedHeaders = serializeHeaders(headersToEncrypt); + var encrypted = cipher.update(serializedHeaders); + + encrypted = Buffer.concat([encrypted, cipher.final()]); + if (algorithm.tagBytes) { + headers['m-encrypt-metadata-aead-tag-length'] = algorithm.tagBytes; + encrypted = Buffer.concat([encrypted, cipher.getAuthTag()]); + } else { + hmac.update(encrypted); + headers['m-encrypt-metadata-hmac'] = hmac.digest('base64'); + } + + headers['m-encrypt-metadata'] = encrypted.toString('base64'); + + keysToEncrypt.forEach(function (keyToDelete) { + delete headers[keyToDelete]; + }); +} + + +function serializeHeaders(headers) { + var result = ''; + var keys = Object.keys(headers) || []; + for (var i = 0, il = keys.length; i < il; ++i) { + var key = keys[i]; + var value = headers[key]; + result += key + ': ' + value + '\n'; + } + + return (result); +} + + +function deserializeHeaders(serializedHeaders) { + var result = {}; + var headers = serializedHeaders.split('\n') || []; + headers.forEach(function (headerStr) { + var header = headerStr.split(': '); + var key = header[0]; + var value = header[1]; + if (key !== '') { + result[key] = value; + } + }); + + return (result); +} + + +function getAlgorithm(cipher) { + cipher = cipher.toUpperCase(); + return (CIPHERS.hasOwnProperty(cipher) && CIPHERS[cipher]); +} + + +function getHmacType(hmac) { + hmac = hmac.toLowerCase(); + for (var i = 0, il = HMACS.length; i < il; ++i) { + var hmacType = HMACS[i]; + if (hmacType.type.toLowerCase() === hmac) { + return (hmacType); + } + } + + var validHmacs = HMACS.map(function (hmacObj) { + return (hmacObj.type); + }); + + return new Error('Unsupported HMAC: ' + hmac + '. Valid HMACs are ' + + validHmacs.join(', ')); +} + + +function calculateContentLength(originalContentLength, headers, algorithm, + hmacBytes) { + + originalContentLength = parseInt(originalContentLength, 10) || 0; + headers['m-encrypt-plaintext-content-length'] = originalContentLength; + + var tagOrHmacBytes = (algorithm.tagBytes || hmacBytes); + var calculatedContentLength = originalContentLength + tagOrHmacBytes; + + // Calculate content-length for padded algorithms + if (algorithm.isPadded) { + var padding = 0; + if (originalContentLength > algorithm.blockBytes) { + padding = originalContentLength % algorithm.blockBytes; + } else { + calculatedContentLength = algorithm.blockBytes; + } + + // e.g. content is 20 bytes, block is 16, padding is 4, result = 32 + if (padding) { + calculatedContentLength = (originalContentLength - padding) + + algorithm.blockBytes; + } + + calculatedContentLength += tagOrHmacBytes; + } + + headers['content-length'] = calculatedContentLength; +} diff --git a/lib/index.js b/lib/index.js index 54d278a..a445f4b 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,7 +1,8 @@ -// Copyright (c) 2013, Joyent, Inc. All rights reserved. +// Copyright (c) 2017, Joyent, Inc. All rights reserved. var auth = require('smartdc-auth'); var cc = require('./create_client'); +var cse = require('./cse'); var manta = require('./client'); var progbar = require('progbar'); var Queue = require('./queue'); @@ -16,6 +17,7 @@ module.exports = { MantaClient: manta.MantaClient, Queue: Queue, ProgressBar: progbar.ProgressBar, + encryptValidateAuthMode: cse.validateAuthMode, createClient: cc.createClient, createClientFromFileSync: cc.createClientFromFileSync, checkBinEnv: cc.checkBinEnv, @@ -45,6 +47,7 @@ module.exports = { cli_logger: cc.setupLogger, cliVersionCheckPrintAndExit: cc.versionCheckPrintAndExit, cliCompletionCheckPrintAndExit: cc.completionCheckPrintAndExit, + encryptValidateAuthMode: cse.validateAuthMode, StringStream: StringStream, path: manta.path, jobPath: manta.jobPath, diff --git a/lib/parse_etm_stream.js b/lib/parse_etm_stream.js new file mode 100644 index 0000000..b44c6b7 --- /dev/null +++ b/lib/parse_etm_stream.js @@ -0,0 +1,89 @@ +// Copyright 2017 Joyent, Inc. + +var assert = require('assert-plus'); +var stream = require('stream'); +var util = require('util'); + + +// Takes cipher + hmac/authTag stream of data and untangles the two +function ParseEtMStream(hmacType, contentLength, tagBytes, options) { + assert.optionalNumber(tagBytes, 'tagBytes'); + if (!tagBytes) { + assert.object(hmacType, 'hmacType'); + assert.number(hmacType.bytes, 'hmacType.bytes'); + } + + contentLength = parseInt(contentLength, 10); + assert.number(contentLength); + + this._tagBytes = tagBytes || 0; + this._offset = (contentLength - (this._tagBytes || hmacType.bytes)); + this._digestOrTag = new Buffer(''); + this._bytesRead = 0; + this._contentLength = contentLength; + + stream.Transform.call(this, options); +} +util.inherits(ParseEtMStream, stream.Transform); + + +// Pass the chunks through until you have reached the offset for the hmac +// After the offset is reached, store the chunks in the _digest variable +ParseEtMStream.prototype._transform = + function _transform(chunk, encoding, callback) { + + var chunkSize = Buffer.byteLength(chunk); + + // Check if we have reached the offset + if ((chunkSize + this._bytesRead) <= this._offset) { + this._bytesRead += chunkSize; + callback(null, chunk); + return; + } + + // Get number of bytes to read from the chunk into the cipher stream + var bytesForCipher = this._offset - this._bytesRead; + this._bytesRead += chunkSize; + + if (bytesForCipher > 0) { + var cipher = chunk.slice(0, bytesForCipher); + var digestOrTag = chunk.slice(bytesForCipher); + this._digestOrTag = Buffer.concat([this._digestOrTag, digestOrTag]); + this._tryEmitTag(); + + callback(null, cipher); + return; + } + + this._digestOrTag = Buffer.concat([this._digestOrTag, chunk]); + this._tryEmitTag(); + + // Mark the stream as processed + if (this._bytesRead === this._contentLength) { + this.push(null); + } + + callback(); +}; + + +ParseEtMStream.prototype.digest = function digest() { + return (this._digestOrTag); +}; + + +ParseEtMStream.prototype.tag = function tag() { + return (this._digestOrTag); +}; + + +ParseEtMStream.prototype._tryEmitTag = function _tryEmitTag() { + if (this._tagBytes && !this._tagEmitted && + Buffer.byteLength(this._digestOrTag) >= this._tagBytes) { + + this._tagEmitted = true; + this.emit('tag', this.tag()); + } +}; + +module.exports = ParseEtMStream; diff --git a/test/client.test.js b/test/client.test.js index 7b27aab..e5217d8 100644 --- a/test/client.test.js +++ b/test/client.test.js @@ -20,11 +20,15 @@ var manta = require('../lib'); var log = logging.createLogger(); +// Only GCM encryption supported after node v1.0.0 +var NODE_MAJOR = parseInt(process.versions.node.split('.')[0], 10); + var JOB; var ROOT = '/' + (process.env.MANTA_USER || 'admin') + '/stor'; var PUBLIC = '/' + (process.env.MANTA_USER || 'admin') + '/public'; var SUBDIR1 = ROOT + '/node-manta-test-client-' + libuuid.v4().split('-')[0]; var SUBDIR2 = SUBDIR1 + '/subdir2-' + libuuid.v4().split('-')[0]; // directory +var SUBDIRCSE = ROOT + '/node-manta-test-cse-' + libuuid.v4().split('-')[0]; var CHILD1 = SUBDIR1 + '/child1-' + libuuid.v4().split('-')[0]; // object var CHILD2 = SUBDIR2 + '/child2-' + libuuid.v4().split('-')[0]; // link var NOENTSUB1 = SUBDIR1 + '/a/b/c'; @@ -33,6 +37,7 @@ var SPECIALOBJ1 = SUBDIR1 + '/' + 'before-\r-after'; var SUBDIR1_NOBJECTS = 1; var SUBDIR1_NDIRECTORIES = 2; +var CSE_KEY = 'FFFFFFFBD96783C6C91E222211112222'; /* @@ -52,6 +57,9 @@ module.exports.setUp = function (cb) { var self = this; var url = process.env.MANTA_URL || 'http://localhost:8080'; var user = process.env.MANTA_USER || 'admin'; + function getKey(keyId, next) { + next(null, CSE_KEY); + } function createClient(signer) { self.client = manta.createClient({ @@ -61,7 +69,10 @@ module.exports.setUp = function (cb) { false : true), sign: signer, url: url, - user: user + user: user, + encrypt: { + getKey: getKey + } }); cb(); @@ -129,6 +140,13 @@ test('mkdir (sub)', function (t) { }); }); +test('mkdir (cse)', function (t) { + this.client.mkdir(SUBDIRCSE, function (err) { + t.ifError(err); + t.done(); + }); +}); + test('put', function (t) { var text = 'The lazy brown fox \nsomething \nsomething foo'; @@ -163,6 +181,59 @@ test('#231: put (special characters)', function (t) { }); }); +test('put (encrypt stream)', function (t) { + var text = 'The lazy brown fox \nsomething \nsomething foo'; + var stream = new MemoryStream(); + var fpath = SUBDIRCSE + '/encrypted'; + var options = { + encrypt: { + key: CSE_KEY, + keyId: 'dev/test', + cipher: 'AES256/CTR/NoPadding' + } + }; + + this.client.put(fpath, stream, options, function (err, res) { + t.ifError(err); + t.ok(res.req._headers['m-encrypt-key-id']); + t.done(); + }); + + process.nextTick(function () { + stream.write(text); + stream.end(); + }); +}); + +test('put (encrypt stream and metadata)', function (t) { + var text = 'The lazy brown fox \nsomething \nsomething foo'; + var stream = new MemoryStream(); + var fpath = SUBDIRCSE + '/metadata'; + var options = { + encrypt: { + key: CSE_KEY, + keyId: 'dev/test', + cipher: 'AES256/CTR/NoPadding' + }, + headers: { + 'e-hello1': 'world1', + 'e-hello2': 'world2' + } + }; + + this.client.put(fpath, stream, options, function (err, res) { + t.ifError(err); + t.ok(res.req._headers['m-encrypt-key-id']); + t.ok(res.req._headers['m-encrypt-metadata']); + t.done(); + }); + + process.nextTick(function () { + stream.write(text); + stream.end(); + }); +}); + test('#231: ls (special characters)', function (t) { this.client.ls(SUBDIR1, function (err, res) { t.ifError(err); @@ -196,6 +267,229 @@ test('#231: get (special characters)', function (t) { }); }); +test('get (range)', function (t) { + this.client.get(SPECIALOBJ1, { headers: { range: 'bytes=0-1' } }, + function (err, stream) { + + t.ifError(err); + + var data = ''; + stream.setEncoding('utf8'); + stream.on('data', function (chunk) { + data += chunk; + }); + stream.on('end', function (chunk) { + t.equal(data, 'my'); + t.done(); + }); + }); +}); + +test('get (decrypt stream & metadata)', function (t) { + var self = this; + var text = 'The lazy brown fox \nsomething \nsomething foo'; + var stream = new MemoryStream(); + var fpath = SUBDIRCSE + '/todecrypt-metadata'; + var key = CSE_KEY; + var options = { + encrypt: { + key: key, + keyId: 'dev/test', + cipher: 'AES256/CTR/NoPadding' + }, + headers: { + 'e-hello1': 'world1', + 'e-hello2': 'world2' + } + }; + + self.client.put(fpath, stream, options, function (putErr, putRes) { + t.ifError(putErr); + t.ok(putRes.req._headers['m-encrypt-key-id']); + setTimeout(function () { + self.client.get(fpath, function (getErr, decrypted, getRes) { + t.ifError(getErr); + + var result = ''; + decrypted.on('data', function (data) { + result += data.toString(); + }); + + decrypted.on('error', function (decErr) { + t.ifError(decErr); + }); + + decrypted.once('end', function () { + t.ok(result === text); + t.ok(getRes.headers['e-hello1'] === 'world1'); + t.ok(getRes.headers['e-hello2'] === 'world2'); + t.done(); + }); + }); + }, 10); + }); + + process.nextTick(function () { + stream.write(text); + stream.end(); + }); +}); + + +test('get (decrypt stream with range)', function (t) { + var self = this; + var text = 'The lazy brown fox \nsomething \nsomething foo'; + var stream = new MemoryStream(); + var fpath = SUBDIRCSE + '/todecrypt-range'; + var key = CSE_KEY; + var options = { + encrypt: { + key: key, + keyId: 'dev/test', + cipher: 'AES256/CTR/NoPadding' + }, + headers: { + 'e-hello1': 'world1', + 'e-hello2': 'world2' + } + }; + + self.client.put(fpath, stream, options, function (putErr, putRes) { + + t.ifError(putErr); + t.ok(putRes.req._headers['m-encrypt-key-id']); + setTimeout(function () { + self.client.get(fpath, { headers: { range: 'bytes=0-10' } }, + function (getErr, decrypted, getRes) { + + t.ifError(getErr); + + var result = ''; + getRes.on('data', function (data) { + result += data.toString(); + }); + + decrypted.on('error', function (decErr) { + t.ifError(decErr); + }); + + getRes.once('end', function () { + t.ok(result); + t.ok(getRes.headers['e-hello1'] === 'world1'); + t.ok(getRes.headers['e-hello2'] === 'world2'); + t.done(); + }); + }); + }, 10); + }); + + process.nextTick(function () { + stream.write(text); + stream.end(); + }); +}); + + +test('get (decrypt stream) by overriding getKey()', function (t) { + var self = this; + var text = 'The lazy brown fox'; + var stream = new MemoryStream(); + var fpath = SUBDIRCSE + '/todecrypt_getKey'; + var key = 'FFFFFFFBD96783C6C91E222211111111'; + var options = { + encrypt: { + key: key, + keyId: 'dev/test', + cipher: 'AES256/CTR/NoPadding' + } + }; + + function getKey(keyId, next) { + next(null, key); + } + + self.client.put(fpath, stream, options, function (putErr, putRes) { + t.ifError(putErr); + t.ok(putRes.req._headers['m-encrypt-key-id']); + setTimeout(function () { + self.client.get(fpath, {encrypt: {getKey: getKey}}, + function (getErr, decrypted, getRes) { + + t.ifError(getErr); + + var result = ''; + decrypted.on('data', function (data) { + result += data.toString(); + }); + + decrypted.once('end', function () { + t.ok(result === text); + t.done(); + }); + }); + }, 10); + }); + + process.nextTick(function () { + stream.write(text); + stream.end(); + }); +}); + +if (NODE_MAJOR) { + test('get (decrypt stream) with AEAD mode', function (t) { + var self = this; + var text = 'The lazy brown fox \nsomething \nsomething foo'; + var stream = new MemoryStream(); + var fpath = SUBDIRCSE + '/todecrypt-aead'; + var key = CSE_KEY; + var options = { + encrypt: { + key: key, + keyId: 'dev/test', + cipher: 'AES256/GCM/NoPadding' + }, + headers: { + 'e-hello1': 'world1', + 'e-hello2': 'world2', + 'e-hello3': 'world3' + } + }; + + self.client.put(fpath, stream, options, function (putErr, putRes) { + t.ifError(putErr); + t.ok(putRes.req._headers['m-encrypt-key-id']); + setTimeout(function () { + self.client.get(fpath, function (getErr, decrypted, getRes) { + t.ifError(getErr); + + var result = ''; + decrypted.on('data', function (data) { + result += data.toString(); + }); + + decrypted.on('error', function (decErr) { + t.ifError(decErr); + }); + + decrypted.once('end', function () { + t.ok(result === text); + t.ok(getRes.headers['e-hello1'] === 'world1'); + t.ok(getRes.headers['e-hello2'] === 'world2'); + t.ok(getRes.headers['e-hello3'] === 'world3'); + t.done(); + }); + }); + }, 10); + }); + + process.nextTick(function () { + stream.write(text); + stream.end(); + }); + }); +} + test('#231: rm (special characters)', function (t) { this.client.unlink(SPECIALOBJ1, function (err) { diff --git a/test/cse.test.js b/test/cse.test.js new file mode 100644 index 0000000..0d865e5 --- /dev/null +++ b/test/cse.test.js @@ -0,0 +1,432 @@ +// Copyright 2017 Joyent, Inc. + +var MemoryStream = require('readable-stream/passthrough.js'); +var cse = require('../lib/cse'); + + +// Only GCM encryption supported after node v1.0.0 +var NODE_MAJOR = parseInt(process.versions.node.split('.')[0], 10); + + +function test(name, testfunc) { + module.exports[name] = testfunc; +} + + +test('isSupported() returns false for invalid versions', function (t) { + var versions = [ + '', + null, + 'client/0', + 'client/0.', + 'client/b.b', + 'client/' + ]; + + versions.forEach(function (version) { + t.ok(!cse.isSupported({ 'm-encrypt-type': version })); + }); + t.done(); +}); + + +test('isSupported() returns true for valid versions', function (t) { + var versions = [ + 'client/1' + ]; + + versions.forEach(function (version) { + t.ok(cse.isSupported({ 'm-encrypt-type': version })); + }); + t.done(); +}); + + +test('encrypt() throws with missing options', function (t) { + var input = new MemoryStream(); + + t.throws(function () { + cse.encrypt(null, input, function (err, res) { + + }); + }, /options \(object\) is required/); + + t.done(); +}); + + +test('encrypt() throws with unsupported cipher alg', function (t) { + var options = { + key: 'FFFFFFFBD96783C6C91E222211112222', + cipher: 'AES/CFB/NoPadding', + keyId: 'something', + headers: {} + }; + var input = new MemoryStream(); + + t.throws(function () { + cse.encrypt(options, input, function (err, res) { + + }); + }, /Unsupported cipher algorithm/); + + t.done(); +}); + +test('encrypt() throws with alg "toString"', function (t) { + var options = { + key: 'FFFFFFFBD96783C6C91E222211112222', + cipher: 'toString', + keyId: 'something', + headers: {} + }; + var input = new MemoryStream(); + + t.throws(function () { + cse.encrypt(options, input, function (err, res) { + + }); + }, /Unsupported cipher algorithm/); + + t.done(); +}); + +test('encrypt() throws with invalid key length', function (t) { + var options = { + key: 'FFFFFF', + cipher: 'AES256/CTR/NoPadding', + keyId: 'something', + headers: {} + }; + var input = new MemoryStream(); + + t.throws(function () { + cse.encrypt(options, input, function (err, res) { + + }); + }, /key size must be/); + + t.done(); +}); + +test('encrypt() throws with invalid input', function (t) { + var options = { + key: 'FFFFFFFBD96783C6C91E222211112222', + keyId: 'dev/test', + cipher: 'AES256/CTR/NoPadding', + headers: {} + }; + + t.throws(function () { + cse.encrypt(options, null, function (err, res) { + + }); + }, /input \(stream\) is required/); + + t.done(); +}); + +test('encrypt() works for aes256 algorithm', function (t) { + var options = { + key: (new Buffer(32).fill('1')).toString(), + keyId: 'dev/test', + cipher: 'AES256/CTR/NoPadding', + headers: { + 'e-header': 'my value' + } + }; + var inputText = 'this is my text'; + var input = new MemoryStream(); + input.write(inputText); + + cse.encrypt(options, input, function (err, output) { + t.ifError(err); + input.end(); + + var result = ''; + output.on('data', function (data) { + result += data.toString(); + }); + + output.on('error', function (outErr) { + t.ifError(outErr); + }); + + output.once('end', function () { + t.ok(result !== inputText); + t.ok(options.headers['m-encrypt-hmac-type'] === 'HmacSHA256'); + t.ok(options.headers['m-encrypt-cipher'] === + 'AES256/CTR/NoPadding'); + t.ok(options.headers['m-encrypt-iv']); + t.ok(options.headers['m-encrypt-key-id'] === options.keyId); + t.ok(options.headers['m-encrypt-metadata']); + t.ok(options.headers['m-encrypt-metadata-hmac']); + t.ok(options.headers['m-encrypt-metadata-iv']); + t.done(); + }); + }); +}); + +if (NODE_MAJOR) { + test('encrypt() works for aes256-gcm algorithm', function (t) { + var options = { + key: (new Buffer(32).fill('1')).toString(), + keyId: 'dev/test', + cipher: 'AES256/GCM/NoPadding', + headers: { + 'e-header': 'my value' + } + }; + var inputText = 'this is my text'; + var input = new MemoryStream(); + input.write(inputText); + + cse.encrypt(options, input, function (err, output) { + t.ifError(err); + input.end(); + + var result = ''; + output.on('data', function (data) { + result += data.toString(); + }); + + output.on('error', function (outErr) { + t.ifError(outErr); + }); + + output.once('end', function () { + t.ok(result !== inputText); + t.ok(!options.headers['m-encrypt-hmac-type']); + t.ok(options.headers['m-encrypt-cipher'] === + 'AES256/GCM/NoPadding'); + t.ok(options.headers['m-encrypt-iv']); + t.ok(options.headers['m-encrypt-key-id'] === options.keyId); + t.ok(options.headers['m-encrypt-aead-tag-length'] === 16); + t.ok(options.headers['m-encrypt-metadata']); + t.ok(!options.headers['m-encrypt-metadata-hmac']); + t.ok(options.headers['m-encrypt-metadata-aead-tag-length']); + t.done(); + }); + }); + }); +} + +test('decrypt() throws with missing options', function (t) { + var input = new MemoryStream(); + + t.throws(function () { + cse.decrypt(null, input, { headers: {} }, function (err, res) { + + }); + }, /options \(object\) is required/); + + t.done(); +}); + +test('decrypt() throws with missing options.encrypt.getKey', function (t) { + var input = new MemoryStream(); + + t.throws(function () { + cse.decrypt({}, input, { headers: {} }, function (err, res) { + + }); + }, /options\.getKey \(func\) is required/); + + t.done(); +}); + +test('decrypt() throws with invalid input', function (t) { + var options = { + cse_getKey: function (keyId, cb) { + cb(); + } + }; + + t.throws(function () { + cse.decrypt(options, null, { headers: {} }, function (err, res) { + + }); + }, /encrypted \(stream\) is required/); + + t.done(); +}); + +test('decrypt() works for aes256-ctr algorithm', function (t) { + var key = (new Buffer(32).fill('1')).toString(); + + var getKey = function (keyId, cb) { + cb(null, key); + }; + var inputText = 'this is my text'; + var options = { + key: key, + keyId: 'dev/test', + cipher: 'AES256/CTR/NoPadding', + headers: { + 'e-header': 'my value', + 'content-length': Buffer.byteLength(new Buffer(inputText)) + } + }; + + var input = new MemoryStream(); + input.write(inputText); + + cse.encrypt(options, input, function (encErr, encrypted) { + t.ifError(encErr); + input.end(); + + var passthrough = new MemoryStream(); + + encrypted.once('end', function () { + var res = { + headers: options.headers + }; + t.ok(options.headers['m-encrypt-metadata']); + + cse.decrypt({ getKey: getKey }, passthrough, res, function (decErr, + decrypted, decRes) { + + t.ifError(decErr); + + var result = ''; + decrypted.on('data', function (data) { + result += data.toString(); + }); + + decrypted.on('error', function (outErr) { + t.ifError(outErr); + }); + + decrypted.once('end', function () { + t.ok(result === inputText); + t.ok(decRes.headers['e-header'] === 'my value'); + t.done(); + }); + }); + }); + + encrypted.pipe(passthrough); + }); +}); + +test('decrypt() works for aes256-cbc algorithm', function (t) { + var key = (new Buffer(32).fill('1')).toString(); + + var inputText = 'this is my text here'; + + var getKey = function (keyId, cb) { + cb(null, key); + }; + var options = { + key: key, + keyId: 'dev/test', + cipher: 'AES256/CBC/PKCS5Padding', + headers: { + 'e-header': 'my value', + 'content-length': Buffer.byteLength(new Buffer(inputText)) + } + }; + var input = new MemoryStream(); + input.write(inputText); + + cse.encrypt(options, input, function (encErr, encrypted) { + t.ifError(encErr); + input.end(); + + var passthrough = new MemoryStream(); + + encrypted.once('end', function () { + var res = { + headers: options.headers + }; + + t.ok(options.headers['m-encrypt-metadata']); + + cse.decrypt({ getKey: getKey }, passthrough, res, function (decErr, + decrypted, decRes) { + + t.ifError(decErr); + + var result = ''; + decrypted.on('data', function (data) { + result += data.toString(); + }); + + decrypted.on('error', function (outErr) { + t.ifError(outErr); + }); + + decrypted.once('end', function () { + t.ok(result === inputText); + t.ok(decRes.headers['e-header'] === 'my value'); + t.done(); + }); + }); + }); + + encrypted.pipe(passthrough); + }); +}); + +if (NODE_MAJOR) { + test('decrypt() works for aes256-gcm algorithm', function (t) { + var key = (new Buffer(32).fill('1')).toString(); + + var getKey = function (keyId, cb) { + cb(null, key); + }; + var options = { + key: key, + keyId: 'dev/test', + cipher: 'AES256/GCM/NoPadding', + headers: { + 'e-header': 'my value' + } + }; + var inputText = 'this is my text'; + var input = new MemoryStream(); + input.write(inputText); + + cse.encrypt(options, input, function (encErr, encrypted) { + t.ifError(encErr); + input.end(); + + var passthrough = new MemoryStream(); + var bytes = 0; + + encrypted.on('data', function (data) { + bytes += Buffer.byteLength(data); + }); + + passthrough.once('finish', function () { + var res = { + headers: options.headers + }; + res.headers['content-length'] = bytes; + t.ok(options.headers['m-encrypt-metadata']); + + cse.decrypt({ getKey: getKey }, passthrough, res, + function (decErr, decrypted, decRes) { + + t.ifError(decErr); + var result = ''; + decrypted.on('data', function (data) { + result += data.toString(); + }); + + decrypted.on('error', function (outErr) { + t.ifError(outErr); + }); + + decrypted.once('end', function () { + t.ok(result === inputText); + t.ok(decRes.headers['e-header'] === 'my value'); + t.done(); + }); + }); + }); + + encrypted.pipe(passthrough); + }); + }); +} diff --git a/test/parse_etm_stream.test.js b/test/parse_etm_stream.test.js new file mode 100644 index 0000000..82119dc --- /dev/null +++ b/test/parse_etm_stream.test.js @@ -0,0 +1,134 @@ +// Copyright 2017 Joyent, Inc. + +var MemoryStream = require('readable-stream/passthrough.js'); +var ParseEtMStream = require('../lib/parse_etm_stream'); + + +function test(name, testfunc) { + module.exports[name] = testfunc; +} + + +test('splits a digest from the parse_etm stream', function (t) { + var inputCipher = new Buffer(50); + var inputDigest = new Buffer(32); + inputCipher.fill('3'); + inputDigest.fill('4'); + + var input = new MemoryStream(); + var output = new ParseEtMStream({ bytes: 32 }, + Buffer.byteLength(inputCipher) + Buffer.byteLength(inputDigest)); + + var cipher = new Buffer(''); + output.on('data', function (data) { + cipher = Buffer.concat([cipher, data]); + }); + + output.once('end', function () { + t.equal(cipher.toString(), inputCipher.toString()); + t.equal(output.digest().toString(), inputDigest.toString()); + t.done(); + }); + + input.pipe(output); + input.write(inputCipher); + input.write(inputDigest); +}); + + +test('splits a multi-chunk digest from the parse_etm stream', function (t) { + var inputCipher = new Buffer(50); + var inputDigest1 = new Buffer(16); + var inputDigest2 = new Buffer(16); + inputCipher.fill('3'); + inputDigest1.fill('4'); + inputDigest2.fill('4'); + + var input = new MemoryStream(); + var output = new ParseEtMStream({ bytes: 32 }, + Buffer.byteLength(inputCipher) + Buffer.byteLength(inputDigest1) + + Buffer.byteLength(inputDigest2)); + + var cipher = new Buffer(''); + output.on('data', function (data) { + cipher = Buffer.concat([cipher, data]); + }); + + output.once('end', function () { + t.equal(cipher.toString(), inputCipher.toString()); + t.equal(output.digest().toString(), inputDigest1.toString() + + inputDigest2.toString()); + t.done(); + }); + + input.pipe(output); + input.write(inputCipher); + input.write(inputDigest1); + input.write(inputDigest2); +}); + +test('splits a multi-chunk digest from multi-chunk cipher', function (t) { + var inputCipher1 = new Buffer(50); + var inputCipher2 = new Buffer(50); + var inputDigest1 = new Buffer(16); + var inputDigest2 = new Buffer(16); + inputCipher1.fill('3'); + inputCipher2.fill('3'); + inputDigest1.fill('4'); + inputDigest2.fill('4'); + + var input = new MemoryStream(); + var output = new ParseEtMStream({ bytes: 32 }, + Buffer.byteLength(inputCipher1) + Buffer.byteLength(inputCipher2) + + Buffer.byteLength(inputDigest1) + Buffer.byteLength(inputDigest2)); + + var cipher = new Buffer(''); + output.on('data', function (data) { + cipher = Buffer.concat([cipher, data]); + }); + + output.once('end', function () { + t.equal(cipher.toString(), inputCipher1.toString() + + inputCipher2.toString()); + t.equal(output.digest().toString(), inputDigest1.toString() + + inputDigest2.toString()); + t.done(); + }); + + input.pipe(output); + input.write(inputCipher1); + input.write(inputCipher2); + input.write(inputDigest1); + input.write(inputDigest2); +}); + +test('splits a multi-chunk tag from chunked cipher', function (t) { + var inputCipher1 = new Buffer(50); + var inputCipher2 = new Buffer(50); + var inputTag = new Buffer(16); + inputCipher1.fill('3'); + inputCipher2.fill('3'); + inputTag.fill('4'); + + var input = new MemoryStream(); + var output = new ParseEtMStream({ bytes: 32 }, + Buffer.byteLength(inputCipher1) + Buffer.byteLength(inputCipher2) + + Buffer.byteLength(inputTag), 16); + + var cipher = new Buffer(''); + output.on('data', function (data) { + cipher = Buffer.concat([cipher, data]); + }); + + output.once('end', function () { + t.equal(cipher.toString(), inputCipher1.toString() + + inputCipher2.toString()); + t.equal(output.tag().toString(), inputTag.toString()); + t.done(); + }); + + input.pipe(output); + input.write(inputCipher1); + input.write(inputCipher2); + input.write(inputTag); +});