joyent/node-manta#296 add client encryption support

.gitignore

@@ -11,3 +11,4 @@ smf/manifests/bapi.xml
 /test/node.paths
 /npm-debug.log
 /share/manta.completion
+.DS_Store

README.md

@@ -87,7 +87,7 @@ A full set of commands for interacting with Manta is in `bin`.
 # More documentation
 
 Docs can be found here:
-[http://apidocs.joyent.com/manta/](http://apidocs.joyent.com/manta/)
+[https://apidocs.joyent.com/manta/](https://apidocs.joyent.com/manta/)
 
 
 # Testing

bin/mget

@@ -8,6 +8,7 @@ var fs = require('fs');
 var http = require('http');
 var path = require('path-platform');
 var url = require('url');
+var util = require('util');
 
 var bunyan = require('bunyan');
 var dashdash = require('dashdash');
@@ -25,8 +26,46 @@ var LOG = bunyan.createLogger({
     stream: process.stderr
 });
 
+
+var ENCRYPT_AUTH_MODES = ['MandatoryAuthentication', 'OptionalAuthentication'];
+
+dashdash.addOptionType({
+    name: 'encryptAuthMode',
+    takesArg: true,
+    helpArg: 'MODE',
+    parseArg: function parseEncryptAuthMode(option, optstr, arg) {
+        manta.encryptValidateAuthMode(arg);
+        return (arg);
+    }
+});
+
 var OPTIONS_PARSER = dashdash.createParser({
     options: manta.DEFAULT_CLI_OPTIONS.concat([
+        {
+            group: 'Client-side encryption options'
+        },
+        {
+            names: ['decrypt', 'e'],
+            type: 'bool',
+            help: 'Expect the object to be encrypted, and decrypt it.'
+        },
+        {
+            names: ['encrypt-key'],
+            type: 'string',
+            env: 'MANTA_ENCRYPT_KEY',
+            helpArg: 'KEY',
+            help: 'Base64 secret key for decrypting remote encrypted objects.'
+        },
+        {
+            names: ['encrypt-auth-mode'],
+            type: 'encryptAuthMode',
+            env: 'MANTA_ENCRYPT_AUTH_MODE',
+            helpArg: 'MODE',
+            help: 'Whether decryption will enforce authentication '
+                + '("MandatoryAuthentication", the default) or allow some '
+                + 'operations, e.g. range requests, which cannot enforce '
+                + 'authentication ("OptionalAuthentication")'
+        },
         {
             group: NAME + ' options'
         },
@@ -55,6 +94,11 @@ var OPTIONS_PARSER = dashdash.createParser({
             names: ['remote-name', 'O'],
             type: 'bool',
             help: 'write output to a file using remote object name as filename'
+        },
+        {
+            names: ['include'],
+            type: 'bool',
+            help: 'Include the HTTP-header in the output.'
         }
     ])
 });
@@ -114,16 +158,37 @@ function parseOptions() {
         opts.headers[tmp[0]] = tmp[1].trim();
     });
 
+    if (opts.decrypt) {
+        if (!opts.encrypt_key) {
+            manta.cli_usage(OPTIONS_PARSER, 'decryption requires ' +
+                '--encrypt-key or the MANTA_ENCRYPT_KEY environment variable');
+        }
+        var base64Key = new Buffer(opts.encrypt_key, 'base64');
+        opts.encrypt = {
+            getKey: function (keyId, cb) {
+                cb(null, base64Key);
+            },
+            authMode: opts.encrypt_auth_mode
+        };
+    }
+
     return (opts);
 }
 
 
-function printEntry(obj) {
-    console.log('%j', obj);
+// The same as `printEntry` from minfo.
+function printHeaders(res) {
+    console.log('HTTP/%s %s %s',
+                res.httpVersion,
+                res.statusCode,
+                http.STATUS_CODES[res.statusCode]);
+    Object.keys(res.headers).forEach(function (k) {
+        console.log('%s: %s', k, res.headers[k]);
+    });
+    console.log();
 }
 
 
-
 ///--- Mainline
 
 (function main() {
@@ -143,6 +208,10 @@ function printEntry(obj) {
         client.get(p, function (err, stream, res) {
             ifError(err);
 
+            if (opts.include) {
+                printHeaders(res);
+            }
+
             var bar;
             var src = stream;
             if (opts.progress || drawProgressBar) {
@@ -162,6 +231,10 @@ function printEntry(obj) {
                 src = stream.pipe(bar.stream());
             }
 
+            src.on('error', function (srcErr) {
+                ifError(srcErr);
+            });
+
             src.pipe(out);
 
             src.on('end', function () {

bin/mput

@@ -29,6 +29,45 @@ var LOG = bunyan.createLogger({
 
 var OPTIONS_PARSER = dashdash.createParser({
     options: manta.DEFAULT_CLI_OPTIONS.concat([
+        {
+            group: 'Client-side encryption options'
+        },
+        {
+            names: ['encrypt', 'e'],
+            type: 'bool',
+            help: 'Encrypt the file before storing it in Manta, using ' +
+            '--encrypt-* options.'
+        },
+        {
+            names: ['encrypt-cipher'],
+            type: 'string',
+            env: 'MANTA_ENCRYPT_CIPHER',
+            helpArg: 'CIPHER',
+            help: 'Algorithm to use for encrypting file.'
+        },
+        {
+            names: ['encrypt-hmac'],
+            type: 'string',
+            env: 'MANTA_ENCRYPT_HMAC',
+            helpArg: 'HMAC',
+            help: 'HMAC type to use when encrypting a file. Types are ' +
+                'HmacMD5, HmacSHA1, HmacSHA256, and HmacSHA512'
+        },
+        {
+            names: ['encrypt-key'],
+            type: 'string',
+            env: 'MANTA_ENCRYPT_KEY',
+            helpArg: 'KEY',
+            help: 'Base64 secret key for encrypting content.'
+        },
+        {
+            names: ['encrypt-key-id'],
+            type: 'string',
+            env: 'MANTA_ENCRYPT_KEY_ID',
+            helpArg: 'KEY_ID',
+            help: 'Key identifier. This can be used for locating the key ' +
+                'for decryption.'
+        },
         {
             group: NAME + ' options'
         },
@@ -146,6 +185,21 @@ function parseOptions() {
         opts['role-tag'] = opts['role-tag'][0].split(/\s*,\s*/);
     }
 
+    if (opts.encrypt) {
+        if (!opts.encrypt_key || !opts.encrypt_key_id || !opts.encrypt_cipher) {
+            manta.cli_usage(OPTIONS_PARSER, 'encrypt requires --encrypt-key, ' +
+              '--encrypt-key-id, and --encrypt-cipher or related ' +
+              'environment variables');
+        }
+
+        opts.encrypt = {
+            key: (new Buffer(opts.encrypt_key, 'base64').toString()),
+            keyId: opts.encrypt_key_id,
+            cipher: opts.encrypt_cipher,
+            hmacType: opts.encrypt_hmac
+        };
+    }
+
     return (opts);
 }
 
@@ -244,7 +298,7 @@ function printEntry(obj) {
                 };
                 var fstream = fs.createReadStream(options.file, f_opts);
                 fstream.pause();
-                fstream.on('open', function () {
+                fstream.once('open', function () {
                     put(fstream, stats, cb);
                 });
             }

docs/man/mget.md

@@ -45,6 +45,16 @@ OPTIONS
 `-a, --account login`
   Authenticate as account (login name).
 
+`--encrypt-key=KEY`
+  Base64 encoded key used to encrypt the file. Will decrypt the stored object
+  when it's encrypted. If this value is missing then the original encrypted
+  object is returned.
+
+`--encrypt-auth-mode=MODE`
+  Determines if the decrypted object size is strictly enforced to be the same
+  as the original (pre-encrypted) object size. Defaults to
+  "MandatoryAuthentication", "OptionalAuthentication" disables strict mode.
+
 `-H, --headers`
   Print HTTP headers on stderr.
 
@@ -60,9 +70,6 @@ OPTIONS
   Authenticate using the SSH key described by FINGERPRINT.  The key must
   either be in `~/.ssh` or loaded in the SSH agent via `ssh-add`.
 
-`--role=ROLE,ROLE,...`
-  Specify which roles to assume for the request.
-
 `-o, --output file`
   Write output to <file> instead of stdout.
 
@@ -73,6 +80,9 @@ OPTIONS
 `-q, --quiet`
   Do not display a progress meter.
 
+`--role=ROLE,ROLE,...`
+  Specify which roles to assume for the request.
+
 `--user user`
   Authenticate as user under account.
 
@@ -84,24 +94,31 @@ OPTIONS
 
 ENVIRONMENT
 -----------
-`MANTA_USER`
-  In place of `-a, --account`
 
-`MANTA_SUBUSER`
-  In place of `--user`.
+`MANTA_ENCRYPT_KEY`
+  In place of `--encrypt-key`
+
+`MANTA_ENCRYPT_AUTH_MODE`
+  In place of `--encrypt-auth-mode`
 
 `MANTA_KEY_ID`
   In place of `-k, --key`.
 
 `MANTA_ROLE`
   In place of `--role`.
 
-`MANTA_URL`
-  In place of `-u, --url`.
+`MANTA_SUBUSER`
+  In place of `--user`.
 
 `MANTA_TLS_INSECURE`
   In place of `-i, --insecure`.
 
+`MANTA_URL`
+  In place of `-u, --url`.
+
+`MANTA_USER`
+  In place of `-a, --account`
+
 The shortcut `~~` is equivalent to `/:login`
 where `:login` is the account login name.
 

docs/man/mput.md

@@ -53,6 +53,22 @@ OPTIONS
 `-c, --copies file`
   Create COPIES copies as a replication factor (default 2).
 
+`-e, --encrypt`
+  Encrypt the file using the provided key, key ID, and algorithm. Any encrypted
+  file will be stored with a m-encrypt-type header set to client/1.
+
+`--encrypt-cipher=CIPHER`
+  Encryption algorithm to use.
+
+`--encrypt-hmac=HMAC`
+  HMAC algorithm to use for none authentication ciphers (default HmacSHA256).
+
+`--encrypt-key=KEY`
+  Base64 encoded key used to encrypt the file.
+
+`--encrypt-key-id=ID`
+  Value to uniquely identify the key used for encryption.
+
 `-f, --file file`
   Create contents of object from file.
 
@@ -96,6 +112,15 @@ OPTIONS
 ENVIRONMENT
 -----------
 
+`MANTA_ENCRYPT_CIPHER`
+  In place of `--encrypt-cipher`
+
+`MANTA_ENCRYPT_KEY`
+  In place of `--encrypt-key`
+
+`MANTA_ENCRYPT_KEY_ID`
+  In place of `--encrypt-key-id`
+
 `MANTA_USER`
   In place of `-a, --account`
 
@@ -117,6 +142,58 @@ ENVIRONMENT
 The shortcut `~~` is equivalent to `/:login`
 where `:login` is the account login name.
 
+CIPHER STRINGS
+--------------
+
+Below is a list of supported cipher strings to use in the `--encrypt-cipher`
+option. Each uses a 16 byte block size and a 16 byte initialization vector.
+The strings are case-insensitive as well.
+
+`AES128/GCM/NoPadding`
+  16 byte key size
+
+`AES192/GCM/NoPadding`
+  24 byte key size
+
+`AES256/GCM/NoPadding`
+  32 byte key size
+
+`AES128/CTR/NoPadding`
+  16 byte key size
+
+`AES192/CTR/NoPadding`
+  24 byte key size
+
+`AES256/CTR/NoPadding`
+  32 byte key size
+
+`AES128/CBC/PKCS5Padding`
+  16 byte key size
+
+`AES192/CBC/PKCS5Padding`
+  24 byte key size
+
+`AES256/CBC/PKCS5Padding`
+  32 byte key size
+
+HMAC STRINGS
+------------
+
+Below is a list of supported HMAC strings to use when using not using a GCM
+algorithm.
+
+`HmacMD5`
+  MD5 digest of 16 bytes
+
+`HmacSHA1`
+  SHA1 digest of 20 bytes
+
+`HmacSHA256`
+  SHA256 digest of 32 bytes
+
+`HmacSHA512`
+  SHA512 digest of 64 bytes
+
 DIAGNOSTICS
 -----------