Added ability to pass in encrypted key/values into the CLI and API

CHANGELOG.rst

@@ -20,6 +20,21 @@ Added
 
 * Adding ``Cache-Control`` header to all API responses so clients will favor
   refresh from API instead of using cached version.
+* Add new ``--pre-encrypted`` flag to ``st2 key set`` CLI command that allows users to pass in
+  values which are already encrypted.
+
+  This attribute signals the API that the value is already encrypted and should be used as-is.
+
+  ``st2 key load`` CLI command has also been updated so it knows how to work with values which are
+  already encrypted. This means that ``st2 key list -n 100 -j < data.json ; st2 key load
+  data.json`` will now also work out of the box for encrypted datastore values (value which have
+  ``encrypted: True`` and ``secret: True`` attributes will be treated as pre-encrypted values as
+  treated as such).
+
+  The most common use case for this feature is migrating / restoring datastore values from one
+  StackStorm instance to another which uses the same crypto key.
+
+  Contributed by Nick Maludy (Encore Technologies)
 
 Changed
 ~~~~~~~

st2api/st2api/controllers/v1/keyvalue.py

@@ -274,6 +274,11 @@ def put(self, kvp, name, requester_user, scope=FULL_SYSTEM_SCOPE):
                                                              user=user,
                                                              require_rbac=True)
 
+        # Validate that pre_encrypted option can only be used by admins
+        pre_encrypted = getattr(kvp, 'pre_encrypted', False)
+        self._validate_pre_encrypted_query_parameter(pre_encrypted=pre_encrypted, scope=scope,
+                                                     requester_user=requester_user)
+
         key_ref = get_key_reference(scope=scope, name=name, user=user)
         lock_name = self._get_lock_name_for_key(name=key_ref, scope=scope)
         LOG.debug('PUT scope: %s, name: %s', scope, name)
@@ -409,6 +414,12 @@ def _validate_decrypt_query_parameter(self, decrypt, scope, requester_user):
             msg = 'Decrypt option requires administrator access'
             raise AccessDeniedError(message=msg, user_db=requester_user)
 
+    def _validate_pre_encrypted_query_parameter(self, pre_encrypted, scope, requester_user):
+        is_admin = rbac_utils.user_is_admin(user_db=requester_user)
+        if pre_encrypted and not is_admin:
+            msg = 'Pre-encrypted option requires administrator access'
+            raise AccessDeniedError(message=msg, user_db=requester_user)
+
     def _validate_scope(self, scope):
         if scope not in ALLOWED_SCOPES:
             msg = 'Scope %s is not in allowed scopes list: %s.' % (scope, ALLOWED_SCOPES)

st2api/tests/unit/controllers/v1/test_kvps.py

@@ -70,6 +70,23 @@
     'secret': True
 }
 
+# value = S3cret!Value
+# encrypted with st2tests/conf/st2_kvstore_tests.crypto.key.json
+ENCRYPTED_KVP = {
+    'name': 'secret_key1',
+    'value': ('3030303030298D848B45A24EDCD1A82FAB4E831E3FCE6E60956817A48A180E4C040801E'
+              'B30170DACF79498F30520236A629912C3584847098D'),
+    'pre_encrypted': True
+}
+
+ENCRYPTED_KVP_SECRET_FALSE = {
+    'name': 'secret_key2',
+    'value': ('3030303030298D848B45A24EDCD1A82FAB4E831E3FCE6E60956817A48A180E4C040801E'
+              'B30170DACF79498F30520236A629912C3584847098D'),
+    'secret': True,
+    'pre_encrypted': True
+}
+
 
 class KeyValuePairControllerTestCase(FunctionalTest):
 
@@ -468,6 +485,62 @@ def test_get_all_decrypt(self):
         self.__do_delete(kvp_id_1)
         self.__do_delete(kvp_id_2)
 
+    def test_put_pre_encrypted_value(self):
+        # 1. pre_encrypted=True, secret=True
+        put_resp = self.__do_put('secret_key1', ENCRYPTED_KVP)
+        kvp_id = self.__get_kvp_id(put_resp)
+
+        # Verify there is no secrets leakage
+        self.assertEqual(put_resp.status_code, 200)
+        self.assertEqual(put_resp.json['name'], 'secret_key1')
+        self.assertEqual(put_resp.json['scope'], 'st2kv.system')
+        self.assertEqual(put_resp.json['encrypted'], True)
+        self.assertEqual(put_resp.json['secret'], True)
+        self.assertEqual(put_resp.json['value'], ENCRYPTED_KVP['value'])
+        self.assertTrue(put_resp.json['value'] != 'S3cret!Value')
+        self.assertTrue(len(put_resp.json['value']) > len('S3cret!Value') * 2)
+
+        get_resp = self.__do_get_one(kvp_id + '?decrypt=True')
+        self.assertEqual(put_resp.json['name'], 'secret_key1')
+        self.assertEqual(put_resp.json['scope'], 'st2kv.system')
+        self.assertEqual(put_resp.json['encrypted'], True)
+        self.assertEqual(put_resp.json['secret'], True)
+        self.assertEqual(put_resp.json['value'], ENCRYPTED_KVP['value'])
+
+        # Verify data integrity post decryption
+        get_resp = self.__do_get_one(kvp_id + '?decrypt=True')
+        self.assertFalse(get_resp.json['encrypted'])
+        self.assertEqual(get_resp.json['value'], 'S3cret!Value')
+        self.__do_delete(self.__get_kvp_id(put_resp))
+
+        # 2. pre_encrypted=True, secret=False
+        # pre_encrypted should always imply secret=True
+        put_resp = self.__do_put('secret_key2', ENCRYPTED_KVP_SECRET_FALSE)
+        kvp_id = self.__get_kvp_id(put_resp)
+
+        # Verify there is no secrets leakage
+        self.assertEqual(put_resp.status_code, 200)
+        self.assertEqual(put_resp.json['name'], 'secret_key2')
+        self.assertEqual(put_resp.json['scope'], 'st2kv.system')
+        self.assertEqual(put_resp.json['encrypted'], True)
+        self.assertEqual(put_resp.json['secret'], True)
+        self.assertEqual(put_resp.json['value'], ENCRYPTED_KVP['value'])
+        self.assertTrue(put_resp.json['value'] != 'S3cret!Value')
+        self.assertTrue(len(put_resp.json['value']) > len('S3cret!Value') * 2)
+
+        get_resp = self.__do_get_one(kvp_id + '?decrypt=True')
+        self.assertEqual(put_resp.json['name'], 'secret_key2')
+        self.assertEqual(put_resp.json['scope'], 'st2kv.system')
+        self.assertEqual(put_resp.json['encrypted'], True)
+        self.assertEqual(put_resp.json['secret'], True)
+        self.assertEqual(put_resp.json['value'], ENCRYPTED_KVP['value'])
+
+        # Verify data integrity post decryption
+        get_resp = self.__do_get_one(kvp_id + '?decrypt=True')
+        self.assertFalse(get_resp.json['encrypted'])
+        self.assertEqual(get_resp.json['value'], 'S3cret!Value')
+        self.__do_delete(self.__get_kvp_id(put_resp))
+
     def test_put_delete(self):
         put_resp = self.__do_put('key1', KVP)
         self.assertEqual(put_resp.status_int, 200)

st2client/st2client/commands/keyvalue.py

@@ -84,7 +84,7 @@ def __init__(self, resource, *args, **kwargs):
         # Filter options
         self.parser.add_argument('--prefix', help=('Only return values with names starting with '
                                                    'the provided prefix.'))
-        self.parser.add_argument('--decrypt', action='store_true',
+        self.parser.add_argument('-d', '--decrypt', action='store_true',
                                  help='Decrypt secrets and displays plain text.')
         self.parser.add_argument('-s', '--scope', default=DEFAULT_LIST_SCOPE, dest='scope',
                                  help='Scope item is under. Example: "user".')
@@ -168,6 +168,10 @@ def __init__(self, resource, *args, **kwargs):
         self.parser.add_argument('-e', '--encrypt', dest='secret',
                                  action='store_true',
                                  help='Encrypt value before saving.')
+        self.parser.add_argument('--pre-encrypted', dest='pre_encrypted',
+                                 action='store_true',
+                                 help=('Value provided is already encrypted with the instance '
+                                 'crypto key and should be stored as-is.'))
         self.parser.add_argument('-s', '--scope', dest='scope', default=DEFAULT_CUD_SCOPE,
                                  help='Specify the scope under which you want ' +
                                       'to place the item.')
@@ -186,6 +190,9 @@ def run(self, args, **kwargs):
         if args.secret:
             instance.secret = args.secret
 
+        if args.pre_encrypted:
+            instance.pre_encrypted = args.pre_encrypted
+
         if args.ttl:
             instance.ttl = args.ttl
 
@@ -312,7 +319,9 @@ def run(self, args, **kwargs):
             # parse optional KeyValuePair properties
             scope = item.get('scope', DEFAULT_CUD_SCOPE)
             user = item.get('user', None)
+            encrypted = item.get('encrypted', False)
             secret = item.get('secret', False)
+            pre_encrypted = item.get('pre_encrypted', False)
             ttl = item.get('ttl', None)
 
             # if the value is not a string, convert it to JSON
@@ -332,13 +341,23 @@ def run(self, args, **kwargs):
             instance.name = name
             instance.value = value
             instance.scope = scope
+
             if user:
                 instance.user = user
+            if encrypted:
+                instance.encrypted = encrypted
             if secret:
                 instance.secret = secret
+            if pre_encrypted:
+                instance.pre_encrypted = pre_encrypted
             if ttl:
                 instance.ttl = ttl
 
+            # encrypted=True and secret=True implies that the value is already encrypted and should
+            # be used as such
+            if encrypted and secret:
+                instance.pre_encrypted = True
+
             # call the API to create/update the KeyValuePair
             self.manager.update(instance, **kwargs)
             instances.append(instance)

st2client/tests/unit/test_keyvalue.py

@@ -51,6 +51,15 @@
     'secret': True
 }
 
+KEYVALUE_PRE_ENCRYPTED = {
+    'id': 'kv_name',
+    'name': 'kv_name.',
+    'value': 'AAABBBCCC1234',
+    'scope': 'system',
+    'encrypted': True,
+    'secret': True
+}
+
 KEYVALUE_TTL = {
     'id': 'kv_name',
     'name': 'kv_name.',
@@ -69,10 +78,11 @@
 KEYVALUE_ALL = {
     'id': 'kv_name',
     'name': 'kv_name.',
-    'value': 'super cool value',
+    'value': 'AAAAABBBBBCCCCCCDDDDD11122345',
     'scope': 'system',
     'user': 'stanley',
     'secret': True,
+    'encrypted': True,
     'ttl': 100
 }
 
@@ -108,6 +118,20 @@ def tearDown(self):
         super(TestKeyValueBase, self).tearDown()
 
 
+class TestKeyValueSet(TestKeyValueBase):
+
+    @mock.patch.object(
+        requests, 'put',
+        mock.MagicMock(return_value=base.FakeResponse(json.dumps(KEYVALUE_PRE_ENCRYPTED), 200,
+                                                      'OK')))
+    def test_set_keyvalue(self):
+        """Test setting key/value pair with optional pre_encrypted field
+        """
+        args = ['key', 'set', '--pre-encrypted', 'kv_name', 'AAABBBCCC1234']
+        retcode = self.shell.run(args)
+        self.assertEqual(retcode, 0)
+
+
 class TestKeyValueLoad(TestKeyValueBase):
 
     @mock.patch.object(
@@ -182,6 +206,25 @@ def test_load_keyvalue_secret(self):
             os.close(fd)
             os.unlink(path)
 
+    @mock.patch.object(
+        requests, 'put',
+        mock.MagicMock(return_value=base.FakeResponse(json.dumps(KEYVALUE_PRE_ENCRYPTED), 200,
+                                                      'OK')))
+    def test_load_keyvalue_already_encrypted(self):
+        """Test loading of key/value pair with the pre-encrypted value
+        """
+        fd, path = tempfile.mkstemp(suffix='.json')
+        try:
+            with open(path, 'a') as f:
+                f.write(json.dumps(KEYVALUE_PRE_ENCRYPTED, indent=4))
+
+            args = ['key', 'load', path]
+            retcode = self.shell.run(args)
+            self.assertEqual(retcode, 0)
+        finally:
+            os.close(fd)
+            os.unlink(path)
+
     @mock.patch.object(
         requests, 'put',
         mock.MagicMock(return_value=base.FakeResponse(json.dumps(KEYVALUE_TTL), 200, 'OK')))

st2common/st2common/models/api/keyvalue.py

@@ -67,6 +67,11 @@ class KeyValuePairAPI(BaseAPI):
                 'required': False,
                 'default': False
             },
+            'pre_encrypted': {
+                'type': 'boolean',
+                'required': False,
+                'default': False
+            },
             'encrypted': {
                 'type': 'boolean',
                 'required': False,
@@ -157,6 +162,7 @@ def to_model(cls, kvp):
         name = getattr(kvp, 'name', None)
         description = getattr(kvp, 'description', None)
         value = kvp.value
+        original_value = value
         secret = False
 
         if getattr(kvp, 'ttl', None):
@@ -165,13 +171,32 @@ def to_model(cls, kvp):
         else:
             expire_timestamp = None
 
+        pre_encrypted = getattr(kvp, 'pre_encrypted', False)
         secret = getattr(kvp, 'secret', False)
 
-        if secret:
-            if not KeyValuePairAPI.crypto_key:
-                msg = ('Crypto key not found in %s. Unable to encrypt value for key %s.' %
-                       (KeyValuePairAPI.crypto_key_path, name))
-                raise CryptoKeyNotSetupException(msg)
+        # If user transmitted the value in an pre-encrypted format, we perform the decryption here
+        # to ensure data integrity. Besides that, we store data as-is.
+        # Keep in mind that pre_encrypted=True also always implies secret=True. If we didn't do
+        # that and supported pre_encrypted=True, secret=False, this would allow users to decrypt
+        # any encrypted value.
+        if pre_encrypted:
+            secret = True
+
+            cls._verif_key_is_set_up(name=name)
+
+            try:
+                symmetric_decrypt(KeyValuePairAPI.crypto_key, value)
+            except Exception:
+                msg = ('Failed to verify the integrity of the provided value for key "%s". Ensure '
+                       'that the value is encrypted with the correct key and not corrupted.' %
+                       (name))
+                raise ValueError(msg)
+
+            # Additional safety check to ensure that the value hasn't been decrypted
+            assert value == original_value
+        elif secret:
+            cls._verif_key_is_set_up(name=name)
+
             value = symmetric_encrypt(KeyValuePairAPI.crypto_key, value)
 
         scope = getattr(kvp, 'scope', FULL_SYSTEM_SCOPE)
@@ -181,12 +206,25 @@ def to_model(cls, kvp):
                 scope, ALLOWED_SCOPES)
             )
 
+        # NOTE: For security reasons, pre_encrypted always implies secret=True. See comment
+        # above for explanation.
+        if pre_encrypted and not secret:
+            raise ValueError('pre_encrypted option can only be used in combination with secret '
+                             'option')
+
         model = cls.model(id=kvp_id, name=name, description=description, value=value,
                           secret=secret, scope=scope,
                           expire_timestamp=expire_timestamp)
 
         return model
 
+    @classmethod
+    def _verif_key_is_set_up(cls, name):
+        if not KeyValuePairAPI.crypto_key:
+            msg = ('Crypto key not found in %s. Unable to encrypt / decrypt value for key %s.' %
+                   (KeyValuePairAPI.crypto_key_path, name))
+            raise CryptoKeyNotSetupException(msg)
+
 
 class KeyValuePairSetAPI(KeyValuePairAPI):
     """

st2common/st2common/openapi.yaml

@@ -4889,6 +4889,9 @@ definitions:
       secret:
         description: Encrypt value before saving the value.
         type: boolean
+      pre_encrypted:
+        description: Value provided is encrypted with this instances crypto key and should be stored as-is. This is useful when migrating / loading keys from one StackStorm instance to another.
+        type: boolean
       scope:
         description: "Scope the item is under. Example: 'user'."
         type: string

st2common/st2common/openapi.yaml.j2

@@ -4885,6 +4885,9 @@ definitions:
       secret:
         description: Encrypt value before saving the value.
         type: boolean
+      decrypt:
+        description: Value provided is encrypted with this instances crypto key and must be decrypted when received. This allows values to be transmitted in encrypted format.
+        type: boolean
       scope:
         description: "Scope the item is under. Example: 'user'."
         type: string