flakiness analyzer: Add kms for encrypt/decrypt secret keys.
BUG=chromium:818020
TEST=Ran unittest.
Ran integration tests.
Change-Id: If18aaeedb2b138c91c42c2a843766997bc3196d7
Reviewed-on: https://chromium-review.googlesource.com/980805
Tested-by: Xixuan Wu <xixuan@chromium.org>
Reviewed-by: Paul Hobbs <phobbs@google.com>
Commit-Queue: Xixuan Wu <xixuan@chromium.org>
diff --git a/.gitignore b/.gitignore
index b4e4bf8..0c346d7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,3 @@
*.pyc
-test_analyzer/credentials/
+test_analyzer/credentials/non_cipher
__pycache__/
diff --git a/Pipfile b/Pipfile
index c905bf1..c351a0e 100644
--- a/Pipfile
+++ b/Pipfile
@@ -16,6 +16,7 @@
[packages]
+google-api-python-client = "~=1.6.5"
google-cloud-bigquery = "~=0.25.0"
diff --git a/Pipfile.lock b/Pipfile.lock
index b9d44c0..22d740f 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
- "sha256": "2f64695858160b2c29a7527765b8b9caed71e1106306c69306ab5e17027dd5bf"
+ "sha256": "c4c2efe8c83baf7df8434ce50a2244e384ed27afbcf389e0ea00da9459173b18"
},
"host-environment-markers": {
"implementation_name": "cpython",
@@ -9,9 +9,9 @@
"os_name": "posix",
"platform_machine": "x86_64",
"platform_python_implementation": "CPython",
- "platform_release": "4.9.0-5-amd64",
+ "platform_release": "4.9.0-6-amd64",
"platform_system": "Linux",
- "platform_version": "#1 SMP Debian 4.9.65-3+deb9u2 (2018-01-04)",
+ "platform_version": "#1 SMP Debian 4.9.82-1+deb9u2 (2018-02-21)",
"python_full_version": "2.7.13",
"python_version": "2.7",
"sys_platform": "linux2"
@@ -36,6 +36,13 @@
],
"version": "==2.0.1"
},
+ "google-api-python-client": {
+ "hashes": [
+ "sha256:2cf9ab83fa62e06717363e8855fb027864caeb35a3197cadb7f0de38356881c4",
+ "sha256:95ce394028754ec537e5791e811511fdd5fabe6f1f8879407a8daed71ecb0b4c"
+ ],
+ "version": "==1.6.5"
+ },
"google-auth": {
"hashes": [
"sha256:34088434cb2a2409360b8f3cbc04195a465df1fb2aafad71ebbded77cbf08803",
@@ -72,34 +79,41 @@
},
"httplib2": {
"hashes": [
- "sha256:e404d3b7bd86c1bc931906098e7c1305d6a3a6dcef141b8bb1059903abb3ceeb"
+ "sha256:f2176149e1e1c59e0520db62c925715018b787b2ae901358803bae5d816fda0b"
],
- "version": "==0.10.3"
+ "version": "==0.11.1"
+ },
+ "oauth2client": {
+ "hashes": [
+ "sha256:cf061f52f75e91d489bf5c276498f8af2655fe331b454f10022441513cf445a6",
+ "sha256:bd3062c06f8b10c6ef7a890b22c2740e5f87d61b6e1f4b1c90d069cdfc9dadb5"
+ ],
+ "version": "==4.1.2"
},
"protobuf": {
"hashes": [
- "sha256:8ba58356fc40ed7749c73eeae3d86f6a9e756ba1ae5f5833990b237b7d61ba09",
- "sha256:e774cd03628c0b2f850a09a8c005fe6113f97e37f6df07a7b20221dc1ee4efd3",
- "sha256:84ed523853c82c76dd1dfd15f31de2d66fa7cb22a48aa42dbc32465868d7e4af",
- "sha256:6e1c0972462ce9dc4d2860d533487b39f89de00b3f30b99c31a6b3e8fbf8b787",
- "sha256:87908d494be2b46a55de5e55ca11d9a2508b59b035c1b0549c3b692a77f57a7b",
- "sha256:88c7958dad426920a43af58c5805d2de860a33f82d47f5a102af25f2788682c7",
- "sha256:14813a3421ff0144e8d4e81ed83a3fbe350d8d85cbe480bf2e81cf45e8083e0d",
- "sha256:24c1cc840b4832a909bbeac664fd8f878cf72b8ab97bfe4fb82a156c3f1f0e15",
- "sha256:ec51286554eceebcf169a3a8604861e113d28fc98094dcbedc6067f058478917",
- "sha256:41e916354265d2f54b95e454305c98f90bb30fafb817119540753e67f193de57",
- "sha256:64a3600d2a531d7c516c371efa431035ce501ab8425dcc8bdb99eddf5a4d34c9",
- "sha256:40c943a8ffb3501164da1d2b537ad2e33d08daf81fbb3e9073bf291726a24467",
- "sha256:59ff8a204aa2ef98d6c25c2adffb13dda81bb4ac6ffb0829c92e801241b6477b",
- "sha256:e457146bb9f997736460b10b2f2a9284603db4bbd60c8c431b5b4b309efbe036",
- "sha256:18a4a387e8378dbbd53ebe9cc925ea2fe2a7b98c497833ea345803cb53b885d9",
- "sha256:94d159e2bbbe4df1b5f0715965e284f2156ce127a7d521a3dcbdd38e945bc4c0",
- "sha256:75e1a7b12248a98b620ffbda3e41767aa2ae57c7cc553a12407a48c44f58f2e7",
- "sha256:c4d531e745168c16fc7abff12922c491d34f4063c1b49fe5417b72be869f5df6",
- "sha256:59610aeb5ade675106dca26c771814a1aa63bf2b3780584853e3dd447ed5c52f",
- "sha256:09879a295fd7234e523b62066223b128c5a8a88f682e3aff62fb115e4a0d8be0"
+ "sha256:ac0067e3c60737865ed72bb7416e02297d229d960902802d874c0e167128c809",
+ "sha256:5c1c8f6a0a68a874e3beff89255959dd80fad45870e96c88944a1b81a22dd5f5",
+ "sha256:7c193e6964e752bd056735594826c5b03274ceb8f07349d3ae47d9766250ba96",
+ "sha256:bcfa99f5a82f5eaaf6e5cee5bfdca5a1670f5740aec1d93dae170645ed1a16b0",
+ "sha256:e269ab7a50bf0fa6fe6a88ea7dcc7a1079ae9450d9ab9b7730ac32916d55508b",
+ "sha256:01ccd6d03449ae75b779fb5bf4ed62177d61afe3c5e6465ccf3f8b2e1a84afbe",
+ "sha256:628a3bf0794a8b3cabb18db11eb67cc10e0cc6e5525d557ae7b682bb73fa2018",
+ "sha256:242e4c7ae565267a8bc8b92d707177f915607ea4bd73244bec6cbf4a49b96661",
+ "sha256:e7fd33a3474cbe18fd5b5620784a0fa21fcae3e402b1806e29c6b450c7f61706",
+ "sha256:cc94079ae6cbcea5ae194464a30f3223f075e06a0446f52bca9ddbeb6e9f412a",
+ "sha256:7222d6616108b33ad6cbeff8117062a73c43cdc8fa8f64f6a322ebeb663e710e",
+ "sha256:3f655e1f99c3e14d56ca900af1b9a4715b691319a295cc38939d7f77eabd5e7c",
+ "sha256:76ef6ca3c50e4cfd044861586d5f1b352e0fe7f17f883df6c165bad5b4d0e10a",
+ "sha256:560a38e692a69957a70ba0e5839aa67430efd63072bf91b0539dac19055694cd",
+ "sha256:d5d9edfdc5a3a01d06062d677b121081629782edf0e05ca1be14f15bb947eeee",
+ "sha256:869e12bcfb5759e683f53ec1dd6155b7be034065431da289f0cb4510040a0799",
+ "sha256:905414e5ea6cdb78d8730f66335755152b46685fcb9fc2f2134024e3ea9e8dcc",
+ "sha256:adf716a89c9cc1891ead79a861c427071ef59172f0e11967b00565a9547b3bd0",
+ "sha256:1d92cc30b0b46cced33adde5853d920179eb5ea8eecdee9552502a7f29cc3f21",
+ "sha256:3b60685732bd0cbdc802dfcb6071efbcf5d927ce3127c13c33ea1a8efae3aa76"
],
- "version": "==3.5.2"
+ "version": "==3.5.2.post1"
},
"pyasn1": {
"hashes": [
@@ -148,6 +162,14 @@
"sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9"
],
"version": "==1.11.0"
+ },
+ "uritemplate": {
+ "hashes": [
+ "sha256:01c69f4fe8ed503b2951bef85d996a9d22434d2431584b5b107b2981ff416fbd",
+ "sha256:1b9c467a940ce9fb9f50df819e8ddd14696f89b9a8cc87ac77952ba416e0a8fd",
+ "sha256:c02643cebe23fc8adb5e6becffe201185bf06c40bda5c0b4028a93f1527d011d"
+ ],
+ "version": "==3.0.0"
}
},
"develop": {
@@ -166,6 +188,14 @@
"markers": "python_version < '3.0'",
"version": "==1.0.2"
},
+ "more-itertools": {
+ "hashes": [
+ "sha256:11a625025954c20145b37ff6309cd54e39ca94f72f6bb9576d1195db6fa2442e",
+ "sha256:0dd8f72eeab0d2c3bd489025bb2f6a1b8342f9b198f6fc37b52d15cfa4531fea",
+ "sha256:c9ce7eccdcb901a2c75d326ea134e0886abfbea5f93e91cc95de9507c0816c44"
+ ],
+ "version": "==4.1.0"
+ },
"pluggy": {
"hashes": [
"sha256:7f8ae7f5bdf75671a718d2daf0a64b7885f74510bcd98b1a0bb420eb9a9d0cff"
@@ -174,17 +204,17 @@
},
"py": {
"hashes": [
- "sha256:8cca5c229d225f8c1e3085be4fcf306090b00850fefad892f9d96c7b6e2f310f",
- "sha256:ca18943e28235417756316bfada6cd96b23ce60dd532642690dcfdaba988a76d"
+ "sha256:983f77f3331356039fdd792e9220b7b8ee1aa6bd2b25f567a963ff1de5a64f6a",
+ "sha256:29c9fab495d7528e80ba1e343b958684f4ace687327e6f789a94bf3d1915f881"
],
- "version": "==1.5.2"
+ "version": "==1.5.3"
},
"pytest": {
"hashes": [
- "sha256:062027955bccbc04d2fcd5d79690947e018ba31abe4c90b2c6721abec734261b",
- "sha256:117bad36c1a787e1a8a659df35de53ba05f9f3398fb9e4ac17e80ad5903eb8c5"
+ "sha256:6266f87ab64692112e5477eba395cfedda53b1933ccd29478e671e73b420c19c",
+ "sha256:fae491d1874f199537fd5872b5e1f0e74a009b979df9d53d1553fd03da1703e1"
],
- "version": "==3.4.2"
+ "version": "==3.5.0"
},
"six": {
"hashes": [
diff --git a/test_analyzer/configs.py b/test_analyzer/configs.py
index fc98d7a..b83f154 100644
--- a/test_analyzer/configs.py
+++ b/test_analyzer/configs.py
@@ -11,9 +11,18 @@
import os
-def GetServiceAccountCredentials(is_stage=False):
- cred_path = os.path.join(os.path.dirname(__file__), 'credentials')
- if is_stage:
- return os.path.join(cred_path, 'chromiumos-test-analyzer-prod.json')
+# Project configs
+PROJECT_ID_PROD = 'chromiumos-test-analyzer'
+PROJECT_ID_STAGING = 'chromeos-test-analyzer-staging'
+
+# Credential configs
+_CREDS_PATH = os.path.join(os.path.dirname(__file__), 'credentials')
+
+
+def GetServiceAccountCredentials(is_stage=True):
+ if not is_stage:
+ return os.path.join(_CREDS_PATH,
+ 'non_cipher/chromiumos-test-analyzer-prod.json')
else:
- return os.path.join(cred_path, 'chromeos-test-analyzer-staging.json')
+ return os.path.join(_CREDS_PATH,
+ 'non_cipher/chromeos-test-analyzer-staging.json')
diff --git a/test_analyzer/credentials/cipher/cipher_api_key.txt b/test_analyzer/credentials/cipher/cipher_api_key.txt
new file mode 100644
index 0000000..fe7ced9
--- /dev/null
+++ b/test_analyzer/credentials/cipher/cipher_api_key.txt
Binary files differ
diff --git a/test_analyzer/credentials/cipher/cipher_api_key_staging.txt b/test_analyzer/credentials/cipher/cipher_api_key_staging.txt
new file mode 100644
index 0000000..8b0b35d
--- /dev/null
+++ b/test_analyzer/credentials/cipher/cipher_api_key_staging.txt
Binary files differ
diff --git a/test_analyzer/integration_test/kms_test.py b/test_analyzer/integration_test/kms_test.py
new file mode 100644
index 0000000..f14ef2e
--- /dev/null
+++ b/test_analyzer/integration_test/kms_test.py
@@ -0,0 +1,18 @@
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Tests for CloudKMS lib."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from test_analyzer import kms
+
+
+def testEncryptDecrypt():
+ plaintext = 'Hello World'
+ ciphertext = kms.Encrypt(kms.DEFAULT_CRYPTO_KEY_FOR_API, plaintext)
+ decrypted_plaintext = kms.Decrypt(kms.DEFAULT_CRYPTO_KEY_FOR_API, ciphertext)
+ assert plaintext == decrypted_plaintext
diff --git a/test_analyzer/kms.py b/test_analyzer/kms.py
new file mode 100644
index 0000000..f51876d
--- /dev/null
+++ b/test_analyzer/kms.py
@@ -0,0 +1,132 @@
+# Copyright 2018 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""lib of Google Cloud KMS, an encryption key management system on GCP.
+
+Used for encrypt/decrypt credentials, e.g. API keys.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import base64
+import collections
+
+from google.oauth2 import service_account # pylint: disable=import-error,no-name-in-module
+from googleapiclient import discovery
+
+from test_analyzer import configs
+
+
+# A KMS Crypto Key's information, including
+# project_id: a string, refers to the project that registers KMS,
+# location_id: a string, refers to the geographical location where the
+# cryptographic keys are stored.
+# key_ring_id: a string, refers to the id of key ring, a concept of a
+# grouping of keys.
+# crypto_key_id: a string, refers to the id of a cryptographic key.
+KMSCryptoKey = collections.namedtuple(
+ 'KMSCryptoKEY',
+ [
+ 'project_id',
+ 'location_id',
+ 'key_ring_id',
+ 'crypto_key_id',
+ ])
+DEFAULT_CRYPTO_KEY_FOR_API = KMSCryptoKey(
+ project_id=configs.PROJECT_ID_PROD,
+ location_id='global',
+ key_ring_id='creds',
+ crypto_key_id='apikey')
+
+
+def _CreateKMSClient():
+ """Create a KMS Client to connect to cloudkms service.
+
+ Returns:
+ A discovery.build client for connecting to cloudkms.
+ """
+ # We will only kms crypte key built in prod instance to encrypt/decrypt.
+ secret_json = configs.GetServiceAccountCredentials(is_stage=False)
+ credentials = service_account.Credentials.from_service_account_file(
+ secret_json)
+ return discovery.build('cloudkms', 'v1', credentials=credentials)
+
+
+def _CreateResourceName(crypto_key_info):
+ """Create a resource name to encrypt/decrypt.
+
+ Args:
+ crypto_key_info: A KMSCryptoKey object, including all required key infos.
+ """
+ return 'projects/%s/locations/%s/keyRings/%s/cryptoKeys/%s' % (
+ crypto_key_info.project_id, crypto_key_info.location_id,
+ crypto_key_info.key_ring_id, crypto_key_info.crypto_key_id)
+
+
+def Encrypt(crypto_key_info, plaintext):
+ """Encrypts data with the provided CryptoKey.
+
+ This function encrypts plaintext using the provided CryptoKey. It can only be
+ recovered with Decrypt().
+
+ Args:
+ crypto_key_info: A KMSCryptoKey object, including all required key infos.
+ plaintext: The string plain text to be encrypted.
+
+ Returns:
+ A string cipher text.
+ """
+ kms_client = _CreateKMSClient()
+ resource_name = _CreateResourceName(crypto_key_info)
+
+ # Use the KMS API to encrypt the data.
+ crypto_keys = kms_client.projects().locations().keyRings().cryptoKeys()
+ request = crypto_keys.encrypt(
+ name=resource_name,
+ body={'plaintext': base64.b64encode(plaintext).decode('ascii')})
+ response = request.execute()
+ return base64.b64decode(response['ciphertext'].encode('ascii'))
+
+
+def Decrypt(crypto_key_info, ciphertext):
+ """Decrypts data with the provided CryptoKey.
+
+ This function decrypts the cipher text which is encrypted using the
+ provided CryptoKey.
+
+ Args:
+ crypto_key_info: A KMSCryptoKey object, including all required key infos.
+ ciphertext: The string cipher text to be decrypted.
+
+ Returns:
+ A string plain text.
+ """
+ kms_client = _CreateKMSClient()
+ resource_name = _CreateResourceName(crypto_key_info)
+
+ # Use the KMS API to decrypt the data.
+ crypto_keys = kms_client.projects().locations().keyRings().cryptoKeys()
+ request = crypto_keys.decrypt(
+ name=resource_name,
+ body={'ciphertext': base64.b64encode(ciphertext).decode('ascii')})
+ response = request.execute()
+ return base64.b64decode(response['plaintext'].encode('ascii'))
+
+
+def DecryptFromFile(crypto_key_info, ciphertext_file_name):
+ """Decrypt data in a ciphertext_file.
+
+ Args:
+ crypto_key_info: A KMSCryptoKey object, including all required key infos.
+ ciphertext_file_name: The string file name to store cipher text.
+
+ Returns:
+ A string plain text after decryption.
+ """
+ with open(ciphertext_file_name, 'rb') as ciphertext_file:
+ ciphertext = ciphertext_file.read()
+
+ return Decrypt(crypto_key_info, ciphertext)