Add support for templating in init-container

.gitignore

@@ -40,4 +40,6 @@ job/
 **/report.json
 **/report.json
 node_modules/
-**/vendor/
+**/vendor/
+init-container/token.txt
+init-container/decrypted/**

init-container/.dockerignore

@@ -0,0 +1 @@
+node_modules/**

init-container/CONTRIBUTING.md

@@ -0,0 +1,47 @@
+# Running the init container locally
+
+1. Make sure you have a working Kubernetes cluster (run `kubectl get pods` to check).
+2. Start by running the decryptor API - run the following in the decryptor folder (`src/decrypt-api`):
+```
+dotnet run
+```
+3. Run the following command to get a valid service account token:
+```
+kubectl get secret $(kubectl get sa default -o jsonpath='{.secrets[0].name}') -o jsonpath='{.data.token}' | base64 -D > token.txt
+```
+4. Set the following env var for the init container to work correctly:
+```
+export set TOKEN_FILE_PATH=$(pwd)/token.txt
+export set KAMUS_URL=http://localhost:5000
+```
+5. Now you can run the init container:
+```
+node index.js -e encrypted -d decrypted -n output.json 
+```
+
+The first argument (`-e`) point to the folder with the encrypted files, the second (`-d`) point to the folder that will contain the decrypted items and (`-n`) is the decrypted file name.
+
+And you should see a file name `output.json` created under `decrypted` folder with the decrypted content.
+Now you can run the init continer locally, debug it and add features. 
+
+Having more questions? Something unclear? Reach out to us via [Slack](https://join.slack.com/t/k8s-kamus/shared_invite/enQtNTQwMjc2MzIxMTM3LTgyYTcwMTUxZjJhN2JiMTljMjNmOTBmYjEyNWNmZTRiNjVhNTUyYjMwZDQ0YWQ3Y2FmMTBlODA5MzFlYjYyNWE) or [file an issue](/~https://github.com/Soluto/kamus/issues/new)
+
+## Tests
+The tests are using [WireMock](http://wiremock.org/) to mock the decryptor api (you can find ore about it [here](https://www.omerlh.info/2019/02/06/wiremock-for-fun-and-mocking/)).
+Running the tests is simple:
+1. Build the init container docker image:
+```
+yarn run build
+``` 
+2. Set the environment variable so the tests will use the local image:
+```
+export set INIT_CONTAINER_IMAGE=init-container
+```
+3. Run the tests
+```
+yarn run test
+```
+
+Repeat steps 1 & 3 each time you change the code.
+
+The tests simply take various parameters and compare the generated file from the init container with the expected files. 

init-container/README.md

@@ -28,4 +28,36 @@ The init container accept the following environmenmt variables:
 | `-e/--encrypted-folders <path>`          |   true        |   Encrypted files folder paths, comma seperated (the volumes mounted with the config map)               |               |
 | `-d/--decrypted-path <path>`          |   false        |   Decrypted file/s folder path mounted. Pass this argument to create one decrypted file per encrypted secret              |               |
 | `-n/--decrypted-file-name <name>`          |   false        |   Decrypted file name. Pass this argument to create one configuration file with the encrypted secrets.             |               |
-| `-f/--output-format <format>`          |   false        |  The format of the output file. Supported types: json, cfg, cfg-strict (surround strings with quotation marks), files           |         JSON      |
+| `-f/--output-format <format>`          |   false        |  The format of the output file. Supported types: json, cfg, cfg-strict (surround strings with quotation marks), files, custom (see above for more bellow)           |         JSON      |
+
+## Custom templating support
+In case you need something more complicated than the support output format, you can provide your own template.
+The init container support [EJS](http://ejs.co/) templates, a powerful template engine for nodejs.
+To use it, provide a key in the supplied config map called "template.ejs":
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: encrypted-secrets-cm
+data:
+  key: 4AD7lM6lc4dGvE3oF+5w8g==:WrcckiNNOAlMhuWHaM0kTw==
+  template.ejs: |
+     <%= secrets["key"] %>
+     hello
+```
+
+This will result in the following file created by the init container:
+```
+<decrypted value>
+hello
+```
+
+Look on EJS docummentation for more details, or on one of the existing [templates](/~https://github.com/Soluto/kamus/tree/master/init-container/templates) for ideas on how you can use it. The template input is:
+```
+{
+  "secrets": [] //array of the decretyped items, key value pairs.
+  "stringifyIfJson": function //apply JSON.stringify if the value is object.
+}
+```
+
+Because the init container support multiple config maps, you can create shared template using config map and mount them where needed. Have a common template? we'll appreciate PRs! 

init-container/encrypted/key

@@ -0,0 +1 @@
+ezBR+Ew+Itwg6fA/tQjxzg==:/DH+kSV3UN8eRUxT/cJp5w==

init-container/index.js

@@ -6,15 +6,26 @@ const readFileAsync = util.promisify(fs.readFile);
 const writeFile = util.promisify(fs.writeFile);
 const axios = require('axios');
 const path = require('path');
+let ejs = require('ejs');
 
 program
     .version('0.1.0')
     .option('-e, --encrypted-folders <path>', 'Encrypted files folder paths, comma separated')
     .option('-d, --decrypted-path <path>', 'Decrypted file/s folder path')
     .option('-n, --decrypted-file-name <name>', 'Decrypted file name' )
-    .option('-f, --output-format <format>', 'The format of the output file, default to JSON. Supported types: json, cfg, files', /^(json|cfg|cfg-strict|files)$/i, 'json')
+    .option('-f, --output-format <format>', 'The format of the output file, default to JSON. Supported types: json, cfg, files, custom', /^(json|cfg|cfg-strict|files|custom)$/i, 'json')
     .parse(process.argv);
 
+//Source: https://blog.raananweber.com/2015/12/15/check-if-a-directory-exists-in-node-js/
+function checkDirectorySync(directory) {  
+  try {
+    fs.statSync(directory);
+  } catch(e) {
+    fs.mkdirSync(directory);
+  }
+}
+
+
 const getEncryptedFiles = async (folder) => {
     return await readfiles(folder, function (err, filename, contents) {
         if (err) throw err;
@@ -30,7 +41,12 @@ const getKamusUrl = () => {
 }
 
 const getBarerToken = async () => {
-    return await readFileAsync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8");
+    var tokenFilePath = process.env.TOKEN_FILE_PATH;
+    if (tokenFilePath == null || tokenFilePath == "")
+    {
+      tokenFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/token";
+    }
+    return await readFileAsync(tokenFilePath, "utf8");
 }
 
 const stringifyIfJson = (secretValue) =>{
@@ -46,37 +62,12 @@ const decryptFile = async (httpClient, filePath, folder) => {
     } catch (e) {
       throw new Error(`request to decrypt API failed: ${e.response ? e.response.status : e.message}`)
     }
-    return response.data;
 }
 
-const serializeToCfgFormat = (secrets) => {
-  var output = "";
-  Object.keys(secrets).forEach(key => {
-    output += `${key}=${stringifyIfJson(secrets[key])}\n`;
-  });
-
-  output = output.substring(0, output.lastIndexOf('\n'));
-
-  return output;
-}
-
-const serializeToCfgFormatStrict = (secrets) => {
-  var output = "";
-  Object.keys(secrets).forEach(key => {
-    switch(typeof(secrets[key]))
-    {
-      case "string":
-        output += `${key}="${secrets[key]}"\n`
-        break;
-      default:
-        output += `${key}=${stringifyIfJson(secrets[key])}\n`
-    }
-
-  });
-
-  output = output.substring(0, output.lastIndexOf('\n'));
-
-  return output;
+const writeFileWithTemplate = async (secrets, templateName, outputFile) => {
+  var template = await readFileAsync(templateName, "utf-8");
+  var rendered = ejs.render(template, {secrets, stringifyIfJson}, {});
+  await writeFile(outputFile, rendered);
 }
 
 async function innerRun() {
@@ -90,29 +81,43 @@ async function innerRun() {
     });
 
     let secrets = {};
+    var templatePath = "";
     for (let folder of program.encryptedFolders.split(",")) {
         let files = await getEncryptedFiles(folder);
         for (let file of files) {
+            if (file === "template.ejs" && program.outputFormat === "custom"){
+              templatePath = path.join(folder, file)
+              continue;
+            }
             secrets[file] = await decryptFile(httpClient, file, folder);
         }
     }
+
+    checkDirectorySync(program.decryptedPath);
 
     const outputFile = path.join(program.decryptedPath, program.decryptedFileName);
     console.log(`Writing output format using ${program.outputFormat} format to file ${outputFile}`);
 
     switch(program.outputFormat.toLowerCase()){
       case "json":
-        await writeFile(outputFile, JSON.stringify(secrets));
+        await writeFileWithTemplate(secrets, "templates/json.ejs", outputFile);
         break;
       case "cfg":
-        await writeFile(outputFile, serializeToCfgFormat(secrets));
+        await writeFileWithTemplate(secrets, "templates/cfg.ejs", outputFile);
         break;
       case "cfg-strict":
-        await writeFile(outputFile, serializeToCfgFormatStrict(secrets));
+        await writeFileWithTemplate(secrets, "templates/cfg-strict.ejs", outputFile);
         break;
       case "files":
         await Promise.all(Object.keys(secrets).map(secretName => writeFile(path.join(program.decryptedPath, secretName), stringifyIfJson(secrets[secretName]))));
         break;
+      case "custom":
+        if (templatePath == "")
+        {
+          throw new Error(`Missing template file, cannot write output`);
+        }
+        await writeFileWithTemplate(secrets, templatePath, outputFile);
+        break;
       default:
         throw new Error(`Unsupported output format: ${program.outputFormat}`);
     }

init-container/package.json

@@ -1,18 +1,23 @@
 {
   "name": "init-decryptor",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "description": "Meant to be used inside init container to read encrypted values from a given folder and decrypt to them into a json in a given folder",
   "main": "index.js",
   "scripts": {
+    "build": "docker build . -t init-container && export set INIT_CONTAINER_IMAGE=init-container",
     "test": "cd tests && ./run_test.sh",
     "snyk-protect": "snyk protect",
-    "prepublish": "npm run snyk-protect"
+    "prepublish": "npm run snyk-protect",
+    "pretest": "ejslint templates/*",
+    "ejslint": "ejslint templates/*"
   },
   "author": "Soluto",
   "license": "MIT",
   "dependencies": {
     "axios": "^0.18.0",
     "commander": "^2.19.0",
+    "ejs": "^2.6.1",
+    "ejs-lint": "^0.3.0",
     "node-readfiles": "^0.2.0",
     "snyk": "^1.161.1"
   },

init-container/templates/cfg-strict.ejs

@@ -0,0 +1,7 @@
+<%_ Object.keys(secrets).forEach(function(key){ -%>
+<%_ if(typeof(secrets[key]) === "string") { -%>
+<%= key %>="<%- secrets[key] %>"
+<%_ } else { -%>
+<%= key %>=<%- stringifyIfJson(secrets[key]) %>
+<%_ } -%>
+<%_ }); -%>

init-container/templates/cfg.ejs

@@ -0,0 +1,3 @@
+<%_ Object.keys(secrets).forEach(function(key){ -%>
+<%= key %>=<%- stringifyIfJson(secrets[key]) %>
+<%_ }); -%>

init-container/templates/json.ejs

@@ -0,0 +1,9 @@
+{ 
+<%_ Object.keys(secrets).forEach(function(key, index){ -%>
+<%_ if(typeof(secrets[key]) === "object") { -%>
+    "<%= key %>":<%- JSON.stringify(secrets[key]) -%> <%_ if(Object.keys(secrets).length - 1 > index) { -%> , <%_ } %>
+<%_ } else { -%>
+    "<%= key %>":"<%- secrets[key] -%>" <%_ if(Object.keys(secrets).length - 1 > index) { -%> , <%_ } %>
+<%_ } -%>
+<%_ }); -%>
+}

init-container/tests/expected-custom.txt

@@ -0,0 +1,2 @@
+super-secret-from-value
+hello

init-container/tests/expected-strict.cfg

@@ -1,3 +1,3 @@
 key.json={"secret_key":"secret_value"}
 key1="super-secret-from-value"
-key2="super-secret"
+key2="super-secret"

init-container/tests/expected.cfg

@@ -1,3 +1,3 @@
 key.json={"secret_key":"secret_value"}
 key1=super-secret-from-value
-key2=super-secret
+key2=super-secret

init-container/tests/expected.json

@@ -1 +1,5 @@
-{"key.json":{"secret_key":"secret_value"},"key1":"super-secret-from-value","key2":"super-secret"}
+{ 
+    "key.json":{"secret_key":"secret_value"} ,
+    "key1":"super-secret-from-value" ,
+    "key2":"super-secret"
+}

init-container/tests/run_test.sh

@@ -42,6 +42,17 @@ diff -q output/out.cfg-strict expected-strict.cfg
 
 rm -rf output
 
+echo "running decryptor - custom format"
+
+cp templates/template.ejs encrypted/
+OUTPUT_FORMAT=custom docker-compose run decryptor
+rm -rf encrypted/template.ejs
+echo "comparing out.custom and expected-custom.txt files"
+
+diff -q output/out.custom expected-custom.txt
+
+rm -rf output
+
 echo "running decryptor - files format"
 
 OUTPUT_FORMAT=files docker-compose run decryptor

init-container/tests/templates/template.ejs

@@ -0,0 +1,2 @@
+<%= secrets["key1"] %>
+hello