Experimental extension for "secret" parameters. (#683)

* Experimental extension for "secret" parameters.

These are string input parameters that represent sensitive information
such as passwords.  This feature deliberately obscures them by
replacing them with a placeholder during workflow evaluation and then
substituting the real value at the last moment.  Staged files containing secrets are deleted when the tool completes.

Author: tetron

cwltool/docker.py

@@ -17,6 +17,7 @@
 from .errors import WorkflowException
 from .job import ContainerCommandLineJob
 from .pathmapper import PathMapper, ensure_writable
+from .secrets import SecretStore
 from .utils import docker_windows_path_adjust, onWindows
 
 _logger = logging.getLogger("cwltool")
@@ -132,8 +133,8 @@ def get_from_requirements(self, r, req, pull_image, dry_run=False):
 
         return None
 
-    def add_volumes(self, pathmapper, runtime):
-        # type: (PathMapper, List[Text]) -> None
+    def add_volumes(self, pathmapper, runtime, secret_store=None):
+        # type: (PathMapper, List[Text], SecretStore) -> None
 
         host_outdir = self.outdir
         container_outdir = self.builder.outdir
@@ -170,13 +171,17 @@ def add_volumes(self, pathmapper, runtime):
                         shutil.copytree(vol.resolved, host_outdir_tgt)
                         ensure_writable(host_outdir_tgt)
             elif vol.type == "CreateFile":
+                if secret_store:
+                    contents = secret_store.retrieve(vol.resolved)
+                else:
+                    contents = vol.resolved
                 if host_outdir_tgt:
                     with open(host_outdir_tgt, "wb") as f:
-                        f.write(vol.resolved.encode("utf-8"))
+                        f.write(contents.encode("utf-8"))
                 else:
                     fd, createtmp = tempfile.mkstemp(dir=self.tmpdir)
                     with os.fdopen(fd, "wb") as f:
-                        f.write(vol.resolved.encode("utf-8"))
+                        f.write(contents.encode("utf-8"))
                     runtime.append(u"--volume=%s:%s:rw" % (
                         docker_windows_path_adjust(createtmp),
                         docker_windows_path_adjust(vol.target)))
@@ -196,9 +201,9 @@ def create_runtime(self, env, rm_container=True, record_container_id=False, cidf
         runtime.append(u"--volume=%s:%s:rw" % (
             docker_windows_path_adjust(os.path.realpath(self.tmpdir)), "/tmp"))
 
-        self.add_volumes(self.pathmapper, runtime)
+        self.add_volumes(self.pathmapper, runtime, secret_store=kwargs.get("secret_store"))
         if self.generatemapper:
-            self.add_volumes(self.generatemapper, runtime)
+            self.add_volumes(self.generatemapper, runtime, secret_store=kwargs.get("secret_store"))
 
         if user_space_docker_cmd:
             runtime = [x.replace(":ro", "") for x in runtime]

cwltool/extensions.yml

@@ -34,3 +34,23 @@ $graph:
         "_type": "@vocab"
     inplaceUpdate:
       type: boolean
+
+- name: Secrets
+  type: record
+  inVocab: false
+  extends: cwl:ProcessRequirement
+  fields:
+    class:
+      type: string
+      doc: "Always 'Secrets'"
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    secrets:
+      type: string[]
+      doc: |
+        List one or more input parameters that are sensitive (such as passwords)
+        which will be deliberately obscured from logging.
+      jsonldPredicate:
+        "_type": "@id"
+        refScope: 0

cwltool/job.py

@@ -25,6 +25,7 @@
 from .pathmapper import PathMapper
 from .process import (UnsupportedRequirement, get_feature,
                       stageFiles)
+from .secrets import SecretStore
 from .utils import bytes2str_in_dicts
 from .utils import copytree_with_merge, onWindows
 
@@ -170,8 +171,8 @@ def _setup(self, kwargs):  # type: (Dict) -> None
             _logger.debug(u"[job %s] initial work dir %s", self.name,
                           json.dumps({p: self.generatemapper.mapper(p) for p in self.generatemapper.files()}, indent=4))
 
-    def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"):
-        # type: (List[Text], MutableMapping[Text, Text], bool, Text) -> None
+    def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move", secret_store=None):
+        # type: (List[Text], MutableMapping[Text, Text], bool, Text, SecretStore) -> None
 
         scr, _ = get_feature(self, "ShellCommandRequirement")
 
@@ -214,6 +215,10 @@ def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"):
                 stdout_path = absout
 
             commands = [Text(x) for x in (runtime + self.command_line)]
+            if secret_store:
+                commands = secret_store.retrieve(commands)
+                env = secret_store.retrieve(env)
+
             job_script_contents = None  # type: Text
             builder = getattr(self, "builder", None)  # type: Builder
             if builder is not None:
@@ -269,6 +274,19 @@ def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"):
         if _logger.isEnabledFor(logging.DEBUG):
             _logger.debug(u"[job %s] %s", self.name, json.dumps(outputs, indent=4))
 
+        if self.generatemapper and secret_store:
+            # Delete any runtime-generated files containing secrets.
+            for f, p in self.generatemapper.items():
+                if p.type == "CreateFile":
+                    if secret_store.has_secret(p.resolved):
+                        host_outdir = self.outdir
+                        container_outdir = self.builder.outdir
+                        host_outdir_tgt = p.target
+                        if p.target.startswith(container_outdir+"/"):
+                            host_outdir_tgt = os.path.join(
+                                host_outdir, p.target[len(container_outdir)+1:])
+                        os.remove(host_outdir_tgt)
+
         with job_output_lock:
             self.output_callback(outputs, processStatus)
 
@@ -307,12 +325,12 @@ def run(self, pull_image=True, rm_container=True,
         if "SYSTEMROOT" not in env and "SYSTEMROOT" in os.environ:
             env["SYSTEMROOT"] = str(os.environ["SYSTEMROOT"]) if onWindows() else os.environ["SYSTEMROOT"]
 
-        stageFiles(self.pathmapper, ignoreWritable=True, symLink=True)
+        stageFiles(self.pathmapper, ignoreWritable=True, symLink=True, secret_store=kwargs.get("secret_store"))
         if self.generatemapper:
-            stageFiles(self.generatemapper, ignoreWritable=self.inplace_update, symLink=True)
+            stageFiles(self.generatemapper, ignoreWritable=self.inplace_update, symLink=True, secret_store=kwargs.get("secret_store"))
             relink_initialworkdir(self.generatemapper, self.outdir, self.builder.outdir, inplace_update=self.inplace_update)
 
-        self._execute([], env, rm_tmpdir=rm_tmpdir, move_outputs=move_outputs)
+        self._execute([], env, rm_tmpdir=rm_tmpdir, move_outputs=move_outputs, secret_store=kwargs.get("secret_store"))
 
 
 class ContainerCommandLineJob(JobBase):
@@ -382,7 +400,7 @@ def run(self, pull_image=True, rm_container=True,
         runtime = self.create_runtime(env, rm_container, record_container_id, cidfile_dir, cidfile_prefix, **kwargs)
         runtime.append(img_id)
 
-        self._execute(runtime, env, rm_tmpdir=rm_tmpdir, move_outputs=move_outputs)
+        self._execute(runtime, env, rm_tmpdir=rm_tmpdir, move_outputs=move_outputs, secret_store=kwargs.get("secret_store"))
 
 
 def _job_popen(

cwltool/main.py

@@ -36,6 +36,7 @@
 from .process import (Process, normalizeFilesDirs,
                       scandeps, shortname, use_custom_schema,
                       use_standard_schema)
+from .secrets import SecretStore
 from .resolver import ga4gh_tool_registries, tool_resolver
 from .software_requirements import (DependenciesConfiguration,
                                     get_container_from_software_requirements)
@@ -154,10 +155,13 @@ def init_job_order(job_order_object,  # type: MutableMapping[Text, Any]
                    stdout=sys.stdout,       # type: IO[Any]
                    make_fs_access=None,     # type: Callable[[Text], StdFsAccess]
                    loader=None,             # type: Loader
-                   input_basedir=""         # type: Text
+                   input_basedir="",        # type: Text
+                   secret_store=None        # type: SecretStore
 ):
     # (...) -> Tuple[Dict[Text, Any], Text]
 
+    secrets_req, _ = t.get_requirement("http://commonwl.org/cwltool#Secrets")
+
     if not job_order_object:
         namemap = {}  # type: Dict[Text, Text]
         records = []  # type: List[Text]
@@ -191,6 +195,9 @@ def init_job_order(job_order_object,  # type: MutableMapping[Text, Any]
 
             job_order_object.update({namemap[k]: v for k, v in cmd_line.items()})
 
+            if secrets_req:
+                secret_store.store([shortname(sc) for sc in secrets_req["secrets"]], job_order_object)
+
             if _logger.isEnabledFor(logging.DEBUG):
                 _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4))
         else:
@@ -245,6 +252,9 @@ def expand_formats(p):
     adjustDirObjs(job_order_object, trim_listing)
     normalizeFilesDirs(job_order_object)
 
+    if secrets_req:
+        secret_store.store([shortname(sc) for sc in secrets_req["secrets"]], job_order_object)
+
     if "cwl:tool" in job_order_object:
         del job_order_object["cwl:tool"]
     if "id" in job_order_object:
@@ -556,14 +566,17 @@ def main(argsl=None,  # type: List[str]
                 setattr(args, 'move_outputs', "copy")
             setattr(args, "tmp_outdir_prefix", args.cachedir)
 
+        secret_store = SecretStore()
+
         try:
             job_order_object = init_job_order(job_order_object, args, tool,
                                               print_input_deps=args.print_input_deps,
                                               relative_deps=args.relative_deps,
                                               stdout=stdout,
                                               make_fs_access=make_fs_access,
                                               loader=jobloader,
-                                              input_basedir=input_basedir)
+                                              input_basedir=input_basedir,
+                                              secret_store=secret_store)
         except SystemExit as e:
             return e.code
 
@@ -585,6 +598,7 @@ def main(argsl=None,  # type: List[str]
                                      makeTool=makeTool,
                                      select_resources=selectResources,
                                      make_fs_access=make_fs_access,
+                                     secret_store=secret_store,
                                      **vars(args))
 
             # This is the workflow output, it needs to be written

cwltool/process.py

@@ -36,9 +36,11 @@
 from .pathmapper import (PathMapper, adjustDirObjs, get_listing,
                          normalizeFilesDirs, visit_class, trim_listing,
                          ensure_writable)
+from .secrets import SecretStore
 from .stdfsaccess import StdFsAccess
 from .utils import aslist, get_feature, copytree_with_merge, onWindows
 
+
 # if six.PY3:
 # AvroSchemaFromJSONData = avro.schema.SchemaFromJSONData
 # else:
@@ -207,8 +209,8 @@ def adjustFilesWithSecondary(rec, op, primary=None):
             adjustFilesWithSecondary(d, op, primary)
 
 
-def stageFiles(pm, stageFunc=None, ignoreWritable=False, symLink=True):
-    # type: (PathMapper, Callable[..., Any], bool, bool) -> None
+def stageFiles(pm, stageFunc=None, ignoreWritable=False, symLink=True, secret_store=None):
+    # type: (PathMapper, Callable[..., Any], bool, bool, SecretStore) -> None
     for f, p in pm.items():
         if not p.staged:
             continue
@@ -240,7 +242,10 @@ def stageFiles(pm, stageFunc=None, ignoreWritable=False, symLink=True):
                 ensure_writable(p.target)
         elif p.type == "CreateFile":
             with open(p.target, "wb") as n:
-                n.write(p.resolved.encode("utf-8"))
+                if secret_store:
+                    n.write(secret_store.retrieve(p.resolved).encode("utf-8"))
+                else:
+                    n.write(p.resolved.encode("utf-8"))
             ensure_writable(p.target)
 
 def collectFilesAndDirs(obj, out):

cwltool/secrets.py

@@ -0,0 +1,53 @@
+import uuid
+import six
+from typing import (List, Dict, Any, Text, MutableMapping)
+
+class SecretStore(object):
+    def __init__(self):
+        # type: () -> None
+        self.secrets = {}  # type: Dict[Text, Text]
+
+    def add(self, value):
+        # type: (Text) -> Text
+        if not isinstance(value, six.string_types):
+            raise Exception("Secret store only accepts strings")
+
+        if value not in self.secrets:
+            placeholder = "(secret-%s)" % Text(uuid.uuid4())
+            self.secrets[placeholder] = value
+            return placeholder
+        else:
+            return value
+
+    def store(self, secrets, job):
+        # type: (List[Text], MutableMapping[Text, Any]) -> None
+        for j in job:
+            if j in secrets:
+                job[j] = self.add(job[j])
+
+    def has_secret(self, value):
+        # type: (Any) -> bool
+        if isinstance(value, six.string_types):
+            for k in self.secrets:
+                if k in value:
+                    return True
+        elif isinstance(value, dict):
+            for v in value.values():
+                if self.has_secret(v):
+                    return True
+        elif isinstance(value, list):
+            for v in value:
+                if self.has_secret(v):
+                    return True
+        return False
+
+    def retrieve(self, value):
+        # type: (Any) -> Any
+        if isinstance(value, six.string_types):
+            for k,v in self.secrets.items():
+                value = value.replace(k, v)
+        elif isinstance(value, dict):
+            return {k: self.retrieve(v) for k,v in value.items()}
+        elif isinstance(value, list):
+            return [self.retrieve(v) for k,v in enumerate(value)]
+        return value

tests/test_secrets.py

@@ -0,0 +1,35 @@
+from __future__ import absolute_import
+
+import unittest
+
+from cwltool.secrets import SecretStore
+from .util import get_data
+
+class TestSecrets(unittest.TestCase):
+    def test_secrets(self):
+        secrets = SecretStore()
+        job = {"foo": "bar",
+               "baz": "quux"}
+        secrets.store(["foo"], job)
+        self.assertNotEquals(job["foo"], "bar")
+        self.assertEquals(job["baz"], "quux")
+        self.assertEquals(secrets.retrieve(job)["foo"], "bar")
+
+        hello = "hello %s" % job["foo"]
+        self.assertTrue(secrets.has_secret(hello))
+        self.assertNotEquals(hello, "hello bar")
+        self.assertEquals(secrets.retrieve(hello), "hello bar")
+
+        hello2 = ["echo", "hello %s" % job["foo"]]
+        self.assertTrue(secrets.has_secret(hello2))
+        self.assertNotEquals(hello2, ["echo", "hello bar"])
+        self.assertEquals(secrets.retrieve(hello2), ["echo", "hello bar"])
+
+        hello3 = {"foo": job["foo"]}
+        print(hello3)
+        self.assertTrue(secrets.has_secret(hello3))
+        self.assertNotEquals(hello3, {"foo": "bar"})
+        self.assertEquals(secrets.retrieve(hello3), {"foo": "bar"})
+
+        self.assertNotEquals(job["foo"], "bar")
+        self.assertEquals(job["baz"], "quux")

tests/wf/secret_job.cwl

@@ -0,0 +1,19 @@
+cwlVersion: v1.0
+class: CommandLineTool
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+hints:
+  "cwltool:Secrets":
+    secrets: [pw]
+requirements:
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: example.conf
+        entry: |
+          username: user
+          password: $(inputs.pw)
+inputs:
+  pw: string
+outputs:
+  out: stdout
+arguments: [cat, example.conf]

tests/wf/secret_wf.cwl

@@ -0,0 +1,21 @@
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+  cwltool: http://commonwl.org/cwltool#
+hints:
+  "cwltool:Secrets":
+    secrets: [pw]
+  DockerRequirement:
+    dockerPull: debian:8
+inputs:
+  pw: string
+outputs:
+  out:
+    type: File
+    outputSource: step1/out
+steps:
+  step1:
+    in:
+      pw: pw
+    out: [out]
+    run: secret_job.cwl