cwltool-1.0.20170114120503/0000755000175000017500000000000013036412107015431 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/0000755000175000017500000000000013036412107017114 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/sandboxjs.py0000644000175000017500000001231513036412052021462 0ustar michaelmichael00000000000000import subprocess import json import threading import errno import logging import select import os import cStringIO from cStringIO import StringIO from typing import Any, Dict, List, Mapping, Text, TypeVar, Union from pkg_resources import resource_stream class JavascriptException(Exception): pass _logger = logging.getLogger("cwltool") JSON = Union[Dict[Text,Any], List[Any], Text, int, long, float, bool, None] localdata = threading.local() have_node_slim = False def new_js_proc(): # type: () -> subprocess.Popen res = resource_stream(__name__, 'cwlNodeEngine.js') nodecode = res.read() nodejs = None trynodes = ("nodejs", "node") for n in trynodes: try: nodejs = subprocess.Popen([n, "--eval", nodecode], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) break except OSError as e: if e.errno == errno.ENOENT: pass else: raise if nodejs is None: try: nodeimg = "node:slim" global have_node_slim if not have_node_slim: dockerimgs = subprocess.check_output(["docker", "images", nodeimg]) if len(dockerimgs.split("\n")) <= 1: nodejsimg = subprocess.check_output(["docker", "pull", nodeimg]) _logger.info("Pulled Docker image %s %s", nodeimg, nodejsimg) have_node_slim = True nodejs = subprocess.Popen(["docker", "run", "--attach=STDIN", "--attach=STDOUT", "--attach=STDERR", "--sig-proxy=true", "--interactive", "--rm", nodeimg, "node", "--eval", nodecode], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except OSError as e: if e.errno == errno.ENOENT: pass else: raise except subprocess.CalledProcessError: pass if nodejs is None: raise JavascriptException( u"cwltool requires Node.js engine to evaluate Javascript " "expressions, but couldn't find it. Tried %s, docker run " "node:slim" % u", ".join(trynodes)) return nodejs def execjs(js, jslib, timeout=None, debug=False): # type: (Union[Mapping, Text], Any, int, bool) -> JSON if not hasattr(localdata, "proc") or localdata.proc.poll() is not None: localdata.proc = new_js_proc() nodejs = localdata.proc fn = u"\"use strict\";\n%s\n(function()%s)()" % (jslib, js if isinstance(js, basestring) and len(js) > 1 and js[0] == '{' else ("{return (%s);}" % js)) killed = [] def term(): try: killed.append(True) nodejs.kill() except OSError: pass if timeout is None: timeout = 20 tm = threading.Timer(timeout, term) tm.start() stdin_buf = StringIO(json.dumps(fn)+"\n") stdout_buf = StringIO() stderr_buf = StringIO() completed = [] # type: List[Union[cStringIO.InputType, cStringIO.OutputType]] while len(completed) < 3: rready, wready, _ = select.select([nodejs.stdout, nodejs.stderr], [nodejs.stdin], []) if nodejs.stdin in wready: b = stdin_buf.read(select.PIPE_BUF) if b: os.write(nodejs.stdin.fileno(), b) elif stdin_buf not in completed: completed.append(stdin_buf) for pipes in ((nodejs.stdout, stdout_buf), (nodejs.stderr, stderr_buf)): if pipes[0] in rready: b = os.read(pipes[0].fileno(), select.PIPE_BUF) if b: pipes[1].write(b) elif pipes[1] not in completed: completed.append(pipes[1]) if stdout_buf.getvalue().endswith("\n"): for buf in (stdout_buf, stderr_buf): if buf not in completed: completed.append(buf) tm.cancel() stdin_buf.close() stdoutdata = stdout_buf.getvalue() stderrdata = stderr_buf.getvalue() def fn_linenum(): # type: () -> Text lines = fn.splitlines() ofs = 0 maxlines = 99 if len(lines) > maxlines: ofs = len(lines)-maxlines lines = lines[-maxlines:] return u"\n".join(u"%02i %s" % (i+ofs+1, b) for i, b in enumerate(lines)) def stdfmt(data): # type: (unicode) -> unicode if "\n" in data: return "\n" + data.strip() return data if debug: info = u"returncode was: %s\nscript was:\n%s\nstdout was: %s\nstderr was: %s\n" % (nodejs.returncode, fn_linenum(), stdfmt(stdoutdata), stdfmt(stderrdata)) else: info = stdfmt(stderrdata) if nodejs.poll() not in (None, 0): if killed: raise JavascriptException(u"Long-running script killed after %s seconds: %s" % (timeout, info)) else: raise JavascriptException(info) else: try: return json.loads(stdoutdata) except ValueError as e: raise JavascriptException(u"%s\nscript was:\n%s\nstdout was: '%s'\nstderr was: '%s'\n" % (e, fn_linenum(), stdoutdata, stderrdata)) cwltool-1.0.20170114120503/cwltool/utils.py0000644000175000017500000000074413036412053020633 0ustar michaelmichael00000000000000# no imports from cwltool allowed from typing import Any, Tuple def aslist(l): # type: (Any) -> List[Any] if isinstance(l, list): return l else: return [l] def get_feature(self, feature): # type: (Any, Any) -> Tuple[Any, bool] for t in reversed(self.requirements): if t["class"] == feature: return (t, True) for t in reversed(self.hints): if t["class"] == feature: return (t, False) return (None, None) cwltool-1.0.20170114120503/cwltool/cwlrdf.py0000644000175000017500000001231113036412052020744 0ustar michaelmichael00000000000000import json import urlparse from .process import Process from schema_salad.ref_resolver import Loader, ContextType from schema_salad.jsonld_context import makerdf from rdflib import Graph, plugin, URIRef from rdflib.serializer import Serializer from typing import Any, Dict, IO, Text, Union def gather(tool, ctx): # type: (Process, ContextType) -> Graph g = Graph() def visitor(t): makerdf(t["id"], t, ctx, graph=g) tool.visit(visitor) return g def printrdf(wf, ctx, sr, stdout): # type: (Process, ContextType, Text, IO[Any]) -> None stdout.write(gather(wf, ctx).serialize(format=sr)) def lastpart(uri): # type: (Any) -> Text uri = Text(uri) if "/" in uri: return uri[uri.rindex("/")+1:] else: return uri def dot_with_parameters(g, stdout): # type: (Graph, IO[Any]) -> None qres = g.query( """SELECT ?step ?run ?runtype WHERE { ?step cwl:run ?run . ?run rdf:type ?runtype . }""") for step, run, runtype in qres: stdout.write(u'"%s" [label="%s"]\n' % (lastpart(step), "%s (%s)" % (lastpart(step), lastpart(run)))) qres = g.query( """SELECT ?step ?inp ?source WHERE { ?wf Workflow:steps ?step . ?step cwl:inputs ?inp . ?inp cwl:source ?source . }""") for step, inp, source in qres: stdout.write(u'"%s" [shape=box]\n' % (lastpart(inp))) stdout.write(u'"%s" -> "%s" [label="%s"]\n' % (lastpart(source), lastpart(inp), "")) stdout.write(u'"%s" -> "%s" [label="%s"]\n' % (lastpart(inp), lastpart(step), "")) qres = g.query( """SELECT ?step ?out WHERE { ?wf Workflow:steps ?step . ?step cwl:outputs ?out . }""") for step, out in qres: stdout.write(u'"%s" [shape=box]\n' % (lastpart(out))) stdout.write(u'"%s" -> "%s" [label="%s"]\n' % (lastpart(step), lastpart(out), "")) qres = g.query( """SELECT ?out ?source WHERE { ?wf cwl:outputs ?out . ?out cwl:source ?source . }""") for out, source in qres: stdout.write(u'"%s" [shape=octagon]\n' % (lastpart(out))) stdout.write(u'"%s" -> "%s" [label="%s"]\n' % (lastpart(source), lastpart(out), "")) qres = g.query( """SELECT ?inp WHERE { ?wf rdf:type cwl:Workflow . ?wf cwl:inputs ?inp . }""") for (inp,) in qres: stdout.write(u'"%s" [shape=octagon]\n' % (lastpart(inp))) def dot_without_parameters(g, stdout): # type: (Graph, IO[Any]) -> None dotname = {} # type: Dict[Text,Text] clusternode = {} stdout.write("compound=true\n") subworkflows = set() qres = g.query( """SELECT ?run WHERE { ?wf rdf:type cwl:Workflow . ?wf Workflow:steps ?step . ?step cwl:run ?run . ?run rdf:type cwl:Workflow . } ORDER BY ?wf""") for (run,) in qres: subworkflows.add(run) qres = g.query( """SELECT ?wf ?step ?run ?runtype WHERE { ?wf rdf:type cwl:Workflow . ?wf Workflow:steps ?step . ?step cwl:run ?run . ?run rdf:type ?runtype . } ORDER BY ?wf""") currentwf = None for wf, step, run, runtype in qres: if step not in dotname: dotname[step] = lastpart(step) if wf != currentwf: if currentwf is not None: stdout.write("}\n") if wf in subworkflows: if wf not in dotname: dotname[wf] = "cluster_" + lastpart(wf) stdout.write(u'subgraph "%s" { label="%s"\n' % (dotname[wf], lastpart(wf))) currentwf = wf clusternode[wf] = step else: currentwf = None if Text(runtype) != "https://w3id.org/cwl/cwl#Workflow": stdout.write(u'"%s" [label="%s"]\n' % (dotname[step], urlparse.urldefrag(Text(step))[1])) if currentwf is not None: stdout.write("}\n") qres = g.query( """SELECT DISTINCT ?src ?sink ?srcrun ?sinkrun WHERE { ?wf1 Workflow:steps ?src . ?wf2 Workflow:steps ?sink . ?src cwl:out ?out . ?inp cwl:source ?out . ?sink cwl:in ?inp . ?src cwl:run ?srcrun . ?sink cwl:run ?sinkrun . }""") for src, sink, srcrun, sinkrun in qres: attr = u"" if srcrun in clusternode: attr += u'ltail="%s"' % dotname[srcrun] src = clusternode[srcrun] if sinkrun in clusternode: attr += u' lhead="%s"' % dotname[sinkrun] sink = clusternode[sinkrun] stdout.write(u'"%s" -> "%s" [%s]\n' % (dotname[src], dotname[sink], attr)) def printdot(wf, ctx, stdout, include_parameters=False): # type: (Process, ContextType, Any, bool) -> None g = gather(wf, ctx) stdout.write("digraph {") #g.namespace_manager.qname(predicate) if include_parameters: dot_with_parameters(g, stdout) else: dot_without_parameters(g, stdout) stdout.write("}") cwltool-1.0.20170114120503/cwltool/__main__.py0000644000175000017500000000006513036412052021206 0ustar michaelmichael00000000000000from . import main import sys sys.exit(main.main()) cwltool-1.0.20170114120503/cwltool/pack.py0000644000175000017500000000560313036412052020407 0ustar michaelmichael00000000000000import copy import json from schema_salad.ref_resolver import Loader from .process import scandeps, shortname, uniquename from typing import Union, Any, cast, Callable, Dict, Tuple, Type, IO, Text def flatten_deps(d, files): # type: (Any, Set[Text]) -> None if isinstance(d, list): for s in d: flatten_deps(s, files) elif isinstance(d, dict): if d["class"] == "File": files.add(d["location"]) if "secondaryFiles" in d: flatten_deps(d["secondaryFiles"], files) if "listing" in d: flatten_deps(d["listing"], files) def find_run(d, runs): # type: (Any, Set[Text]) -> None if isinstance(d, list): for s in d: find_run(s, runs) elif isinstance(d, dict): if "run" in d and isinstance(d["run"], (str, unicode)): runs.add(d["run"]) for s in d.values(): find_run(s, runs) def replace_refs(d, rewrite, stem, newstem): # type: (Any, Dict[Text, Text], Text, Text) -> None if isinstance(d, list): for s,v in enumerate(d): if isinstance(v, (str, unicode)) and v.startswith(stem): d[s] = newstem + v[len(stem):] else: replace_refs(v, rewrite, stem, newstem) elif isinstance(d, dict): if "run" in d and isinstance(d["run"], (str, unicode)): d["run"] = rewrite[d["run"]] for s,v in d.items(): if isinstance(v, (str, unicode)) and v.startswith(stem): d[s] = newstem + v[len(stem):] replace_refs(v, rewrite, stem, newstem) def pack(document_loader, processobj, uri, metadata): # type: (Loader, Union[Dict[Text, Any], List[Dict[Text, Any]]], Text, Dict[Text, Text]) -> Dict[Text, Any] def loadref(b, u): # type: (Text, Text) -> Union[Dict, List, Text] return document_loader.resolve_ref(u, base_url=b)[0] deps = scandeps(uri, processobj, set(("run",)), set(), loadref) fdeps = set((uri,)) flatten_deps(deps, fdeps) runs = set() # type: Set[Text] for f in fdeps: find_run(document_loader.idx[f], runs) names = set() # type: Set[Text] rewrite = {} if isinstance(processobj, list): for p in processobj: rewrite[p["id"]] = "#" + uniquename(shortname(p["id"]), names) else: rewrite[uri] = "#main" for r in sorted(runs): rewrite[r] = "#" + uniquename(shortname(r), names) packed = {"$graph": [], "cwlVersion": metadata["cwlVersion"] } # type: Dict[Text, Any] for r in sorted(rewrite.keys()): v = rewrite[r] dc = cast(Dict[Text, Any], copy.deepcopy(document_loader.idx[r])) dc["id"] = v for n in ("name", "cwlVersion"): if n in dc: del dc[n] replace_refs(dc, rewrite, r+"/" if "#" in r else r+"#", v+"/") packed["$graph"].append(dc) return packed cwltool-1.0.20170114120503/cwltool/pathmapper.py0000644000175000017500000002014713036412052021632 0ustar michaelmichael00000000000000import os import logging import stat import collections import uuid import urllib import urlparse from functools import partial from typing import Any, Callable, Set, Text, Tuple, Union import schema_salad.validate as validate from schema_salad.sourceline import SourceLine from schema_salad.ref_resolver import uri_file_path _logger = logging.getLogger("cwltool") MapperEnt = collections.namedtuple("MapperEnt", ["resolved", "target", "type"]) def adjustFiles(rec, op): # type: (Any, Union[Callable[..., Any], partial[Any]]) -> None """Apply a mapping function to each File path in the object `rec`.""" if isinstance(rec, dict): if rec.get("class") == "File": rec["path"] = op(rec["path"]) for d in rec: adjustFiles(rec[d], op) if isinstance(rec, list): for d in rec: adjustFiles(d, op) def adjustFileObjs(rec, op): # type: (Any, Union[Callable[..., Any], partial[Any]]) -> None """Apply an update function to each File object in the object `rec`.""" if isinstance(rec, dict): if rec.get("class") == "File": op(rec) for d in rec: adjustFileObjs(rec[d], op) if isinstance(rec, list): for d in rec: adjustFileObjs(d, op) def adjustDirObjs(rec, op): # type: (Any, Union[Callable[..., Any], partial[Any]]) -> None """Apply an update function to each Directory object in the object `rec`.""" if isinstance(rec, dict): if rec.get("class") == "Directory": op(rec) for key in rec: adjustDirObjs(rec[key], op) if isinstance(rec, list): for d in rec: adjustDirObjs(d, op) def normalizeFilesDirs(job): # type: (Union[List[Dict[Text, Any]], Dict[Text, Any]]) -> None def addLocation(d): if "location" not in d: if d["class"] == "File" and ("contents" not in d): raise validate.ValidationException("Anonymous file object must have 'contents' and 'basename' fields.") if d["class"] == "Directory" and ("listing" not in d or "basename" not in d): raise validate.ValidationException("Anonymous directory object must have 'listing' and 'basename' fields.") d["location"] = "_:" + Text(uuid.uuid4()) if "basename" not in d: d["basename"] = Text(uuid.uuid4()) if "basename" not in d: parse = urlparse.urlparse(d["location"]) d["basename"] = os.path.basename(urllib.url2pathname(parse.path)) adjustFileObjs(job, addLocation) adjustDirObjs(job, addLocation) def abspath(src, basedir): # type: (Text, Text) -> Text if src.startswith(u"file://"): ab = unicode(uri_file_path(str(src))) else: ab = src if os.path.isabs(src) else os.path.join(basedir, src) return ab def dedup(listing): # type: (List[Any]) -> List[Any] marksub = set() def mark(d): marksub.add(d["location"]) for l in listing: if l["class"] == "Directory": for e in l.get("listing", []): adjustFileObjs(e, mark) adjustDirObjs(e, mark) dd = [] markdup = set() # type: Set[Text] for r in listing: if r["location"] not in marksub and r["location"] not in markdup: dd.append(r) markdup.add(r["location"]) return dd class PathMapper(object): """Mapping of files from relative path provided in the file to a tuple of (absolute local path, absolute container path) The tao of PathMapper: The initializer takes a list of File and Directory objects, a base directory (for resolving relative references) and a staging directory (where the files are mapped to). The purpose of the setup method is to determine where each File or Directory should be placed on the target file system (relative to stagedir). If separatedirs=True, unrelated files will be isolated in their own directories under stagedir. If separatedirs=False, files and directories will all be placed in stagedir (with the possibility for name collisions...) The path map maps the "location" of the input Files and Directory objects to a tuple (resolved, target, type). The "resolved" field is the "real" path on the local file system (after resolving relative paths and traversing symlinks). The "target" is the path on the target file system (under stagedir). The type is the object type (one of File, Directory, CreateFile, WritableFile). The latter two (CreateFile, WritableFile) are used by InitialWorkDirRequirement to indicate files that are generated on the fly (CreateFile, in this case "resolved" holds the file contents instead of the path because they file doesn't exist) or copied into the output directory so they can be opened for update ("r+" or "a") (WritableFile). """ def __init__(self, referenced_files, basedir, stagedir, separateDirs=True): # type: (List[Any], Text, Text, bool) -> None self._pathmap = {} # type: Dict[Text, MapperEnt] self.stagedir = stagedir self.separateDirs = separateDirs self.setup(dedup(referenced_files), basedir) def visitlisting(self, listing, stagedir, basedir): # type: (List[Dict[Text, Any]], Text, Text) -> None for ld in listing: tgt = os.path.join(stagedir, ld["basename"]) if ld["class"] == "Directory": self.visit(ld, stagedir, basedir, copy=ld.get("writable", False)) else: self.visit(ld, stagedir, basedir, copy=ld.get("writable", False)) def visit(self, obj, stagedir, basedir, copy=False): # type: (Dict[Text, Any], Text, Text, bool) -> None tgt = os.path.join(stagedir, obj["basename"]) if obj["class"] == "Directory": self._pathmap[obj["location"]] = MapperEnt(obj["location"], tgt, "Directory") self.visitlisting(obj.get("listing", []), tgt, basedir) elif obj["class"] == "File": path = obj["location"] if path in self._pathmap: return ab = abspath(path, basedir) if "contents" in obj and obj["location"].startswith("_:"): self._pathmap[obj["location"]] = MapperEnt(obj["contents"], tgt, "CreateFile") else: if copy: self._pathmap[path] = MapperEnt(ab, tgt, "WritableFile") else: with SourceLine(obj, "location", validate.ValidationException): # Dereference symbolic links deref = ab st = os.lstat(deref) while stat.S_ISLNK(st.st_mode): rl = os.readlink(deref) deref = rl if os.path.isabs(rl) else os.path.join( os.path.dirname(deref), rl) st = os.lstat(deref) self._pathmap[path] = MapperEnt(deref, tgt, "File") self.visitlisting(obj.get("secondaryFiles", []), stagedir, basedir) def setup(self, referenced_files, basedir): # type: (List[Any], Text) -> None # Go through each file and set the target to its own directory along # with any secondary files. stagedir = self.stagedir for fob in referenced_files: if self.separateDirs: stagedir = os.path.join(self.stagedir, "stg%s" % uuid.uuid4()) self.visit(fob, stagedir, basedir) def mapper(self, src): # type: (Text) -> MapperEnt if u"#" in src: i = src.index(u"#") p = self._pathmap[src[:i]] return MapperEnt(p.resolved, p.target + src[i:], None) else: return self._pathmap[src] def files(self): # type: () -> List[Text] return self._pathmap.keys() def items(self): # type: () -> List[Tuple[Text, MapperEnt]] return self._pathmap.items() def reversemap(self, target): # type: (Text) -> Tuple[Text, Text] for k, v in self._pathmap.items(): if v[1] == target: return (k, v[0]) return None cwltool-1.0.20170114120503/cwltool/workflow.py0000644000175000017500000007342413036412053021352 0ustar michaelmichael00000000000000import copy import logging import random import os from collections import namedtuple import functools import urlparse import tempfile import shutil import json from typing import Any, Callable, cast, Generator, Iterable, List, Text, Union import schema_salad.validate as validate from schema_salad.sourceline import SourceLine from . import job from . import draft2tool from .utils import aslist from .process import Process, get_feature, empty_subtree, shortname, uniquename from .errors import WorkflowException from . import expression from .load_tool import load_tool _logger = logging.getLogger("cwltool") WorkflowStateItem = namedtuple('WorkflowStateItem', ['parameter', 'value']) def defaultMakeTool(toolpath_object, **kwargs): # type: (Dict[Text, Any], **Any) -> Process if not isinstance(toolpath_object, dict): raise WorkflowException(u"Not a dict: `%s`" % toolpath_object) if "class" in toolpath_object: if toolpath_object["class"] == "CommandLineTool": return draft2tool.CommandLineTool(toolpath_object, **kwargs) elif toolpath_object["class"] == "ExpressionTool": return draft2tool.ExpressionTool(toolpath_object, **kwargs) elif toolpath_object["class"] == "Workflow": return Workflow(toolpath_object, **kwargs) raise WorkflowException(u"Missing or invalid 'class' field in %s, expecting one of: CommandLineTool, ExpressionTool, Workflow" % toolpath_object["id"]) def findfiles(wo, fn=None): # type: (Any, List) -> List[Dict[Text, Any]] if fn is None: fn = [] if isinstance(wo, dict): if wo.get("class") == "File": fn.append(wo) findfiles(wo.get("secondaryFiles", None), fn) else: for w in wo.values(): findfiles(w, fn) elif isinstance(wo, list): for w in wo: findfiles(w, fn) return fn def match_types(sinktype, src, iid, inputobj, linkMerge, valueFrom): # type: (Union[List[Text],Text], WorkflowStateItem, Text, Dict[Text, Any], Text, Text) -> bool if isinstance(sinktype, list): # Sink is union type for st in sinktype: if match_types(st, src, iid, inputobj, linkMerge, valueFrom): return True elif isinstance(src.parameter["type"], list): # Source is union type # Check that every source type is compatible with the sink. for st in src.parameter["type"]: srccopy = copy.deepcopy(src) srccopy.parameter["type"] = st if not match_types(st, srccopy, iid, inputobj, linkMerge, valueFrom): return False return True elif linkMerge: if iid not in inputobj: inputobj[iid] = [] if linkMerge == "merge_nested": inputobj[iid].append(src.value) elif linkMerge == "merge_flattened": if isinstance(src.value, list): inputobj[iid].extend(src.value) else: inputobj[iid].append(src.value) else: raise WorkflowException(u"Unrecognized linkMerge enum '%s'" % linkMerge) return True elif valueFrom is not None or can_assign_src_to_sink(src.parameter["type"], sinktype) or sinktype == "Any": # simply assign the value from state to input inputobj[iid] = copy.deepcopy(src.value) return True return False def can_assign_src_to_sink(src, sink): # type: (Any, Any) -> bool """Check for identical type specifications, ignoring extra keys like inputBinding. """ if sink == "Any": return True if isinstance(src, dict) and isinstance(sink, dict): if src["type"] == "array" and sink["type"] == "array": return can_assign_src_to_sink(src["items"], sink["items"]) elif src["type"] == "record" and sink["type"] == "record": return _compare_records(src, sink) elif isinstance(src, list): for t in src: if can_assign_src_to_sink(t, sink): return True elif isinstance(sink, list): for t in sink: if can_assign_src_to_sink(src, t): return True else: return src == sink return False def _compare_records(src, sink): # type: (Dict[Text, Any], Dict[Text, Any]) -> bool """Compare two records, ensuring they have compatible fields. This handles normalizing record names, which will be relative to workflow step, so that they can be compared. """ def _rec_fields(rec): # type: (Dict[Text, Any]) -> Dict[Text, Any] out = {} for field in rec["fields"]: name = shortname(field["name"]) out[name] = field["type"] return out srcfields = _rec_fields(src) sinkfields = _rec_fields(sink) for key in sinkfields.iterkeys(): if (not can_assign_src_to_sink( srcfields.get(key, "null"), sinkfields.get(key, "null")) and sinkfields.get(key) is not None): _logger.info("Record comparison failure for %s and %s\n" "Did not match fields for %s: %s and %s" % (src["name"], sink["name"], key, srcfields.get(key), sinkfields.get(key))) return False return True def object_from_state(state, parms, frag_only, supportsMultipleInput, sourceField, incomplete=False): # type: (Dict[Text, WorkflowStateItem], List[Dict[Text, Any]], bool, bool, Text, bool) -> Dict[Text, Any] inputobj = {} # type: Dict[Text, Any] for inp in parms: iid = inp["id"] if frag_only: iid = shortname(iid) if sourceField in inp: if (isinstance(inp[sourceField], list) and not supportsMultipleInput): raise WorkflowException( "Workflow contains multiple inbound links to a single " "parameter but MultipleInputFeatureRequirement is not " "declared.") connections = aslist(inp[sourceField]) for src in connections: if src in state and state[src] is not None: if not match_types( inp["type"], state[src], iid, inputobj, inp.get("linkMerge", ("merge_nested" if len(connections) > 1 else None)), valueFrom=inp.get("valueFrom")): raise WorkflowException( u"Type mismatch between source '%s' (%s) and " "sink '%s' (%s)" % (src, state[src].parameter["type"], inp["id"], inp["type"])) elif src not in state: raise WorkflowException( u"Connect source '%s' on parameter '%s' does not " "exist" % (src, inp["id"])) elif not incomplete: return None elif "default" in inp: inputobj[iid] = inp["default"] elif "valueFrom" in inp: inputobj[iid] = None else: raise WorkflowException(u"Value for %s not specified" % (inp["id"])) return inputobj class WorkflowJobStep(object): def __init__(self, step): # type: (Any) -> None self.step = step self.tool = step.tool self.id = step.id self.submitted = False self.completed = False self.iterable = None # type: Iterable self.name = uniquename(u"step %s" % shortname(self.id)) def job(self, joborder, output_callback, **kwargs): # type: (Dict[Text, Text], functools.partial[None], **Any) -> Generator kwargs["part_of"] = self.name kwargs["name"] = shortname(self.id) for j in self.step.job(joborder, output_callback, **kwargs): yield j class WorkflowJob(object): def __init__(self, workflow, **kwargs): # type: (Workflow, **Any) -> None self.workflow = workflow self.tool = workflow.tool self.steps = [WorkflowJobStep(s) for s in workflow.steps] self.state = None # type: Dict[Text, WorkflowStateItem] self.processStatus = None # type: Text if "outdir" in kwargs: self.outdir = kwargs["outdir"] elif "tmp_outdir_prefix" in kwargs: self.outdir = tempfile.mkdtemp(prefix=kwargs["tmp_outdir_prefix"]) else: # tmp_outdir_prefix defaults to tmp, so this is unlikely to be used self.outdir = tempfile.mkdtemp() self.name = uniquename(u"workflow %s" % kwargs.get("name", shortname(self.workflow.tool.get("id", "embedded")))) _logger.debug(u"[%s] initialized from %s", self.name, self.tool.get("id", "workflow embedded in %s" % kwargs.get("part_of"))) def receive_output(self, step, outputparms, jobout, processStatus): # type: (WorkflowJobStep, List[Dict[Text,Text]], Dict[Text,Text], Text) -> None for i in outputparms: if "id" in i: if i["id"] in jobout: self.state[i["id"]] = WorkflowStateItem(i, jobout[i["id"]]) else: _logger.error(u"[%s] Output is missing expected field %s", step.name, i["id"]) processStatus = "permanentFail" if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"[%s] produced output %s", step.name, json.dumps(jobout, indent=4)) if processStatus != "success": if self.processStatus != "permanentFail": self.processStatus = processStatus _logger.warn(u"[%s] completed %s", step.name, processStatus) else: _logger.info(u"[%s] completed %s", step.name, processStatus) step.completed = True def try_make_job(self, step, **kwargs): # type: (WorkflowJobStep, **Any) -> Generator inputparms = step.tool["inputs"] outputparms = step.tool["outputs"] supportsMultipleInput = bool(self.workflow.get_requirement( "MultipleInputFeatureRequirement")[0]) try: inputobj = object_from_state( self.state, inputparms, False, supportsMultipleInput, "source") if inputobj is None: _logger.debug(u"[%s] job step %s not ready", self.name, step.id) return if step.submitted: return _logger.debug(u"[%s] starting %s", self.name, step.name) callback = functools.partial(self.receive_output, step, outputparms) valueFrom = { i["id"]: i["valueFrom"] for i in step.tool["inputs"] if "valueFrom" in i} if len(valueFrom) > 0 and not bool(self.workflow.get_requirement("StepInputExpressionRequirement")[0]): raise WorkflowException("Workflow step contains valueFrom but StepInputExpressionRequirement not in requirements") vfinputs = {shortname(k): v for k,v in inputobj.iteritems()} def postScatterEval(io): # type: (Dict[Text, Any]) -> Dict[Text, Any] shortio = {shortname(k): v for k,v in io.iteritems()} def valueFromFunc(k, v): # type: (Any, Any) -> Any if k in valueFrom: return expression.do_eval( valueFrom[k], shortio, self.workflow.requirements, None, None, {}, context=v) else: return v return {k: valueFromFunc(k, v) for k,v in io.items()} if "scatter" in step.tool: scatter = aslist(step.tool["scatter"]) method = step.tool.get("scatterMethod") if method is None and len(scatter) != 1: raise WorkflowException("Must specify scatterMethod when scattering over multiple inputs") kwargs["postScatterEval"] = postScatterEval if method == "dotproduct" or method is None: jobs = dotproduct_scatter(step, inputobj, scatter, cast( # known bug with mypy # https://github.com/python/mypy/issues/797 Callable[[Any], Any],callback), **kwargs) elif method == "nested_crossproduct": jobs = nested_crossproduct_scatter(step, inputobj, scatter, cast(Callable[[Any], Any], callback), # known bug in mypy # https://github.com/python/mypy/issues/797 **kwargs) elif method == "flat_crossproduct": jobs = cast(Generator, flat_crossproduct_scatter(step, inputobj, scatter, cast(Callable[[Any], Any], # known bug in mypy # https://github.com/python/mypy/issues/797 callback), 0, **kwargs)) else: if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"[job %s] job input %s", step.name, json.dumps(inputobj, indent=4)) inputobj = postScatterEval(inputobj) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"[job %s] evaluated job input to %s", step.name, json.dumps(inputobj, indent=4)) jobs = step.job(inputobj, callback, **kwargs) step.submitted = True for j in jobs: yield j except WorkflowException: raise except Exception: _logger.exception("Unhandled exception") self.processStatus = "permanentFail" step.completed = True def run(self, **kwargs): _logger.debug(u"[%s] workflow starting", self.name) def job(self, joborder, output_callback, **kwargs): # type: (Dict[Text, Any], Callable[[Any, Any], Any], **Any) -> Generator self.state = {} self.processStatus = "success" if "outdir" in kwargs: del kwargs["outdir"] for e, i in enumerate(self.tool["inputs"]): with SourceLine(self.tool["inputs"], e, WorkflowException): iid = shortname(i["id"]) if iid in joborder: self.state[i["id"]] = WorkflowStateItem(i, copy.deepcopy(joborder[iid])) elif "default" in i: self.state[i["id"]] = WorkflowStateItem(i, copy.deepcopy(i["default"])) else: raise WorkflowException(u"Input '%s' not in input object and does not have a default value." % (i["id"])) for s in self.steps: for out in s.tool["outputs"]: self.state[out["id"]] = None completed = 0 while completed < len(self.steps): made_progress = False for step in self.steps: if kwargs.get("on_error", "stop") == "stop" and self.processStatus != "success": break if not step.submitted: try: step.iterable = self.try_make_job(step, **kwargs) except WorkflowException as e: _logger.error(u"[%s] Cannot make job: %s", step.name, e) _logger.debug("", exc_info=True) self.processStatus = "permanentFail" if step.iterable: try: for newjob in step.iterable: if kwargs.get("on_error", "stop") == "stop" and self.processStatus != "success": break if newjob: made_progress = True yield newjob else: break except WorkflowException as e: _logger.error(u"[%s] Cannot make job: %s", step.name, e) _logger.debug("", exc_info=True) self.processStatus = "permanentFail" completed = sum(1 for s in self.steps if s.completed) if not made_progress and completed < len(self.steps): if self.processStatus != "success": break else: yield None supportsMultipleInput = bool(self.workflow.get_requirement("MultipleInputFeatureRequirement")[0]) try: wo = object_from_state(self.state, self.tool["outputs"], True, supportsMultipleInput, "outputSource", incomplete=True) except WorkflowException as e: _logger.error(u"[%s] Cannot collect workflow output: %s", self.name, e) wo = {} self.processStatus = "permanentFail" _logger.info(u"[%s] outdir is %s", self.name, self.outdir) output_callback(wo, self.processStatus) class Workflow(Process): def __init__(self, toolpath_object, **kwargs): # type: (Dict[Text, Any], **Any) -> None super(Workflow, self).__init__(toolpath_object, **kwargs) kwargs["requirements"] = self.requirements kwargs["hints"] = self.hints makeTool = kwargs.get("makeTool") self.steps = [WorkflowStep(step, n, **kwargs) for n,step in enumerate(self.tool.get("steps", []))] random.shuffle(self.steps) # TODO: statically validate data links instead of doing it at runtime. def job(self, joborder, output_callback, **kwargs): # type: (Dict[Text, Text], Callable[[Any, Any], Any], **Any) -> Generator builder = self._init_job(joborder, **kwargs) wj = WorkflowJob(self, **kwargs) yield wj kwargs["part_of"] = u"workflow %s" % wj.name for w in wj.job(builder.job, output_callback, **kwargs): yield w def visit(self, op): op(self.tool) for s in self.steps: s.visit(op) class WorkflowStep(Process): def __init__(self, toolpath_object, pos, **kwargs): # type: (Dict[Text, Any], int, **Any) -> None if "id" in toolpath_object: self.id = toolpath_object["id"] else: self.id = "#step" + Text(pos) kwargs["requirements"] = kwargs.get("requirements", []) + toolpath_object.get("requirements", []) kwargs["hints"] = kwargs.get("hints", []) + toolpath_object.get("hints", []) try: if isinstance(toolpath_object["run"], dict): self.embedded_tool = kwargs.get("makeTool")(toolpath_object["run"], **kwargs) else: self.embedded_tool = load_tool( toolpath_object["run"], kwargs.get("makeTool"), kwargs, enable_dev=kwargs.get("enable_dev"), strict=kwargs.get("strict"), fetcher_constructor=kwargs.get("fetcher_constructor")) except validate.ValidationException as v: raise WorkflowException( u"Tool definition %s failed validation:\n%s" % (toolpath_object["run"], validate.indent(str(v)))) self.tool = toolpath_object = copy.deepcopy(toolpath_object) for stepfield, toolfield in (("in", "inputs"), ("out", "outputs")): toolpath_object[toolfield] = [] for step_entry in toolpath_object[stepfield]: if isinstance(step_entry, (str, unicode)): param = {} # type: Dict[Text, Any] inputid = step_entry else: param = copy.copy(step_entry) inputid = step_entry["id"] shortinputid = shortname(inputid) found = False for tool_entry in self.embedded_tool.tool[toolfield]: frag = shortname(tool_entry["id"]) if frag == shortinputid: param.update(tool_entry) found = True break if not found: if stepfield == "in": param["type"] = "Any" else: raise WorkflowException("[%s] Workflow step output '%s' not found in the outputs of the tool (expected one of '%s')" % ( self.id, shortname(step_entry), "', '".join([shortname(tool_entry["id"]) for tool_entry in self.embedded_tool.tool[toolfield]]))) param["id"] = inputid toolpath_object[toolfield].append(param) super(WorkflowStep, self).__init__(toolpath_object, **kwargs) if self.embedded_tool.tool["class"] == "Workflow": (feature, _) = self.get_requirement("SubworkflowFeatureRequirement") if not feature: raise WorkflowException("Workflow contains embedded workflow but SubworkflowFeatureRequirement not in requirements") if "scatter" in self.tool: (feature, _) = self.get_requirement("ScatterFeatureRequirement") if not feature: raise WorkflowException("Workflow contains scatter but ScatterFeatureRequirement not in requirements") inputparms = copy.deepcopy(self.tool["inputs"]) outputparms = copy.deepcopy(self.tool["outputs"]) scatter = aslist(self.tool["scatter"]) method = self.tool.get("scatterMethod") if method is None and len(scatter) != 1: raise WorkflowException("Must specify scatterMethod when scattering over multiple inputs") inp_map = {i["id"]: i for i in inputparms} for s in scatter: if s not in inp_map: raise WorkflowException(u"Scatter parameter '%s' does not correspond to an input parameter of this step, inputs are %s" % (s, inp_map.keys())) inp_map[s]["type"] = {"type": "array", "items": inp_map[s]["type"]} if self.tool.get("scatterMethod") == "nested_crossproduct": nesting = len(scatter) else: nesting = 1 for r in xrange(0, nesting): for i in outputparms: i["type"] = {"type": "array", "items": i["type"]} self.tool["inputs"] = inputparms self.tool["outputs"] = outputparms def receive_output(self, output_callback, jobout, processStatus): # type: (Callable[...,Any], Dict[Text, Text], Text) -> None #_logger.debug("WorkflowStep output from run is %s", jobout) output = {} for i in self.tool["outputs"]: field = shortname(i["id"]) if field in jobout: output[i["id"]] = jobout[field] else: processStatus = "permanentFail" output_callback(output, processStatus) def job(self, joborder, output_callback, **kwargs): # type: (Dict[Text, Any], Callable[...,Any], **Any) -> Generator for i in self.tool["inputs"]: p = i["id"] field = shortname(p) joborder[field] = joborder[i["id"]] del joborder[i["id"]] try: for t in self.embedded_tool.job(joborder, functools.partial(self.receive_output, output_callback), **kwargs): yield t except WorkflowException: _logger.error(u"Exception on step '%s'", kwargs.get("name")) raise except Exception as e: _logger.exception("Unexpected exception") raise WorkflowException(Text(e)) def visit(self, op): self.embedded_tool.visit(op) class ReceiveScatterOutput(object): def __init__(self, output_callback, dest): # type: (Callable[..., Any], Dict[Text,List[Text]]) -> None self.dest = dest self.completed = 0 self.processStatus = u"success" self.total = None # type: int self.output_callback = output_callback def receive_scatter_output(self, index, jobout, processStatus): # type: (int, Dict[Text, Text], Text) -> None for k,v in jobout.items(): self.dest[k][index] = v if processStatus != "success": if self.processStatus != "permanentFail": self.processStatus = processStatus self.completed += 1 if self.completed == self.total: self.output_callback(self.dest, self.processStatus) def setTotal(self, total): # type: (int) -> None self.total = total if self.completed == self.total: self.output_callback(self.dest, self.processStatus) def parallel_steps(steps, rc, kwargs): # type: (List[Generator], ReceiveScatterOutput, Dict[str, Any]) -> Generator while rc.completed < rc.total: made_progress = False for index in xrange(len(steps)): step = steps[index] if kwargs.get("on_error", "stop") == "stop" and rc.processStatus != "success": break try: for j in step: if kwargs.get("on_error", "stop") == "stop" and rc.processStatus != "success": break if j: made_progress = True yield j else: break except WorkflowException as e: _logger.error(u"Cannot make scatter job: %s", e) _logger.debug("", exc_info=True) rc.receive_scatter_output(index, {}, "permanentFail") if not made_progress and rc.completed < rc.total: yield None def dotproduct_scatter(process, joborder, scatter_keys, output_callback, **kwargs): # type: (WorkflowJobStep, Dict[Text, Any], List[Text], Callable[..., Any], **Any) -> Generator l = None for s in scatter_keys: if l is None: l = len(joborder[s]) elif l != len(joborder[s]): raise WorkflowException("Length of input arrays must be equal when performing dotproduct scatter.") output = {} # type: Dict[Text,List[Text]] for i in process.tool["outputs"]: output[i["id"]] = [None] * l rc = ReceiveScatterOutput(output_callback, output) steps = [] for n in range(0, l): jo = copy.copy(joborder) for s in scatter_keys: jo[s] = joborder[s][n] jo = kwargs["postScatterEval"](jo) steps.append(process.job(jo, functools.partial(rc.receive_scatter_output, n), **kwargs)) rc.setTotal(l) return parallel_steps(steps, rc, kwargs) def nested_crossproduct_scatter(process, joborder, scatter_keys, output_callback, **kwargs): # type: (WorkflowJobStep, Dict[Text, Any], List[Text], Callable[..., Any], **Any) -> Generator scatter_key = scatter_keys[0] l = len(joborder[scatter_key]) output = {} # type: Dict[Text,List[Text]] for i in process.tool["outputs"]: output[i["id"]] = [None] * l rc = ReceiveScatterOutput(output_callback, output) steps = [] for n in range(0, l): jo = copy.copy(joborder) jo[scatter_key] = joborder[scatter_key][n] if len(scatter_keys) == 1: jo = kwargs["postScatterEval"](jo) steps.append(process.job(jo, functools.partial(rc.receive_scatter_output, n), **kwargs)) else: steps.append(nested_crossproduct_scatter(process, jo, scatter_keys[1:], cast( # known bug with mypy # https://github.com/python/mypy/issues/797g Callable[[Any], Any], functools.partial(rc.receive_scatter_output, n)), **kwargs)) rc.setTotal(l) return parallel_steps(steps, rc, kwargs) def crossproduct_size(joborder, scatter_keys): # type: (Dict[Text, Any], List[Text]) -> int scatter_key = scatter_keys[0] if len(scatter_keys) == 1: sum = len(joborder[scatter_key]) else: sum = 0 for n in range(0, len(joborder[scatter_key])): jo = copy.copy(joborder) jo[scatter_key] = joborder[scatter_key][n] sum += crossproduct_size(joborder, scatter_keys[1:]) return sum def flat_crossproduct_scatter(process, joborder, scatter_keys, output_callback, startindex, **kwargs): # type: (WorkflowJobStep, Dict[Text, Any], List[Text], Union[ReceiveScatterOutput,Callable[..., Any]], int, **Any) -> Union[List[Generator], Generator] scatter_key = scatter_keys[0] l = len(joborder[scatter_key]) rc = None # type: ReceiveScatterOutput if startindex == 0 and not isinstance(output_callback, ReceiveScatterOutput): output = {} # type: Dict[Text,List[Text]] for i in process.tool["outputs"]: output[i["id"]] = [None] * crossproduct_size(joborder, scatter_keys) rc = ReceiveScatterOutput(output_callback, output) elif isinstance(output_callback, ReceiveScatterOutput): rc = output_callback else: raise Exception("Unhandled code path. Please report this.") steps = [] put = startindex for n in range(0, l): jo = copy.copy(joborder) jo[scatter_key] = joborder[scatter_key][n] if len(scatter_keys) == 1: jo = kwargs["postScatterEval"](jo) steps.append(process.job(jo, functools.partial(rc.receive_scatter_output, put), **kwargs)) put += 1 else: add = flat_crossproduct_scatter(process, jo, scatter_keys[1:], rc, put, **kwargs) put += len(cast(List[Generator], add)) steps.extend(add) if startindex == 0 and not isinstance(output_callback, ReceiveScatterOutput): rc.setTotal(put) return parallel_steps(steps, rc, kwargs) else: return steps cwltool-1.0.20170114120503/cwltool/docker.py0000644000175000017500000001042713036412052020740 0ustar michaelmichael00000000000000import subprocess import logging import sys import requests import os from .errors import WorkflowException import re import tempfile from typing import Any, Text, Union _logger = logging.getLogger("cwltool") def get_image(dockerRequirement, pull_image, dry_run=False): # type: (Dict[Text, Text], bool, bool) -> bool found = False if "dockerImageId" not in dockerRequirement and "dockerPull" in dockerRequirement: dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"] for ln in subprocess.check_output( ["docker", "images", "--no-trunc", "--all"]).splitlines(): try: m = re.match(r"^([^ ]+)\s+([^ ]+)\s+([^ ]+)", ln) sp = dockerRequirement["dockerImageId"].split(":") if len(sp) == 1: sp.append("latest") # check for repository:tag match or image id match if ((sp[0] == m.group(1) and sp[1] == m.group(2)) or dockerRequirement["dockerImageId"] == m.group(3)): found = True break except ValueError: pass if not found and pull_image: cmd = [] # type: List[str] if "dockerPull" in dockerRequirement: cmd = ["docker", "pull", str(dockerRequirement["dockerPull"])] _logger.info(Text(cmd)) if not dry_run: subprocess.check_call(cmd, stdout=sys.stderr) found = True elif "dockerFile" in dockerRequirement: dockerfile_dir = str(tempfile.mkdtemp()) with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as df: df.write(dockerRequirement["dockerFile"]) cmd = ["docker", "build", "--tag=%s" % str(dockerRequirement["dockerImageId"]), dockerfile_dir] _logger.info(Text(cmd)) if not dry_run: subprocess.check_call(cmd, stdout=sys.stderr) found = True elif "dockerLoad" in dockerRequirement: cmd = ["docker", "load"] _logger.info(Text(cmd)) if not dry_run: if os.path.exists(dockerRequirement["dockerLoad"]): _logger.info(u"Loading docker image from %s", dockerRequirement["dockerLoad"]) with open(dockerRequirement["dockerLoad"], "rb") as f: loadproc = subprocess.Popen(cmd, stdin=f, stdout=sys.stderr) else: loadproc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=sys.stderr) _logger.info(u"Sending GET request to %s", dockerRequirement["dockerLoad"]) req = requests.get(dockerRequirement["dockerLoad"], stream=True) n = 0 for chunk in req.iter_content(1024*1024): n += len(chunk) _logger.info("\r%i bytes" % (n)) loadproc.stdin.write(chunk) loadproc.stdin.close() rcode = loadproc.wait() if rcode != 0: raise WorkflowException("Docker load returned non-zero exit status %i" % (rcode)) found = True elif "dockerImport" in dockerRequirement: cmd = ["docker", "import", str(dockerRequirement["dockerImport"]), str(dockerRequirement["dockerImageId"])] _logger.info(Text(cmd)) if not dry_run: subprocess.check_call(cmd, stdout=sys.stderr) found = True return found def get_from_requirements(r, req, pull_image, dry_run=False): # type: (Dict[Text, Text], bool, bool, bool) -> Text if r: errmsg = None try: subprocess.check_output(["docker", "version"]) except subprocess.CalledProcessError as e: errmsg = "Cannot communicate with docker daemon: " + Text(e) except OSError as e: errmsg = "'docker' executable not found: " + Text(e) if errmsg: if req: raise WorkflowException(errmsg) else: return None if get_image(r, pull_image, dry_run): return r["dockerImageId"] else: if req: raise WorkflowException(u"Docker image %s not found" % r["dockerImageId"]) return None cwltool-1.0.20170114120503/cwltool/docker_uid.py0000644000175000017500000000737213036412052021606 0ustar michaelmichael00000000000000import subprocess from typing import Text, Union def docker_vm_uid(): # type: () -> int """ Returns the UID of the default docker user inside the VM When a host is using boot2docker or docker-machine to run docker with boot2docker.iso (As on Mac OS X), the UID that mounts the shared filesystem inside the VirtualBox VM is likely different than the user's UID on the host. :return: The numeric UID (as a string) of the docker account inside the boot2docker VM """ if boot2docker_running(): return boot2docker_uid() elif docker_machine_running(): return docker_machine_uid() else: return None def check_output_and_strip(cmd): # type: (List[Text]) -> Text """ Passes a command list to subprocess.check_output, returning None if an expected exception is raised :param cmd: The command to execute :return: Stripped string output of the command, or None if error """ try: result = subprocess.check_output(cmd, stderr=subprocess.STDOUT) return result.strip() except (OSError, subprocess.CalledProcessError, TypeError, AttributeError): # OSError is raised if command doesn't exist # CalledProcessError is raised if command returns nonzero # AttributeError is raised if result cannot be strip()ped return None def docker_machine_name(): # type: () -> Text """ Get the machine name of the active docker-machine machine :return: Name of the active machine or None if error """ return check_output_and_strip(['docker-machine', 'active']) def cmd_output_matches(check_cmd, expected_status): # type: (List[Text], Text) -> bool """ Runs a command and compares output to expected :param check_cmd: Command list to execute :param expected_status: Expected output, e.g. "Running" or "poweroff" :return: Boolean value, indicating whether or not command result matched """ if check_output_and_strip(check_cmd) == expected_status: return True else: return False def boot2docker_running(): # type: () -> bool """ Checks if boot2docker CLI reports that boot2docker vm is running :return: True if vm is running, False otherwise """ return cmd_output_matches(['boot2docker', 'status'], 'running') def docker_machine_running(): # type: () -> bool """ Asks docker-machine for active machine and checks if its VM is running :return: True if vm is running, False otherwise """ machine_name = docker_machine_name() return cmd_output_matches(['docker-machine', 'status', machine_name], 'Running') def cmd_output_to_int(cmd): # type: (List[Text]) -> int """ Runs the provided command and returns the integer value of the result :param cmd: The command to run :return: Integer value of result, or None if an error occurred """ result = check_output_and_strip(cmd) # may return None if result is not None: try: return int(result) except ValueError: # ValueError is raised if int conversion fails return None def boot2docker_uid(): # type: () -> int """ Gets the UID of the docker user inside a running boot2docker vm :return: the UID, or None if error (e.g. boot2docker not present or stopped) """ return cmd_output_to_int(['boot2docker', 'ssh', 'id', '-u']) def docker_machine_uid(): # type: () -> int """ Asks docker-machine for active machine and gets the UID of the docker user inside the vm :return: the UID, or None if error (e.g. docker-machine not present or stopped) """ machine_name = docker_machine_name() return cmd_output_to_int(['docker-machine', 'ssh', machine_name, "id -u"]) if __name__ == '__main__': print docker_vm_uid() cwltool-1.0.20170114120503/cwltool/draft2tool.py0000644000175000017500000005770113036412052021557 0ustar michaelmichael00000000000000import shutil from functools import partial import json import copy import os import glob import logging import hashlib import re import urlparse import tempfile import errno import avro.schema import schema_salad.validate as validate from schema_salad.ref_resolver import file_uri, uri_file_path import shellescape from typing import Any, Callable, cast, Generator, Text, Union from .process import Process, shortname, uniquename, getListing, normalizeFilesDirs, compute_checksums from .errors import WorkflowException from .utils import aslist from . import expression from .builder import CONTENT_LIMIT, substitute, Builder, adjustFileObjs, adjustDirObjs from .pathmapper import PathMapper from .job import CommandLineJob from .stdfsaccess import StdFsAccess from schema_salad.sourceline import SourceLine, indent ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$") ACCEPTLIST_EN_RELAXED_RE = re.compile(r"^[ a-zA-Z0-9._+-]+$") # with spaces ACCEPTLIST_RE = ACCEPTLIST_EN_STRICT_RE from .flatten import flatten _logger = logging.getLogger("cwltool") class ExpressionTool(Process): def __init__(self, toolpath_object, **kwargs): # type: (Dict[Text, Any], **Any) -> None super(ExpressionTool, self).__init__(toolpath_object, **kwargs) class ExpressionJob(object): def __init__(self): # type: () -> None self.builder = None # type: Builder self.requirements = None # type: Dict[Text, Text] self.hints = None # type: Dict[Text, Text] self.collect_outputs = None # type: Callable[[Any], Any] self.output_callback = None # type: Callable[[Any, Any], Any] self.outdir = None # type: Text self.tmpdir = None # type: Text self.script = None # type: Dict[Text, Text] def run(self, **kwargs): # type: (**Any) -> None try: ev = self.builder.do_eval(self.script) normalizeFilesDirs(ev) self.output_callback(ev, "success") except Exception as e: _logger.warn(u"Failed to evaluate expression:\n%s", e, exc_info=kwargs.get('debug')) self.output_callback({}, "permanentFail") def job(self, joborder, output_callback, **kwargs): # type: (Dict[Text, Text], Callable[[Any, Any], Any], **Any) -> Generator[ExpressionTool.ExpressionJob, None, None] builder = self._init_job(joborder, **kwargs) j = ExpressionTool.ExpressionJob() j.builder = builder j.script = self.tool["expression"] j.output_callback = output_callback j.requirements = self.requirements j.hints = self.hints j.outdir = None j.tmpdir = None yield j def remove_path(f): # type: (Dict[Text, Any]) -> None if "path" in f: del f["path"] def revmap_file(builder, outdir, f): # type: (Builder, Text, Dict[Text, Any]) -> Union[Dict[Text, Any], None] """Remap a file from internal path to external path. For Docker, this maps from the path inside tho container to the path outside the container. Recognizes files in the pathmapper or remaps internal output directories to the external directory. """ split = urlparse.urlsplit(outdir) if not split.scheme: outdir = file_uri(str(outdir)) if "location" in f: if f["location"].startswith("file://"): path = uri_file_path(f["location"]) revmap_f = builder.pathmapper.reversemap(path) if revmap_f: f["location"] = revmap_f[1] elif path.startswith(builder.outdir): f["location"] = builder.fs_access.join(outdir, path[len(builder.outdir)+1:]) return f if "path" in f: path = f["path"] del f["path"] revmap_f = builder.pathmapper.reversemap(path) if revmap_f: f["location"] = revmap_f[1] return f elif path.startswith(builder.outdir): f["location"] = builder.fs_access.join(outdir, path[len(builder.outdir)+1:]) return f else: raise WorkflowException(u"Output file path %s must be within designated output directory (%s) or an input file pass through." % (path, builder.outdir)) raise WorkflowException(u"Output File object is missing both `location` and `path` fields: %s" % f) class CallbackJob(object): def __init__(self, job, output_callback, cachebuilder, jobcache): # type: (CommandLineTool, Callable[[Any, Any], Any], Builder, Text) -> None self.job = job self.output_callback = output_callback self.cachebuilder = cachebuilder self.outdir = jobcache def run(self, **kwargs): # type: (**Any) -> None self.output_callback(self.job.collect_output_ports( self.job.tool["outputs"], self.cachebuilder, self.outdir, kwargs.get("compute_checksum", True)), "success") # map files to assigned path inside a container. We need to also explicitly # walk over input as implicit reassignment doesn't reach everything in builder.bindings def check_adjust(builder, f): # type: (Builder, Dict[Text, Any]) -> Dict[Text, Any] f["path"] = builder.pathmapper.mapper(f["location"])[1] f["dirname"], f["basename"] = os.path.split(f["path"]) if f["class"] == "File": f["nameroot"], f["nameext"] = os.path.splitext(f["basename"]) if not ACCEPTLIST_RE.match(f["basename"]): raise WorkflowException("Invalid filename: '%s' contains illegal characters" % (f["basename"])) return f class CommandLineTool(Process): def __init__(self, toolpath_object, **kwargs): # type: (Dict[Text, Any], **Any) -> None super(CommandLineTool, self).__init__(toolpath_object, **kwargs) def makeJobRunner(self): # type: () -> CommandLineJob return CommandLineJob() def makePathMapper(self, reffiles, stagedir, **kwargs): # type: (List[Any], Text, **Any) -> PathMapper dockerReq, _ = self.get_requirement("DockerRequirement") return PathMapper(reffiles, kwargs["basedir"], stagedir) def job(self, joborder, output_callback, **kwargs): # type: (Dict[Text, Text], Callable[..., Any], **Any) -> Generator[Union[CommandLineJob, CallbackJob], None, None] jobname = uniquename(kwargs.get("name", shortname(self.tool.get("id", "job")))) if kwargs.get("cachedir"): cacheargs = kwargs.copy() cacheargs["outdir"] = "/out" cacheargs["tmpdir"] = "/tmp" cacheargs["stagedir"] = "/stage" cachebuilder = self._init_job(joborder, **cacheargs) cachebuilder.pathmapper = PathMapper(cachebuilder.files, kwargs["basedir"], cachebuilder.stagedir, separateDirs=False) _check_adjust = partial(check_adjust, cachebuilder) adjustFileObjs(cachebuilder.files, _check_adjust) adjustFileObjs(cachebuilder.bindings, _check_adjust) adjustDirObjs(cachebuilder.files, _check_adjust) adjustDirObjs(cachebuilder.bindings, _check_adjust) cmdline = flatten(map(cachebuilder.generate_arg, cachebuilder.bindings)) (docker_req, docker_is_req) = self.get_requirement("DockerRequirement") if docker_req and kwargs.get("use_container") is not False: dockerimg = docker_req.get("dockerImageId") or docker_req.get("dockerPull") cmdline = ["docker", "run", dockerimg] + cmdline keydict = {u"cmdline": cmdline} for _,f in cachebuilder.pathmapper.items(): if f.type == "File": st = os.stat(f.resolved) keydict[f.resolved] = [st.st_size, int(st.st_mtime * 1000)] interesting = {"DockerRequirement", "EnvVarRequirement", "CreateFileRequirement", "ShellCommandRequirement"} for rh in (self.requirements, self.hints): for r in reversed(rh): if r["class"] in interesting and r["class"] not in keydict: keydict[r["class"]] = r keydictstr = json.dumps(keydict, separators=(',',':'), sort_keys=True) cachekey = hashlib.md5(keydictstr).hexdigest() _logger.debug("[job %s] keydictstr is %s -> %s", jobname, keydictstr, cachekey) jobcache = os.path.join(kwargs["cachedir"], cachekey) jobcachepending = jobcache + ".pending" if os.path.isdir(jobcache) and not os.path.isfile(jobcachepending): if docker_req and kwargs.get("use_container") is not False: cachebuilder.outdir = kwargs.get("docker_outdir") or "/var/spool/cwl" else: cachebuilder.outdir = jobcache _logger.info("[job %s] Using cached output in %s", jobname, jobcache) yield CallbackJob(self, output_callback, cachebuilder, jobcache) return else: _logger.info("[job %s] Output of job will be cached in %s", jobname, jobcache) shutil.rmtree(jobcache, True) os.makedirs(jobcache) kwargs["outdir"] = jobcache open(jobcachepending, "w").close() def rm_pending_output_callback(output_callback, jobcachepending, outputs, processStatus): if processStatus == "success": os.remove(jobcachepending) output_callback(outputs, processStatus) output_callback = cast( Callable[..., Any], # known bug in mypy # https://github.com/python/mypy/issues/797 partial(rm_pending_output_callback, output_callback, jobcachepending)) builder = self._init_job(joborder, **kwargs) reffiles = copy.deepcopy(builder.files) j = self.makeJobRunner() j.builder = builder j.joborder = builder.job j.stdin = None j.stderr = None j.stdout = None j.successCodes = self.tool.get("successCodes") j.temporaryFailCodes = self.tool.get("temporaryFailCodes") j.permanentFailCodes = self.tool.get("permanentFailCodes") j.requirements = self.requirements j.hints = self.hints j.name = jobname if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"[job %s] initializing from %s%s", j.name, self.tool.get("id", ""), u" as part of %s" % kwargs["part_of"] if "part_of" in kwargs else "") _logger.debug(u"[job %s] %s", j.name, json.dumps(joborder, indent=4)) builder.pathmapper = None make_path_mapper_kwargs = kwargs if "stagedir" in make_path_mapper_kwargs: make_path_mapper_kwargs = make_path_mapper_kwargs.copy() del make_path_mapper_kwargs["stagedir"] builder.pathmapper = self.makePathMapper(reffiles, builder.stagedir, **make_path_mapper_kwargs) builder.requirements = j.requirements if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"[job %s] path mappings is %s", j.name, json.dumps({p: builder.pathmapper.mapper(p) for p in builder.pathmapper.files()}, indent=4)) _check_adjust = partial(check_adjust, builder) adjustFileObjs(builder.files, _check_adjust) adjustFileObjs(builder.bindings, _check_adjust) adjustDirObjs(builder.files, _check_adjust) adjustDirObjs(builder.bindings, _check_adjust) if self.tool.get("stdin"): with SourceLine(self.tool, "stdin", validate.ValidationException): j.stdin = builder.do_eval(self.tool["stdin"]) reffiles.append({"class": "File", "path": j.stdin}) if self.tool.get("stderr"): with SourceLine(self.tool, "stderr", validate.ValidationException): j.stderr = builder.do_eval(self.tool["stderr"]) if os.path.isabs(j.stderr) or ".." in j.stderr: raise validate.ValidationException("stderr must be a relative path, got '%s'" % j.stderr) if self.tool.get("stdout"): with SourceLine(self.tool, "stdout", validate.ValidationException): j.stdout = builder.do_eval(self.tool["stdout"]) if os.path.isabs(j.stdout) or ".." in j.stdout or not j.stdout: raise validate.ValidationException("stdout must be a relative path, got '%s'" % j.stdout) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"[job %s] command line bindings is %s", j.name, json.dumps(builder.bindings, indent=4)) dockerReq = self.get_requirement("DockerRequirement")[0] if dockerReq and kwargs.get("use_container"): out_prefix = kwargs.get("tmp_outdir_prefix") j.outdir = kwargs.get("outdir") or tempfile.mkdtemp(prefix=out_prefix) tmpdir_prefix = kwargs.get('tmpdir_prefix') j.tmpdir = kwargs.get("tmpdir") or tempfile.mkdtemp(prefix=tmpdir_prefix) j.stagedir = tempfile.mkdtemp(prefix=tmpdir_prefix) else: j.outdir = builder.outdir j.tmpdir = builder.tmpdir j.stagedir = builder.stagedir initialWorkdir = self.get_requirement("InitialWorkDirRequirement")[0] j.generatefiles = {"class": "Directory", "listing": [], "basename": ""} if initialWorkdir: ls = [] # type: List[Dict[Text, Any]] if isinstance(initialWorkdir["listing"], (str, Text)): ls = builder.do_eval(initialWorkdir["listing"]) else: for t in initialWorkdir["listing"]: if "entry" in t: et = {u"entry": builder.do_eval(t["entry"])} if "entryname" in t: et["entryname"] = builder.do_eval(t["entryname"]) else: et["entryname"] = None et["writable"] = t.get("writable", False) ls.append(et) else: ls.append(builder.do_eval(t)) for i,t in enumerate(ls): if "entry" in t: if isinstance(t["entry"], basestring): ls[i] = { "class": "File", "basename": t["entryname"], "contents": t["entry"], "writable": t.get("writable") } else: if t["entryname"]: t = copy.deepcopy(t) t["entry"]["basename"] = t["entryname"] t["entry"]["writable"] = t.get("writable") ls[i] = t["entry"] j.generatefiles[u"listing"] = ls normalizeFilesDirs(j.generatefiles) j.environment = {} evr = self.get_requirement("EnvVarRequirement")[0] if evr: for t in evr["envDef"]: j.environment[t["envName"]] = builder.do_eval(t["envValue"]) shellcmd = self.get_requirement("ShellCommandRequirement")[0] if shellcmd: cmd = [] # type: List[Text] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): arg = [shellescape.quote(a) for a in aslist(arg)] cmd.extend(aslist(arg)) j.command_line = ["/bin/sh", "-c", " ".join(cmd)] else: j.command_line = flatten(map(builder.generate_arg, builder.bindings)) j.pathmapper = builder.pathmapper j.collect_outputs = partial( self.collect_output_ports, self.tool["outputs"], builder, compute_checksum=kwargs.get("compute_checksum", True)) j.output_callback = output_callback yield j def collect_output_ports(self, ports, builder, outdir, compute_checksum=True): # type: (Set[Dict[Text, Any]], Builder, Text, bool) -> Dict[Text, Union[Text, List[Any], Dict[Text, Any]]] try: ret = {} # type: Dict[Text, Union[Text, List[Any], Dict[Text, Any]]] fs_access = builder.make_fs_access(outdir) custom_output = fs_access.join(outdir, "cwl.output.json") if fs_access.exists(custom_output): with fs_access.open(custom_output, "r") as f: ret = json.load(f) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"Raw output from %s: %s", custom_output, json.dumps(ret, indent=4)) else: for i, port in enumerate(ports): with SourceLine(ports, i, WorkflowException): fragment = shortname(port["id"]) try: ret[fragment] = self.collect_output(port, builder, outdir, fs_access, compute_checksum=compute_checksum) except Exception as e: _logger.debug( u"Error collecting output for parameter '%s'" % shortname(port["id"]), exc_info=True) raise WorkflowException( u"Error collecting output for parameter '%s':\n%s" % (shortname(port["id"]), indent(unicode(e)))) if ret: adjustFileObjs(ret, cast(Callable[[Any], Any], # known bug in mypy # https://github.com/python/mypy/issues/797 partial(revmap_file, builder, outdir))) adjustFileObjs(ret, remove_path) adjustDirObjs(ret, remove_path) normalizeFilesDirs(ret) if compute_checksum: adjustFileObjs(ret, partial(compute_checksums, fs_access)) validate.validate_ex(self.names.get_name("outputs_record_schema", ""), ret) return ret if ret is not None else {} except validate.ValidationException as e: raise WorkflowException("Error validating output record, " + Text(e) + "\n in " + json.dumps(ret, indent=4)) def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=True): # type: (Dict[Text, Any], Builder, Text, StdFsAccess, bool) -> Union[Dict[Text, Any], List[Union[Dict[Text, Any], Text]]] r = [] # type: List[Any] if "outputBinding" in schema: binding = schema["outputBinding"] globpatterns = [] # type: List[Text] revmap = partial(revmap_file, builder, outdir) if "glob" in binding: with SourceLine(binding, "glob", WorkflowException): for gb in aslist(binding["glob"]): gb = builder.do_eval(gb) if gb: globpatterns.extend(aslist(gb)) for gb in globpatterns: if gb.startswith(outdir): gb = gb[len(outdir)+1:] elif gb == ".": gb = outdir elif gb.startswith("/"): raise WorkflowException("glob patterns must not start with '/'") try: r.extend([{"location": g, "class": "File" if fs_access.isfile(g) else "Directory"} for g in fs_access.glob(fs_access.join(outdir, gb))]) except (OSError, IOError) as e: _logger.warn(Text(e)) for files in r: if files["class"] == "Directory" and "listing" not in files: getListing(fs_access, files) else: with fs_access.open(files["location"], "rb") as f: contents = "" if binding.get("loadContents") or compute_checksum: contents = f.read(CONTENT_LIMIT) if binding.get("loadContents"): files["contents"] = contents if compute_checksum: checksum = hashlib.sha1() while contents != "": checksum.update(contents) contents = f.read(1024*1024) files["checksum"] = "sha1$%s" % checksum.hexdigest() f.seek(0, 2) filesize = f.tell() files["size"] = filesize if "format" in schema: files["format"] = builder.do_eval(schema["format"], context=files) optional = False single = False if isinstance(schema["type"], list): if "null" in schema["type"]: optional = True if "File" in schema["type"] or "Directory" in schema["type"]: single = True elif schema["type"] == "File" or schema["type"] == "Directory": single = True if "outputEval" in binding: with SourceLine(binding, "outputEval", WorkflowException): r = builder.do_eval(binding["outputEval"], context=r) if single: if not r and not optional: with SourceLine(binding, "glob", WorkflowException): raise WorkflowException("Did not find output file with glob pattern: '{}'".format(globpatterns)) elif not r and optional: pass elif isinstance(r, list): if len(r) > 1: raise WorkflowException("Multiple matches for output item that is a single file.") else: r = r[0] # Ensure files point to local references outside of the run environment adjustFileObjs(r, cast( # known bug in mypy # https://github.com/python/mypy/issues/797 Callable[[Any], Any], revmap)) if "secondaryFiles" in schema: with SourceLine(schema, "secondaryFiles", WorkflowException): for primary in aslist(r): if isinstance(primary, dict): primary["secondaryFiles"] = [] for sf in aslist(schema["secondaryFiles"]): if isinstance(sf, dict) or "$(" in sf or "${" in sf: sfpath = builder.do_eval(sf, context=primary) if isinstance(sfpath, basestring): sfpath = revmap({"location": sfpath, "class": "File"}) else: sfpath = {"location": substitute(primary["location"], sf), "class": "File"} for sfitem in aslist(sfpath): if fs_access.exists(sfitem["location"]): primary["secondaryFiles"].append(sfitem) if not r and optional: r = None if (not r and isinstance(schema["type"], dict) and schema["type"]["type"] == "record"): out = {} for f in schema["type"]["fields"]: out[shortname(f["name"])] = self.collect_output( # type: ignore f, builder, outdir, fs_access, compute_checksum=compute_checksum) return out return r cwltool-1.0.20170114120503/cwltool/__init__.py0000644000175000017500000000005313036412052021222 0ustar michaelmichael00000000000000__author__ = 'peter.amstutz@curoverse.com' cwltool-1.0.20170114120503/cwltool/main.py0000755000175000017500000007665213036412052020434 0ustar michaelmichael00000000000000#!/usr/bin/env python import argparse import json import os import sys import logging import copy import tempfile import ruamel.yaml as yaml import urlparse import hashlib import pkg_resources # part of setuptools import functools import rdflib import requests from typing import (Union, Any, AnyStr, cast, Callable, Dict, Sequence, Text, Tuple, Type, IO) from schema_salad.ref_resolver import Loader, Fetcher, file_uri, uri_file_path import schema_salad.validate as validate import schema_salad.jsonld_context import schema_salad.makedoc from schema_salad.sourceline import strip_dup_lineno from . import workflow from .errors import WorkflowException, UnsupportedRequirement from .cwlrdf import printrdf, printdot from .process import shortname, Process, getListing, relocateOutputs, cleanIntermediate, scandeps, normalizeFilesDirs from .load_tool import fetch_document, validate_document, make_tool from . import draft2tool from .resolver import tool_resolver from .builder import adjustFileObjs, adjustDirObjs from .stdfsaccess import StdFsAccess from .pack import pack _logger = logging.getLogger("cwltool") defaultStreamHandler = logging.StreamHandler() _logger.addHandler(defaultStreamHandler) _logger.setLevel(logging.INFO) def arg_parser(): # type: () -> argparse.ArgumentParser parser = argparse.ArgumentParser(description='Reference executor for Common Workflow Language') parser.add_argument("--basedir", type=Text) parser.add_argument("--outdir", type=Text, default=os.path.abspath('.'), help="Output directory, default current directory") parser.add_argument("--no-container", action="store_false", default=True, help="Do not execute jobs in a Docker container, even when specified by the CommandLineTool", dest="use_container") parser.add_argument("--preserve-environment", type=Text, action="append", help="Preserve specific environment variable when running CommandLineTools. May be provided multiple times.", metavar="ENVVAR", default=["PATH"], dest="preserve_environment") parser.add_argument("--preserve-entire-environment", action="store_true", help="Preserve entire parent environment when running CommandLineTools.", default=False, dest="preserve_entire_environment") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--rm-container", action="store_true", default=True, help="Delete Docker container used by jobs after they exit (default)", dest="rm_container") exgroup.add_argument("--leave-container", action="store_false", default=True, help="Do not delete Docker container used by jobs after they exit", dest="rm_container") parser.add_argument("--tmpdir-prefix", type=Text, help="Path prefix for temporary directories", default="tmp") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--tmp-outdir-prefix", type=Text, help="Path prefix for intermediate output directories", default="tmp") exgroup.add_argument("--cachedir", type=Text, default="", help="Directory to cache intermediate workflow outputs to avoid recomputing steps.") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--rm-tmpdir", action="store_true", default=True, help="Delete intermediate temporary directories (default)", dest="rm_tmpdir") exgroup.add_argument("--leave-tmpdir", action="store_false", default=True, help="Do not delete intermediate temporary directories", dest="rm_tmpdir") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--move-outputs", action="store_const", const="move", default="move", help="Move output files to the workflow output directory and delete intermediate output directories (default).", dest="move_outputs") exgroup.add_argument("--leave-outputs", action="store_const", const="leave", default="move", help="Leave output files in intermediate output directories.", dest="move_outputs") exgroup.add_argument("--copy-outputs", action="store_const", const="copy", default="move", help="Copy output files to the workflow output directory, don't delete intermediate output directories.", dest="move_outputs") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--enable-pull", default=True, action="store_true", help="Try to pull Docker images", dest="enable_pull") exgroup.add_argument("--disable-pull", default=True, action="store_false", help="Do not try to pull Docker images", dest="enable_pull") parser.add_argument("--rdf-serializer", help="Output RDF serialization format used by --print-rdf (one of turtle (default), n3, nt, xml)", default="turtle") parser.add_argument("--eval-timeout", help="Time to wait for a Javascript expression to evaluate before giving an error, default 20s.", type=float, default=20) exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--print-rdf", action="store_true", help="Print corresponding RDF graph for workflow and exit") exgroup.add_argument("--print-dot", action="store_true", help="Print workflow visualization in graphviz format and exit") exgroup.add_argument("--print-pre", action="store_true", help="Print CWL document after preprocessing.") exgroup.add_argument("--print-deps", action="store_true", help="Print CWL document dependencies.") exgroup.add_argument("--print-input-deps", action="store_true", help="Print input object document dependencies.") exgroup.add_argument("--pack", action="store_true", help="Combine components into single document and print.") exgroup.add_argument("--version", action="store_true", help="Print version and exit") exgroup.add_argument("--validate", action="store_true", help="Validate CWL document only.") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--strict", action="store_true", help="Strict validation (unrecognized or out of place fields are error)", default=True, dest="strict") exgroup.add_argument("--non-strict", action="store_false", help="Lenient validation (ignore unrecognized fields)", default=True, dest="strict") exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--verbose", action="store_true", help="Default logging") exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.") exgroup.add_argument("--debug", action="store_true", help="Print even more logging") parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool") parser.add_argument("--relative-deps", choices=['primary', 'cwd'], default="primary", help="When using --print-deps, print paths " "relative to primary file or current working directory.") parser.add_argument("--enable-dev", action="store_true", help="Allow loading and running development versions " "of CWL spec.", default=False) parser.add_argument("--default-container", help="Specify a default docker container that will be used if the workflow fails to specify one.") parser.add_argument("--no-match-user", action="store_true", help="Disable passing the current uid to 'docker run --user`") parser.add_argument("--disable-net", action="store_true", help="Use docker's default networking for containers;" " the default is to enable networking.") parser.add_argument("--custom-net", type=Text, help="Will be passed to `docker run` as the '--net' " "parameter. Implies '--enable-net'.") parser.add_argument("--on-error", type=str, help="Desired workflow behavior when a step fails. One of 'stop' or 'continue'. " "Default is 'stop'.", default="stop", choices=("stop", "continue")) exgroup = parser.add_mutually_exclusive_group() exgroup.add_argument("--compute-checksum", action="store_true", default=True, help="Compute checksum of contents while collecting outputs", dest="compute_checksum") exgroup.add_argument("--no-compute-checksum", action="store_false", help="Do not compute checksum of contents while collecting outputs", dest="compute_checksum") parser.add_argument("--relax-path-checks", action="store_true", default=False, help="Relax requirements on path names. Currently " "allows spaces.", dest="relax_path_checks") parser.add_argument("workflow", type=Text, nargs="?", default=None) parser.add_argument("job_order", nargs=argparse.REMAINDER) return parser def single_job_executor(t, job_order_object, **kwargs): # type: (Process, Dict[Text, Any], **Any) -> Tuple[Dict[Text, Any], Text] final_output = [] final_status = [] def output_callback(out, processStatus): final_status.append(processStatus) final_output.append(out) if "basedir" not in kwargs: raise WorkflowException("Must provide 'basedir' in kwargs") output_dirs = set() finaloutdir = kwargs.get("outdir") kwargs["outdir"] = tempfile.mkdtemp(prefix=kwargs["tmp_outdir_prefix"]) if kwargs.get("tmp_outdir_prefix") else tempfile.mkdtemp() output_dirs.add(kwargs["outdir"]) jobReqs = None if "cwl:requirements" in job_order_object: jobReqs = job_order_object["cwl:requirements"] elif ("cwl:defaults" in t.metadata and "cwl:requirements" in t.metadata["cwl:defaults"]): jobReqs = t.metadata["cwl:defaults"]["cwl:requirements"] if jobReqs: for req in jobReqs: t.requirements.append(req) jobiter = t.job(job_order_object, output_callback, **kwargs) try: for r in jobiter: if r: if r.outdir: output_dirs.add(r.outdir) r.run(**kwargs) else: _logger.error("Workflow cannot make any more progress.") break except WorkflowException: raise except Exception as e: _logger.exception("Got workflow error") raise WorkflowException(Text(e)) if final_output and final_output[0] and finaloutdir: final_output[0] = relocateOutputs(final_output[0], finaloutdir, output_dirs, kwargs.get("move_outputs")) if kwargs.get("rm_tmpdir"): cleanIntermediate(output_dirs) if final_output and final_status: return (final_output[0], final_status[0]) else: return (None, "permanentFail") class FSAction(argparse.Action): objclass = None # type: Text def __init__(self, option_strings, dest, nargs=None, **kwargs): # type: (List[Text], Text, Any, **Any) -> None if nargs is not None: raise ValueError("nargs not allowed") super(FSAction, self).__init__(option_strings, dest, **kwargs) def __call__(self, parser, namespace, values, option_string=None): # type: (argparse.ArgumentParser, argparse.Namespace, Union[AnyStr, Sequence[Any], None], AnyStr) -> None setattr(namespace, self.dest, # type: ignore {"class": self.objclass, "location": file_uri(str(os.path.abspath(cast(AnyStr, values))))}) class FSAppendAction(argparse.Action): objclass = None # type: Text def __init__(self, option_strings, dest, nargs=None, **kwargs): # type: (List[Text], Text, Any, **Any) -> None if nargs is not None: raise ValueError("nargs not allowed") super(FSAppendAction, self).__init__(option_strings, dest, **kwargs) def __call__(self, parser, namespace, values, option_string=None): # type: (argparse.ArgumentParser, argparse.Namespace, Union[AnyStr, Sequence[Any], None], AnyStr) -> None g = getattr(namespace, self.dest # type: ignore ) if not g: g = [] setattr(namespace, self.dest, # type: ignore g) g.append( {"class": self.objclass, "location": file_uri(str(os.path.abspath(cast(AnyStr, values))))}) class FileAction(FSAction): objclass = "File" class DirectoryAction(FSAction): objclass = "Directory" class FileAppendAction(FSAppendAction): objclass = "File" class DirectoryAppendAction(FSAppendAction): objclass = "Directory" def add_argument(toolparser, name, inptype, records, description="", default=None): # type: (argparse.ArgumentParser, Text, Any, List[Text], Text, Any) -> None if len(name) == 1: flag = "-" else: flag = "--" required = True if isinstance(inptype, list): if inptype[0] == "null": required = False if len(inptype) == 2: inptype = inptype[1] else: _logger.debug(u"Can't make command line argument from %s", inptype) return None ahelp = description.replace("%", "%%") action = None # type: Union[argparse.Action, Text] atype = None # type: Any if inptype == "File": action = cast(argparse.Action, FileAction) elif inptype == "Directory": action = cast(argparse.Action, DirectoryAction) elif isinstance(inptype, dict) and inptype["type"] == "array": if inptype["items"] == "File": action = cast(argparse.Action, FileAppendAction) elif inptype["items"] == "Directory": action = cast(argparse.Action, DirectoryAppendAction) else: action = "append" elif isinstance(inptype, dict) and inptype["type"] == "enum": atype = Text elif isinstance(inptype, dict) and inptype["type"] == "record": records.append(name) for field in inptype['fields']: fieldname = name+"."+shortname(field['name']) fieldtype = field['type'] fielddescription = field.get("doc", "") add_argument( toolparser, fieldname, fieldtype, records, fielddescription) return if inptype == "string": atype = Text elif inptype == "int": atype = int elif inptype == "double": atype = float elif inptype == "float": atype = float elif inptype == "boolean": action = "store_true" if default: required = False if not atype and not action: _logger.debug(u"Can't make command line argument from %s", inptype) return None if inptype != "boolean": typekw = { 'type': atype } else: typekw = {} toolparser.add_argument( # type: ignore flag + name, required=required, help=ahelp, action=action, default=default, **typekw) def generate_parser(toolparser, tool, namemap, records): # type: (argparse.ArgumentParser, Process, Dict[Text, Text], List[Text]) -> argparse.ArgumentParser toolparser.add_argument("job_order", nargs="?", help="Job input json file") namemap["job_order"] = "job_order" for inp in tool.tool["inputs"]: name = shortname(inp["id"]) namemap[name.replace("-", "_")] = name inptype = inp["type"] description = inp.get("doc", "") default = inp.get("default", None) add_argument(toolparser, name, inptype, records, description, default) return toolparser def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False, stdout=sys.stdout, make_fs_access=None, fetcher_constructor=None): # type: (argparse.Namespace, Process, IO[Any], bool, bool, IO[Any], Callable[[Text], StdFsAccess], Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]) -> Union[int, Tuple[Dict[Text, Any], Text]] job_order_object = None jobloaderctx = { u"path": {u"@type": u"@id"}, u"location": {u"@type": u"@id"}, u"format": {u"@type": u"@id"}, u"id": u"@id"} jobloaderctx.update(t.metadata.get("$namespaces", {})) loader = Loader(jobloaderctx, fetcher_constructor=fetcher_constructor) if len(args.job_order) == 1 and args.job_order[0][0] != "-": job_order_file = args.job_order[0] elif len(args.job_order) == 1 and args.job_order[0] == "-": job_order_object = yaml.load(stdin) job_order_object, _ = loader.resolve_all(job_order_object, "") else: job_order_file = None if job_order_object: input_basedir = args.basedir if args.basedir else os.getcwd() elif job_order_file: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(job_order_file)) try: job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 toolparser = None else: input_basedir = args.basedir if args.basedir else os.getcwd() namemap = {} # type: Dict[Text, Text] records = [] # type: List[Text] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), t, namemap, records) if toolparser: if args.tool_help: toolparser.print_help() return 0 cmd_line = vars(toolparser.parse_args(args.job_order)) for record_name in records: record = {} record_items = { k:v for k,v in cmd_line.iteritems() if k.startswith(record_name)} for key, value in record_items.iteritems(): record[key[len(record_name)+1:]] = value del cmd_line[key] cmd_line[str(record_name)] = record if cmd_line["job_order"]: try: input_basedir = args.basedir if args.basedir else os.path.abspath(os.path.dirname(cmd_line["job_order"])) job_order_object = loader.resolve_ref(cmd_line["job_order"]) except Exception as e: _logger.error(Text(e), exc_info=args.debug) return 1 else: job_order_object = {"id": args.workflow} job_order_object.update({namemap[k]: v for k,v in cmd_line.items()}) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"Parsed job order from command line: %s", json.dumps(job_order_object, indent=4)) else: job_order_object = None for inp in t.tool["inputs"]: if "default" in inp and (not job_order_object or shortname(inp["id"]) not in job_order_object): if not job_order_object: job_order_object = {} job_order_object[shortname(inp["id"])] = inp["default"] if not job_order_object and len(t.tool["inputs"]) > 0: if toolparser: print u"\nOptions for %s " % args.workflow toolparser.print_help() _logger.error("") _logger.error("Input object required, use --help for details") return 1 if print_input_deps: printdeps(job_order_object, loader, stdout, relative_deps, "", basedir=file_uri(input_basedir+"/")) return 0 def pathToLoc(p): if "location" not in p and "path" in p: p["location"] = p["path"] del p["path"] adjustDirObjs(job_order_object, pathToLoc) adjustFileObjs(job_order_object, pathToLoc) normalizeFilesDirs(job_order_object) adjustDirObjs(job_order_object, cast(Callable[..., Any], functools.partial(getListing, make_fs_access(input_basedir)))) if "cwl:tool" in job_order_object: del job_order_object["cwl:tool"] if "id" in job_order_object: del job_order_object["id"] return (job_order_object, input_basedir) def makeRelative(base, ob): u = ob.get("location", ob.get("path")) if ":" in u.split("/")[0] and not u.startswith("file://"): pass else: if u.startswith("file://"): u = uri_file_path(u) ob["location"] = os.path.relpath(u, base) def printdeps(obj, document_loader, stdout, relative_deps, uri, basedir=None): # type: (Dict[Text, Any], Loader, IO[Any], bool, Text, Text) -> None deps = {"class": "File", "location": uri} # type: Dict[Text, Any] def loadref(b, u): return document_loader.fetch(document_loader.fetcher.urljoin(b, u)) sf = scandeps( basedir if basedir else uri, obj, set(("$import", "run")), set(("$include", "$schemas", "location")), loadref) if sf: deps["secondaryFiles"] = sf if relative_deps: if relative_deps == "primary": base = basedir if basedir else os.path.dirname(uri_file_path(str(uri))) elif relative_deps == "cwd": base = os.getcwd() else: raise Exception(u"Unknown relative_deps %s" % relative_deps) adjustFileObjs(deps, functools.partial(makeRelative, base)) adjustDirObjs(deps, functools.partial(makeRelative, base)) stdout.write(json.dumps(deps, indent=4)) def print_pack(document_loader, processobj, uri, metadata): # type: (Loader, Union[Dict[unicode, Any], List[Dict[unicode, Any]]], unicode, Dict[unicode, Any]) -> str packed = pack(document_loader, processobj, uri, metadata) if len(packed["$graph"]) > 1: return json.dumps(packed, indent=4) else: return json.dumps(packed["$graph"][0], indent=4) def versionstring(): # type: () -> Text pkg = pkg_resources.require("cwltool") if pkg: return u"%s %s" % (sys.argv[0], pkg[0].version) else: return u"%s %s" % (sys.argv[0], "unknown version") def main(argsl=None, # type: List[str] args=None, # type: argparse.Namespace executor=single_job_executor, # type: Callable[..., Tuple[Dict[Text, Any], Text]] makeTool=workflow.defaultMakeTool, # type: Callable[..., Process] selectResources=None, # type: Callable[[Dict[Text, int]], Dict[Text, int]] stdin=sys.stdin, # type: IO[Any] stdout=sys.stdout, # type: IO[Any] stderr=sys.stderr, # type: IO[Any] versionfunc=versionstring, # type: Callable[[], Text] job_order_object=None, # type: Union[Tuple[Dict[Text, Any], Text], int] make_fs_access=StdFsAccess, # type: Callable[[Text], StdFsAccess] fetcher_constructor=None, # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher] resolver=tool_resolver, logger_handler=None ): # type: (...) -> int _logger.removeHandler(defaultStreamHandler) if logger_handler: stderr_handler = logger_handler else: stderr_handler = logging.StreamHandler(stderr) _logger.addHandler(stderr_handler) try: if args is None: if argsl is None: argsl = sys.argv[1:] args = arg_parser().parse_args(argsl) # If caller provided custom arguments, it may be not every expected # option is set, so fill in no-op defaults to avoid crashing when # dereferencing them in args. for k,v in {'print_deps': False, 'print_pre': False, 'print_rdf': False, 'print_dot': False, 'relative_deps': False, 'tmp_outdir_prefix': 'tmp', 'tmpdir_prefix': 'tmp', 'print_input_deps': False, 'cachedir': None, 'quiet': False, 'debug': False, 'version': False, 'enable_dev': False, 'strict': True, 'rdf_serializer': None, 'basedir': None, 'tool_help': False, 'workflow': None, 'job_order': None, 'pack': False, 'on_error': 'continue', 'relax_path_checks': False, 'validate': False}.iteritems(): if not hasattr(args, k): setattr(args, k, v) if args.quiet: _logger.setLevel(logging.WARN) if args.debug: _logger.setLevel(logging.DEBUG) if args.version: print versionfunc() return 0 else: _logger.info(versionfunc()) if not args.workflow: if os.path.isfile("CWLFile"): setattr(args, "workflow", "CWLFile") else: _logger.error("") _logger.error("CWL document required, try --help for details") return 1 if args.relax_path_checks: draft2tool.ACCEPTLIST_RE = draft2tool.ACCEPTLIST_EN_RELAXED_RE try: document_loader, workflowobj, uri = fetch_document(args.workflow, resolver=resolver, fetcher_constructor=fetcher_constructor) if args.print_deps: printdeps(workflowobj, document_loader, stdout, args.relative_deps, uri) return 0 document_loader, avsc_names, processobj, metadata, uri \ = validate_document(document_loader, workflowobj, uri, enable_dev=args.enable_dev, strict=args.strict, preprocess_only=args.print_pre or args.pack, fetcher_constructor=fetcher_constructor) if args.validate: return 0 if args.pack: stdout.write(print_pack(document_loader, processobj, uri, metadata)) return 0 if args.print_pre: stdout.write(json.dumps(processobj, indent=4)) return 0 tool = make_tool(document_loader, avsc_names, metadata, uri, makeTool, vars(args)) if args.print_rdf: printrdf(tool, document_loader.ctx, args.rdf_serializer, stdout) return 0 if args.print_dot: printdot(tool, document_loader.ctx, stdout) return 0 except (validate.ValidationException) as exc: _logger.error(u"Tool definition failed validation:\n%s", exc, exc_info=args.debug) return 1 except (RuntimeError, WorkflowException) as exc: _logger.error(u"Tool definition failed initialization:\n%s", exc, exc_info=args.debug) return 1 except Exception as exc: _logger.error( u"I'm sorry, I couldn't load this CWL file%s", ", try again with --debug for more information.\nThe error was: " "%s" % exc if not args.debug else ". The error was:", exc_info=args.debug) return 1 if isinstance(tool, int): return tool for dirprefix in ("tmpdir_prefix", "tmp_outdir_prefix", "cachedir"): if getattr(args, dirprefix) and getattr(args, dirprefix) != 'tmp': sl = "/" if getattr(args, dirprefix).endswith("/") or dirprefix == "cachedir" else "" setattr(args, dirprefix, os.path.abspath(getattr(args, dirprefix))+sl) if not os.path.exists(os.path.dirname(getattr(args, dirprefix))): try: os.makedirs(os.path.dirname(getattr(args, dirprefix))) except Exception as e: _logger.error("Failed to create directory: %s", e) return 1 if args.cachedir: if args.move_outputs == "move": setattr(args, 'move_outputs', "copy") setattr(args, "tmp_outdir_prefix", args.cachedir) if job_order_object is None: job_order_object = load_job_order(args, tool, stdin, print_input_deps=args.print_input_deps, relative_deps=args.relative_deps, stdout=stdout, make_fs_access=make_fs_access, fetcher_constructor=fetcher_constructor) if isinstance(job_order_object, int): return job_order_object try: setattr(args, 'basedir', job_order_object[1]) del args.workflow del args.job_order (out, status) = executor(tool, job_order_object[0], makeTool=makeTool, select_resources=selectResources, make_fs_access=make_fs_access, **vars(args)) # This is the workflow output, it needs to be written if out is not None: def locToPath(p): if p["location"].startswith("file://"): p["path"] = uri_file_path(p["location"]) adjustDirObjs(out, locToPath) adjustFileObjs(out, locToPath) if isinstance(out, basestring): stdout.write(out) else: stdout.write(json.dumps(out, indent=4)) stdout.write("\n") stdout.flush() if status != "success": _logger.warn(u"Final process status is %s", status) return 1 else: _logger.info(u"Final process status is %s", status) return 0 except (validate.ValidationException) as exc: _logger.error(u"Input object failed validation:\n%s", exc, exc_info=args.debug) return 1 except UnsupportedRequirement as exc: _logger.error( u"Workflow or tool uses unsupported feature:\n%s", exc, exc_info=args.debug) return 33 except WorkflowException as exc: _logger.error( u"Workflow error, try again with --debug for more " "information:\n%s", strip_dup_lineno(unicode(exc)), exc_info=args.debug) return 1 except Exception as exc: _logger.error( u"Unhandled error, try again with --debug for more information:\n" " %s", exc, exc_info=args.debug) return 1 return 0 finally: _logger.removeHandler(stderr_handler) _logger.addHandler(defaultStreamHandler) if __name__ == "__main__": sys.exit(main(sys.argv[1:])) cwltool-1.0.20170114120503/cwltool/schemas/0000755000175000017500000000000013036412107020537 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/draft-2/0000755000175000017500000000000013036412107021776 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/draft-2/cwl-avro.yml0000644000175000017500000021224313036412052024256 0ustar michaelmichael00000000000000- name: "Common Workflow Language, Draft 2" type: doc doc: | 7 July 2015 This version: * https://w3id.org/cwl/draft-2/ Current version: * https://w3id.org/cwl/ Authors: * Peter Amstutz , Curoverse * Nebojša Tijanić , Seven Bridges Genomics Contributers: * Luka Stojanovic , Seven Bridges Genomics * John Chilton , Galaxy Project, Pennsylvania State University * Michael R. Crusoe , Michigan State University * Hervé Ménager , Institut Pasteur * Maxim Mikheev , BioDatomics * Stian Soiland-Reyes [soiland-reyes@cs.manchester.ac.uk](mailto:soiland-reyes@cs.manchester.ac.uk), University of Manchester # Abstract A Workflow is an analysis task represented by a directed graph describing a sequence of operations that transform an input data set to output. This specification defines the Common Workflow Language (CWL), a vendor-neutral standard for representing workflows and concrete process steps intended to be portable across a variety of computing platforms. # Status of This Document This document is the product of the [Common Workflow Language working group](https://groups.google.com/forum/#!forum/common-workflow-language). The latest version of this document is available in the "specification" directory at https://github.com/common-workflow-language/common-workflow-language The products of the CWL working group (including this document) are made available under the terms of the Apache License, version 2.0. # Introduction The Common Workflow Language (CWL) working group is an informal, multi-vendor working group consisting of various organizations and individuals that have an interest in portability of data analysis workflows. The goal is to create specifications like this one that enable data scientists to describe analysis tools and workflows that are powerful, easy to use, portable, and support reproducibility. ## Introduction to draft 2 This specification represents the second milestone of the CWL group. Since draft-1, this draft introduces a number of major changes and additions: * Use of Avro schema (instead of JSON-schema) and JSON-LD for data modeling. * Significant refactoring of the Command Line Tool description. * Data and execution model for Workflows. * Extension mechanism though "hints" and "requirements". ## Purpose CWL is designed to express workflows for data-intensive science, such as Bioinformatics, Chemistry, Physics, and Astronomy. This specification is intended to define a data and execution model for Workflows and Command Line Tools that can be implemented on top of a variety of computing platforms, ranging from an individual workstation to cluster, grid, cloud, and high performance computing systems. ## References to Other Specifications * [JSON](http://json.org) * [JSON-LD](http://json-ld.org) * [JSON Pointer](https://tools.ietf.org/html/draft-ietf-appsawg-json-pointer-04) * [YAML](http://yaml.org) * [Avro](https://avro.apache.org/docs/current/spec.html) * [Uniform Resource Identifier (URI): Generic Syntax](https://tools.ietf.org/html/rfc3986) * [UTF-8](https://www.ietf.org/rfc/rfc2279.txt) * [Portable Operating System Interface (POSIX.1-2008)](http://pubs.opengroup.org/onlinepubs/9699919799/) * [Resource Description Framework (RDF)](http://www.w3.org/RDF/) ## Scope This document describes the CWL syntax, execution, and object model. It is not intended to document a specific implementation of CWL, however it may serve as a reference for the behavior of conforming implementations. ## Terminology The terminology used to describe CWL documents is defined in the Concepts section of the specification. The terms defined in the following list are used in building those definitions and in describing the actions of an CWL implementation: **may**: Conforming CWL documents and CWL implementations are permitted but not required to behave as described. **must**: Conforming CWL documents and CWL implementations are required to behave as described; otherwise they are in error. **error**: A violation of the rules of this specification; results are undefined. Conforming implementations may detect and report an error and may recover from it. **fatal error**: A violation of the rules of this specification; results are undefined. Conforming implementations must not continue to execute the current process and may report an error. **at user option**: Conforming software may or must (depending on the modal verb in the sentence) behave as described; if it does, it must provide users a means to enable or disable the behavior described. # Data model ## Data concepts An **object** is a data structure equivalent to the "object" type in JSON, consisting of a unordered set of name/value pairs (referred to here as **fields**) and where the name is a string and the value is a string, number, boolean, array, or object. A **document** is a file containing a serialized object, or an array of objects. A **process** is a basic unit of computation which accepts input data, performs some computation, and produces output data. An **input object** is an object describing the inputs to a invocation of process. An **output object** is an object describing the output of an invocation of a process. An **input schema** describes the valid format (required fields, data types) for an input object. An **output schema** describes the valid format for a output object. **Metadata** is information about workflows, tools, or input items that is not used directly in the computation. ## Syntax Documents containing CWL objects are serialized and loaded using YAML syntax and UTF-8 text encoding. A conforming implementation must accept all valid YAML documents. The CWL schema is defined using Avro Linked Data (avro-ld). Avro-ld is an extension of the Apache Avro schema language to support additional annotations mapping Avro fields to RDF predicates via JSON-LD. A CWL document may be validated by transforming the avro-ld schema to a base Apache Avro schema. An implementation may interpret a CWL document as [JSON-LD](http://json-ld.org) and convert a CWL document to a [Resource Description Framework (RDF)](http://www.w3.org/RDF/) using the CWL [JSON-LD Context](https://w3id.org/cwl/draft-2/context) (extracted from the avro-ld schema). The CWL [RDFS schema](https://w3id.org/cwl/draft-2/cwl.ttl) defines the classes and properties used by CWL as JSON-LD. The latest draft-2 schema is defined here: https://github.com/common-workflow-language/common-workflow-language/blob/master/schemas/draft-2/cwl-avro.yml ## Identifiers If an object contains an `id` field, that is used to uniquely identify the object in that document. The value of the `id` field must be unique over the entire document. The format of the `id` field is that of a [relative fragment identifier](https://tools.ietf.org/html/rfc3986#section-3.5), and must start with a hash `#` character. An implementation may choose to only honor references to object types for which the `id` field is explicitly listed in this specification. When loading a CWL document, an implementation may resolve relative identifiers to absolute URI references. For example, "my_tool.cwl" located in the directory "/home/example/work/" may be transformed to "file:///home/example/work/my_tool.cwl" and a relative fragment reference "#input" in this file may be transformed to "file:///home/example/work/my_tool.cwl#input". ## Document preprocessing An implementation must resolve `import` directives. An `import` directive is an object consisting of the field `import` specifying a URI. The URI referenced by `import` must be loaded as a CWL document (including recursive preprocessing) and then the `import` object is implicitly replaced by the external resource. URIs may include document fragments referring to objects identified by their `id` field, in which case the `import` directive is replaced by only the fragment object. An implementation must resolve `include` directives. An `include` directive is an object consisting of the field `include` specifying a URI. The URI referenced by `include` must be loaded as UTF-8 encoded text document and the `include` directive implicitly replaced by a string with the contents of the document. Because the loaded resource is unparsed, URIs used with `include` must not include fragments. ## Extensions and Metadata Implementation extensions not required for correct execution (for example, fields related to GUI rendering) may be stored in [process hints](#requirements_and_hints). Input metadata (for example, a lab sample identifier) may be explicitly represented within a workflow using input parameters which are propagated to output. Future versions of this specification may define additional facilities for working with input/output metadata. Fields for tool and workflow metadata (for example, authorship for use in citations) are not defined in this specification. Future versions of this specification may define such fields. # Execution model ## Execution concepts A **parameter** is a named symbolic input or output of process, with an associated datatype or schema. During execution, values are assigned to parameters to make the input object or output object used for concrete process invocation. A **command line tool** is a process characterized by the execution of a standalone, non-interactive program which is invoked on some input, produces output, and then terminates. A **workflow** is a process characterized by multiple subprocess steps, where step outputs are connected to the inputs of other downstream steps to form a directed graph, and independent steps may run concurrently. A **runtime environment** is the actual hardware and software environment when executing a command line tool. It includes, but is not limited to, the hardware architecture, hardware resources, operating system, software runtime (if applicable, such as the Python interpreter or the JVM), libraries, modules, packages, utilities, and data files required to run the tool. A **workflow platform** is a specific hardware and software implementation capable of interpreting a CWL document and executing the processes specified by the document. The responsibilities of the workflow platform may include scheduling process invocation, setting up the necessary runtime environment, making input data available, invoking the tool process, and collecting output. It is intended that the workflow platform has broad leeway outside of this specification to optimize use of computing resources and enforce policies not covered by this specifcation. Some areas that are currently out of scope for CWL specification but may be handled by a specific workflow platform include: * Data security and permissions. * Scheduling tool invocations on remote cluster or cloud compute nodes. * Using virtual machines or operating system containers to manage the runtime (except as described in [DockerRequirement](#dockerrequirement)). * Using remote or distributed file systems to manage input and output files. * Translating or rewriting file paths. * Determining if a process has previously been executed, skipping it and reusing previous results. * Pausing and resuming of processes or workflows. Conforming CWL processes must not assume anything about the runtime environment or workflow platform unless explicitly declared though the use of [process requirements](#processrequirement). ## Generic execution process The generic execution sequence of a CWL process (including both workflows and concrete process implementations) is as follows. 1. Load and validate CWL document, yielding a process object. 2. Load input object. 3. Validate the input object against the `inputs` schema for the process. 4. Validate that process requirements are met. 5. Perform any further setup required by the specific process type. 6. Execute the process. 7. Capture results of process execution into the output object. 8. Validate the output object against the `outputs` schema for the process. 9. Report the output object to the process caller. ## Requirements and hints A **[process requirement](#processrequirement)** modifies the semantics or runtime environment of a process. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. A **hint** is similar to a requirement, however it is not an error if an implementation cannot satisfy all hints. The implementation may report a warning if a hint cannot be satisfied. Requirements are inherited. A requirement specified in a Workflow applies to all workflow steps; a requirement specified on a workflow step will apply to the process implementation. If the same process requirement appears at different levels of the workflow, the most specific instance of the requirement is used, that is, an entry in `requirements` on a process implementation such as CommandLineTool will take precendence over an entry in `requirements` specified in a workflow step, and an entry in `requirements` on a workflow step takes precedence over the workflow. Entries in `hints` are resolved the same way. Requirements override hints. If a process implementation provides a process requirement in `hints` which is also provided in `requirements` by an enclosing workflow or workflow step, the enclosing `requirements` takes precedence. Process requirements are the primary mechanism for specifying extensions to the CWL core specification. ## Expressions An expression is a fragment of executable code which is evaluated by the workflow platform to affect the inputs, outputs, or behavior of a process. In the generic execution sequence, expressions may be evaluated during step 5 (process setup), step 6 (execute process), and/or step 7 (capture output). Expressions are distinct from regular processes in that they are intended to modify the behavior of the workflow itself rather than perform the primary work of the workflow. An implementation must provide the predefined `cwl:JsonPointer` expression engine. This expression engine specifies a [JSON Pointer](https://tools.ietf.org/html/draft-ietf-appsawg-json-pointer-04) into an expression input object consisting of the `job` and `context` fields described below. An expression engine defined with [ExpressionEngineRequirement](#expressionenginerequirement) is a command line program following the following protocol: * On standard input, receive a JSON object with the following fields: - **engineConfig**: A list of strings from the `engineConfig` field. Null if `engineConfig` is not specified. - **job**: The input object of the current Process (context dependent). - **context**: The specific value being transformed (context dependent). May be null. - **script**: The code fragment to evaluate. - **outdir**: When used in the context of a CommandLineTool, this is the designated output directory that will be used when executing the tool. Null if not applicable. - **tmpdir**: When used in the context of a CommandLineTool, this is the designated temporary directory that will be used when executing the tool. Null if not applicable. * On standard output, print a single JSON value (string, number, array, object, boolean, or null) for the return value. Expressions must be evaluated in an isolated context (a "sandbox") which permits no side effects to leak outside the context, and permit no outside data to leak into the context. Implementations may apply limits, such as process isolation, timeouts, and operating system containers/jails to minimize the security risks associated with running untrusted code. The order in which expressions are evaluated within a process or workflow is undefined. ## Workflow graph A workflow describes a set of **steps** and the **dependencies** between those processes. When a process produces output that will be consumed by a second process, the first process is a dependency of the second process. When there is a dependency, the workflow engine must execute the dependency process and wait for it to successfully produce output before executing the dependent process. If two processes are defined in the workflow graph that are not directly or indirectly dependent, these processes are **independent**, and may execute in any order or execute concurrently. A workflow is complete when all steps have been executed. ## Success and failure A completed process must result in one of `success`, `temporaryFailure` or `permanentFailure` states. An implementation may choose to retry a process execution which resulted in `temporaryFailure`. An implementation may choose to either continue running other steps of a workflow, or terminate immediately upon `permanentFailure`. * If any step of a workflow execution results in `permanentFailure`, then the workflow status is `permanentFailure`. * If one or more steps result in `temporaryFailure` and all other steps complete `success` or are not executed, then the workflow status is `temporaryFailure`. * If all workflow steps are executed and complete with `success`, then the workflow status is `success`. ## Executing CWL documents as scripts By convention, a CWL document may begin with `#!/usr/bin/env cwl-runner` and be marked as executable (the POSIX "+x" permission bits) to enable it to be executed directly. A workflow platform may support this mode of operation; if so, it must provide `cwl-runner` as an alias for the platform's CWL implementation. # Sample CWL workflow revtool.cwl: ``` #!/usr/bin/env cwl-runner # # Simplest example command line program wrapper for the Unix tool "rev". # class: CommandLineTool description: "Reverse each line using the `rev` command" # The "inputs" array defines the structure of the input object that describes # the inputs to the underlying program. Here, there is one input field # defined that will be called "input" and will contain a "File" object. # # The input binding indicates that the input value should be turned into a # command line argument. In this example inputBinding is an empty object, # which indicates that the file name should be added to the command line at # a default location. inputs: - id: "#input" type: File inputBinding: {} # The "outputs" array defines the structure of the output object that # describes the outputs of the underlying program. Here, there is one # output field defined that will be called "output", must be a "File" type, # and after the program executes, the output value will be the file # output.txt in the designated output directory. outputs: - id: "#output" type: File outputBinding: glob: output.txt # The actual program to execute. baseCommand: rev # Specify that the standard output stream must be redirected to a file called # output.txt in the designated output directory. stdout: output.txt ``` sorttool.cwl: ``` #!/usr/bin/env cwl-runner # # Example command line program wrapper for the Unix tool "sort" # demonstrating command line flags. class: CommandLineTool description: "Sort lines using the `sort` command" # This example is similar to the previous one, with an additional input # parameter called "reverse". It is a boolean parameter, which is # intepreted as a command line flag. The value of "prefix" is used for # flag to put on the command line if "reverse" is true. If "reverse" is # false, no flag is added. # # This example also introduced the "position" field. This indicates the # sorting order of items on the command line. Lower numbers are placed # before higher numbers. Here, the "--reverse" flag (if present) will be # added to the command line before the input file path. inputs: - id: "#reverse" type: boolean inputBinding: position: 1 prefix: "--reverse" - id: "#input" type: File inputBinding: position: 2 outputs: - id: "#output" type: File outputBinding: glob: output.txt baseCommand: sort stdout: output.txt ``` revsort.cwl: ``` #!/usr/bin/env cwl-runner # # This is a two-step workflow which uses "revtool" and "sorttool" defined above. # class: Workflow description: "Reverse the lines in a document, then sort those lines." # Requirements specify prerequisites and extensions to the workflow. # In this example, DockerRequirement specifies a default Docker container # in which the command line tools will execute. requirements: - class: DockerRequirement dockerPull: debian:8 # The inputs array defines the structure of the input object that describes # the inputs to the workflow. # # The "reverse_sort" input parameter demonstrates the "default" field. If the # field "reverse_sort" is not provided in the input object, the default value will # be used. inputs: - id: "#input" type: File description: "The input file to be processed." - id: "#reverse_sort" type: boolean default: true description: "If true, reverse (descending) sort" # The "outputs" array defines the structure of the output object that describes # the outputs of the workflow. # # Each output field must be connected to the output of one of the workflow # steps using the "connect" field. Here, the parameter "#output" of the # workflow comes from the "#sorted" output of the "sort" step. outputs: - id: "#output" type: File source: "#sorted.output" description: "The output with the lines reversed and sorted." # The "steps" array lists the executable steps that make up the workflow. # The tool to execute each step is listed in the "run" field. # # In the first step, the "inputs" field of the step connects the upstream # parameter "#input" of the workflow to the input parameter of the tool # "revtool.cwl#input". # # In the second step, the "inputs" field of the step connects the output # parameter "#reversed" from the first step to the input parameter of the # tool "sorttool.cwl#input". steps: - inputs: - { id: "#rev.input", source: "#input" } outputs: - { id: "#rev.output" } run: { import: revtool.cwl } - inputs: - { id: "#sorted.input", source: "#rev.output" } - { id: "#sorted.reverse", source: "#reverse_sort" } outputs: - { id: "#sorted.output" } run: { import: sorttool.cwl } ``` Sample input object: ``` { "input": { "class": "File", "path": "whale.txt" } } ``` Sample output object: ``` { "output": { "path": "/tmp/tmpdeI_p_/output.txt", "size": 1111, "class": "File", "checksum": "sha1$b9214658cc453331b62c2282b772a5c063dbd284" } } ``` jsonldPrefixes: { "cwl": "https://w3id.org/cwl/cwl#", "avro": "https://w3id.org/cwl/avro#", "dct": "http://purl.org/dc/terms/", "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdfs": "http://www.w3.org/2000/01/rdf-schema#" } jsonldVocab: cwl - name: Reference type: doc doc: This section specifies the core object types that make up a CWL document. - name: Datatype type: enum docAfter: ProcessRequirement symbols: - "null" - boolean - int - long - float - double - bytes - string - record - enum - array - map - File - Any jsonldPrefix: avro jsonldPredicate: - symbol: File predicate: "cwl:File" - symbol: Any predicate: "cwl:Any" doc: | CWL data types are based on Avro schema declarations. Refer to the [Avro schema declaration documentation](https://avro.apache.org/docs/current/spec.html#schemas) for detailed information. In addition, CWL defines [`File`](#file) as a special record type. ## Primitive types * **null**: no value * **boolean**: a binary value * **int**: 32-bit signed integer * **long**: 64-bit signed integer * **float**: single precision (32-bit) IEEE 754 floating-point number * **double**: double precision (64-bit) IEEE 754 floating-point number * **bytes**: sequence of uninterpreted 8-bit unsigned bytes * **string**: Unicode character sequence ## Complex types * **record**: An object with one or more fields defined by name and type * **enum**: A value from a finite set of symbolic values * **array**: An ordered sequence of values * **map**: An unordered collection of key/value pairs ## File type See [File](#file) below. ## Any type See [Any](#any) below. - name: File type: record docParent: Datatype doc: | Represents a file (or group of files if `secondaryFiles` is specified) that must be accessible by tools using standard POSIX file system call API such as open(2) and read(2). fields: - name: "class" type: type: enum name: "File_class" symbols: - File jsonldPredicate: "@id": "@type" "@type": "@vocab" doc: Must be `File` to indicate this object describes a file. - name: "path" type: "string" doc: The path to the file. - name: "checksum" type: ["null", "string"] doc: | Optional hash code for validating file integrity. Currently must be in the form "sha1$ + hexidecimal string" using the SHA-1 algorithm. - name: "size" type: ["null", "long"] doc: Optional file size. - name: "secondaryFiles" type: - "null" - type: array items: File doc: | A list of additional files that are associated with the primary file and must be transferred alongside the primary file. Examples include indexes of the primary file, or external references which must be included when loading primary document. A file object listed in `secondaryFiles` may itself include `secondaryFiles` for which the same rules apply. - name: Any type: enum docParent: Datatype symbols: ["Any"] doc: | The **Any** type validates for any non-null value. - name: Schema type: record doc: "A schema defines a parameter type." docParent: Parameter fields: - name: type doc: "The data type of this parameter." type: - "Datatype" - "Schema" - "string" - type: "array" items: [ "Datatype", "Schema", "string" ] jsonldPredicate: "@id": "avro:type" "@type": "@vocab" - name: fields type: - "null" - type: "array" items: "Schema" jsonldPredicate: "@id": "avro:fields" "@container": "@list" doc: "When `type` is `record`, defines the fields of the record." - name: "symbols" type: - "null" - type: "array" items: "string" jsonldPredicate: "@id": "avro:symbols" "@container": "@list" doc: "When `type` is `enum`, defines the set of valid symbols." - name: items type: - "null" - "Datatype" - "Schema" - "string" - type: "array" items: [ "Datatype", "Schema", "string" ] jsonldPredicate: "@id": "avro:items" "@container": "@list" doc: "When `type` is `array`, defines the type of the array elements." - name: "values" type: - "null" - "Datatype" - "Schema" - "string" - type: "array" items: [ "Datatype", "Schema", "string" ] jsonldPredicate: "@id": "avro:values" "@container": "@list" doc: "When `type` is `map`, defines the value type for the key/value pairs." - name: Parameter type: record docParent: Process abstract: true doc: | Define an input or output parameter to a process. fields: - name: type type: - "null" - Datatype - Schema - string - type: array items: - Datatype - Schema - string jsonldPredicate: "@id": "avro:type" "@type": "@vocab" doc: | Specify valid types of data that may be assigned to this parameter. - name: label type: - "null" - string jsondldPredicate: "rdfs:label" doc: "A short, human-readable label of this parameter object." - name: description type: - "null" - string jsondldPredicate: "rdfs:comment" doc: "A long, human-readable description of this parameter object." - name: streamable type: ["null", "boolean"] doc: | Currently only applies if `type` is `File`. A value of `true` indicates that the file is read or written sequentially without seeking. An implementation may use this flag to indicate whether it is valid to stream file contents using a named pipe. Default: `false`. - name: default type: ["null", Any] doc: | The default value for this parameter if not provided in the input object. - name: JsonPointer type: enum docParent: Expression symbols: - "JsonPointer" jsonldPrefix: "cwl" - type: record name: Expression docAfter: ExpressionTool doc: | Define an expression that will be evaluated and used to modify the behavior of a tool or workflow. See [Expressions](#expressions) for more information about expressions and [ExpressionEngineRequirement](#expressionenginerequirement) for information on how to define a expression engine. fields: - name: engine type: - JsonPointer - string doc: | Either `cwl:JsonPointer` or a reference to an ExpressionEngineRequirement defining which engine to use. jsonldPredicate: "@id": "cwl:engine" "@type": "@id" - name: script type: string doc: "The code to be executed by the expression engine." - name: Binding type: record docParent: Parameter fields: - name: loadContents type: - "null" - boolean doc: | Only applies when `type` is `File`. Read up to the first 64 KiB of text from the file and place it in the "contents" field of the file object for manipulation by expressions. - name: secondaryFiles type: - "null" - "string" - Expression - type: "array" items: ["string", "Expression"] doc: | Only applies when `type` is `File`. Describes files that must be included alongside the primary file. If the value is an expression, the context of the expression is the input or output File parameter to which this binding applies. If the value is a string, it specifies that the following pattern should be applied to the primary file: 1. If string begins with one or more caret `^` characters, for each caret, remove the last file extension from the path (the last period `.` and all following characters). If there are no file extensions, the path is unchanged. 2. Append the remainder of the string to the end of the file path. - name: InputSchema type: record extends: Schema docParent: InputParameter specialize: {Schema: InputSchema} fields: - name: inputBinding type: [ "null", "Binding" ] doc: | Describes how to handle a value in the input object convert it into a concrete form for execution, such as command line parameters. - name: OutputSchema type: record extends: Schema docParent: OutputParameter specialize: {Schema: OutputSchema} - name: InputParameter type: record extends: Parameter docAfter: Parameter specialize: {Schema: InputSchema} fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this parameter object." - name: "inputBinding" type: [ "null", "Binding" ] doc: | Describes how to handle the inputs of a process and convert them into a concrete form for execution, such as command line parameters. - name: OutputParameter type: record extends: Parameter docAfter: Parameter specialize: {Schema: OutputSchema} fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this parameter object." - type: record name: "FileDef" docParent: CreateFileRequirement doc: | Define a file that must be placed in the designated output directory prior to executing the command line tool. May be the result of executing an expression, such as building a configuration file from a template. fields: - name: "filename" type: ["string", "Expression"] doc: "The name of the file to create in the output directory." - name: "fileContent" type: ["string", "Expression"] doc: | If the value is a string literal or an expression which evaluates to a string, a new file must be created with the string as the file contents. If the value is an expression that evaluates to a File object, this indicates the referenced file should be added to the designated output directory prior to executing the tool. Files added in this way may be read-only, and may be provided by bind mounts or file system links to avoid unnecessary copying of the input file. - type: record name: EnvironmentDef docParent: EnvVarRequirement doc: | Define an environment variable that will be set in the runtime environment by the workflow platform when executing the command line tool. May be the result of executing an expression, such as getting a parameter from input. fields: - name: "envName" type: "string" doc: The environment variable name - name: "envValue" type: ["string", "Expression"] doc: The environment variable value - type: record name: SchemaDef extends: Schema docParent: SchemaDefRequirement fields: - name: name type: string doc: "The type name being defined." - type: record name: ProcessRequirement docAfter: ExpressionTool abstract: true doc: | A process requirement declares a prerequisite that may or must be fulfilled before executing a process. See [`Process.hints`](#process) and [`Process.requirements`](#process). Process requirements are the primary mechanism for specifying extensions to the CWL core specification. fields: - name: "class" type: "string" doc: "The specific requirement type." jsonldPredicate: "@id": "@type" "@type": "@vocab" - type: record name: Process abstract: true docAfter: ProcessRequirement doc: | The base executable type in CWL is the `Process` object defined by the document. Note that the `Process` object is abstract and cannot be directly executed. fields: - name: id type: ["null", string] jsonldPredicate: "@id" doc: "The unique identifier for this process object." - name: inputs type: type: array items: InputParameter doc: | Defines the input parameters of the process. The process is ready to run when all required input parameters are associated with concrete values. Input parameters include a schema for each parameter which is used to validate the input object. It may also be used build a user interface for constructing the input object. - name: outputs type: type: array items: OutputParameter doc: | Defines the parameters representing the output of the process. May be used to generate and/or validate the output object. - name: requirements type: - "null" - type: array items: ProcessRequirement doc: > Declares requirements that apply to either the runtime environment or the workflow engine that must be met in order to execute this process. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. - name: hints type: - "null" - type: array items: Any doc: > Declares hints applying to either the runtime environment or the workflow engine that may be helpful in executing this process. It is not an error if an implementation cannot satisfy all hints, however the implementation may report a warning. - name: label type: - "null" - string jsondldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: description type: - "null" - string jsondldPredicate: "rdfs:comment" doc: "A long, human-readable description of this process object." - type: record name: CommandLineBinding extends: Binding docParent: CommandInputParameter doc: | When listed under `inputBinding` in the input schema, the term "value" refers to the the corresponding value in the input object. For binding objects listed in `CommandLineTool.arguments`, the term "value" refers to the effective value after evaluating `valueFrom`. The binding behavior when building the command line depends on the data type of the value. If there is a mismatch between the type described by the input schema and the effective value, such as resulting from an expression evaluation, an implementation must use the data type of the effective value. - **string**: Add `prefix` and the string to the command line. - **number**: Add `prefix` and decimal representation to command line. - **boolean**: If true, add `prefix` to the command line. If false, add nothing. - **File**: Add `prefix` and the value of [`File.path`](#file) to the command line. - **array**: If `itemSeparator` is specified, add `prefix` and the join the array into a single string with `itemSeparator` separating the items. Otherwise first add `prefix`, then recursively process individual elements. - **object**: Add `prefix` only, and recursively add object fields for which `inputBinding` is specified. - **null**: Add nothing. fields: - name: "position" type: ["null", "int"] doc: "The sorting key. Default position is 0." - name: "prefix" type: [ "null", "string"] doc: "Command line prefix to add before the value." - name: "separate" type: ["null", boolean] doc: | If true (default), then the prefix and value must be added as separate command line arguments; if false, prefix and value must be concatenated into a single command line argument. - name: "itemSeparator" type: ["null", "string"] doc: | Join the array elements into a single string with the elements separated by by `itemSeparator`. - name: "valueFrom" type: - "null" - "string" - "Expression" doc: | If `valueFrom` is a constant string value, use this as the value and apply the binding rules above. If `valueFrom` is an expression, evaluate the expression to yield the actual value to use to build the command line and apply the binding rules above. If the inputBinding is associated with an input parameter, the "context" of the expression will be the value of the input parameter. When a binding is part of the `CommandLineTool.arguments` field, the `valueFrom` field is required. - type: record name: CommandOutputBinding extends: Binding docParent: CommandOutputParameter doc: | Describes how to generate an output parameter based on the files produced by a CommandLineTool. The output parameter is generated by applying these operations in the following order: - glob - loadContents - outputEval fields: - name: glob type: - "null" - string - Expression - type: array items: string doc: | Find files relative to the output directory, using POSIX glob(3) pathname matching. If provided an array, find files that match any pattern in the array. If provided an expression, the expression must return a string or an array of strings, which will then be evaluated as one or more glob patterns. Only files which actually exist will be matched and returned. - name: outputEval type: - "null" - Expression doc: | Evaluate an expression to generate the output value. If `glob` was specified, the script `context` will be an array containing any files that were matched. Additionally, if `loadContents` is `true`, the File objects will include up to the first 64 KiB of file contents in the `contents` field. - type: record name: CommandInputSchema extends: InputSchema docParent: CommandInputParameter specialize: InputSchema: CommandInputSchema Binding: CommandLineBinding - type: record name: CommandOutputSchema extends: OutputSchema docParent: CommandOutputParameter specialize: OutputSchema: CommandOutputSchema fields: - name: "outputBinding" type: [ "null", CommandOutputBinding ] doc: | Describes how to handle the concrete outputs of a process step (such as files created by a program) and describe them in the process output parameter. - type: record name: CommandInputParameter extends: InputParameter docParent: CommandLineTool doc: An input parameter for a CommandLineTool. specialize: InputSchema: CommandInputSchema Binding: CommandLineBinding - type: record name: CommandOutputParameter extends: OutputParameter docParent: CommandLineTool doc: An output parameter for a CommandLineTool. specialize: OutputSchema: CommandOutputSchema fields: - name: "outputBinding" type: [ "null", CommandOutputBinding ] doc: | Describes how to handle the concrete outputs of a process step (such as files created by a program) and describe them in the process output parameter. - type: record name: CommandLineTool extends: Process docAfter: Workflow specialize: InputParameter: CommandInputParameter OutputParameter: CommandOutputParameter doc: | A CommandLineTool process is a process implementation for executing a non-interactive application in a POSIX environment. To accommodate the enormous variety in syntax and semantics for input, runtime environment, invocation, and output of arbitrary programs, CommandLineTool uses an "input binding" that describes how to translate input parameters to an actual program invocation, and an "output binding" that describes how to generate output parameters from program output. # Input binding The tool command line is built by applying command line bindings to the input object. Bindings are listed either as part of an [input parameter](#commandinputparameter) using the `inputBinding` field, or separately using the `arguments` field of the CommandLineTool. The algorithm to build the command line is as follows. In this algorithm, the sort key is a list consisting of one or more numeric or string elements. Strings are sorted lexicographically based on UTF-8 encoding. 1. Collect `CommandLineBinding` objects from `arguments`. Assign a sorting key `[position, i]` where `position` is [`CommandLineBinding.position`](#commandlinebinding) and `i` is the index in the `arguments` list. 2. Collect `CommandLineBinding` objects from the `inputs` schema and associate them with values from the input object. Where the input type is a record, array, or map, recursively walk the schema and input object, collecting nested `CommandLineBinding` objects and associating them with values from the input object. 3. Create a sorting key by taking the value of the `position` field at each level leading to each leaf binding object. If `position` is not specified, it is not added to the sorting key. For bindings on arrays and maps, the sorting key must include the array index or map key following the position. If and only if two bindings have the same sort key, the tie must be broken using the ordering of the field or parameter name immediately containing the leaf binding. 4. Sort elements using the assigned sorting keys. Numeric entries sort before strings. 5. In the sorted order, apply the rules defined in [`CommandLineBinding`](#commandlinebinding) to convert bindings to actual command line elements. 6. Insert elements from `baseCommand` at the beginning of the command line. # Runtime environment All files listed in the input object must be made available in the runtime environment. The implementation may use a shared or distributed file system or transfer files via explicit download. Implementations may choose not to provide access to files not explicitly specified by the input object or process requirements. Output files produced by tool execution must be written to the **designated output directory**. The initial current working directory when executing the tool must be the designated output directory. When executing the tool, the child process must not inherit environment variables from the parent process. The tool must execute in a new, empty environment, containing only environment variables defined by [EnvVarRequirement](#envvarrequirement), the default environment of the Docker container specified in [DockerRequirement](#dockerrequirement) (if applicable), and `TMPDIR`. The `TMPDIR` environment variable must be set in the runtime environment to the **designated temporary directory**. Any files written to the designated temporary directory may be deleted by the workflow platform when the tool invocation is complete. An implementation may forbid the tool from writing to any location in the runtime environment file system other than the designated temporary directory and designated output directory. An implementation may provide read-only input files, and disallow in-place update of input files. The standard input stream and standard output stream may be redirected as described in the `stdin` and `stdout` fields. ## Extensions [DockerRequirement](#dockerrequirement), [CreateFileRequirement](#createfilerequirement), and [EnvVarRequirement](#envvarrequirement) are available as standard extensions to core command line tool semantics for defining the runtime environment. # Execution Once the command line is built and the runtime environment is created, the actual tool is executed. The standard error stream and standard output stream (unless redirected by setting `stdout`) may be captured by platform logging facilities for storage and reporting. Tools may be multithreaded or spawn child processes; however, when the parent process exits, the tool is considered finished regardless of whether any detached child processes are still running. Tools must not require any kind of console, GUI, or web based user interaction in order to start and run to completion. The exit code of the process indicates if the process completed successfully. By convention, an exit code of zero is treated as success and non-zero exit codes are treated as failure. This may be customized by providing the fields `successCodes`, `temporaryFailCodes`, and `permanentFailCodes`. An implementation may choose to default unspecified non-zero exit codes to either `temporaryFailure` or `permanentFailure`. # Output binding If the output directory contains a file named "cwl.output.json", that file must be loaded and used as the output object. Otherwise, the output object must be generated by walking the parameters listed in `outputs` and applying output bindings to the tool output. Output bindings are associated with output parameters using the `outputBinding` field. See [`CommandOutputBinding`](#commandoutputbinding) for details. fields: - name: "class" jsonldPredicate: "@id": "@type" "@type": "@vocab" type: string - name: baseCommand doc: | Specifies the program to execute. If the value is an array, the first element is the program to execute, and subsequent elements are placed at the beginning of the command line in prior to any command line bindings. If the program includes a path separator character it must be an absolute path, otherwise it is an error. If the program does not include a path separator, search the `$PATH` variable in the runtime environment of the workflow runner find the absolute path of the executable. type: - string - type: array items: string jsonldPredicate: "@id": "cwl:baseCommand" "@container": "@list" - name: arguments doc: | Command line bindings which are not directly associated with input parameters. type: - "null" - type: array items: [string, CommandLineBinding] jsonldPredicate: "@id": "cwl:arguments" "@container": "@list" - name: stdin type: ["null", string, Expression] doc: | A path to a file whose contents must be piped into the command's standard input stream. - name: stdout type: ["null", string, Expression] doc: | Capture the command's standard output stream to a file written to the designated output directory. If `stdout` is a string, it specifies the file name to use. If `stdout` is an expression, the expression is evaluated and must return a string with the file name to use to capture stdout. If the return value is not a string, or the resulting path contains illegal characters (such as the path separator `/`) it is an error. - name: successCodes type: - "null" - type: array items: int doc: | Exit codes that indicate the process completed successfully. - name: temporaryFailCodes type: - "null" - type: array items: int doc: | Exit codes that indicate the process failed due to a possibly temporary condition, where excuting the process with the same runtime environment and inputs may produce different results. - name: permanentFailCodes type: - "null" - type: array items: int doc: Exit codes that indicate the process failed due to a permanent logic error, where excuting the process with the same runtime environment and same inputs is expected to always fail. - type: record name: ExpressionTool extends: Process docAfter: CommandLineTool doc: | Execute an expression as a process step. fields: - name: "class" jsonldPredicate: "@id": "@type" "@type": "@vocab" type: string - name: expression type: Expression doc: | The expression to execute. The expression must return a JSON object which matches the output parameters of the ExpressionTool. - name: LinkMergeMethod type: enum docParent: WorkflowStepInput doc: The input link merge method, described in [WorkflowStepInput](#workflowstepinput). symbols: - merge_nested - merge_flattened - name: WorkflowOutputParameter type: record extends: OutputParameter docParent: Workflow doc: | Describe an output parameter of a workflow. The parameter must be connected to one or more parameters defined in the workflow that will provide the value of the output parameter. fields: - name: source doc: | Specifies one or more workflow parameters that will provide this output value. jsonldPredicate: "@id": "cwl:source" "@type": "@id" type: - "null" - string - type: array items: string - name: linkMerge type: ["null", LinkMergeMethod] doc: | The method to use to merge multiple inbound links into a single array. If not specified, the default method is "merge_nested". - type: record name: WorkflowStepInput docParent: WorkflowStep doc: | The input of a workflow step connects an upstream parameter (from the workflow inputs, or the outputs of other workflows steps) with the input parameters of the underlying process. ## Input object A WorkflowStepInput object must contain an `id` field in the form `#fieldname` or `#stepname.fieldname`. When the `id` field contains a period `.` the field name consists of the characters following the final period. This defines a field of the workflow step input object with the value of the `source` parameter(s). ## Merging If the sink parameter is an array, or named in a [workflow scatter](#workflowstep) operation, there may be multiple inbound data links listed in the `connect` field. The values from the input links are merged depending on the method specified in the `linkMerge` field. If not specified, the default method is "merge_nested". * **merge_nested** The input must be an array consisting of exactly one entry for each input link. If "merge_nested" is specified with a single link, the value from the link must be wrapped in a single-item list. * **merge_flattened** 1. The source and sink parameters must be compatible types, or the source type must be compatible with single element from the "items" type of the destination array parameter. 2. Source parameters which are arrays are concatenated. Source parameters which are single element types are appended as single elements. fields: - name: id type: string jsonldPredicate: "@id" doc: "A unique identifier for this workflow input parameter." - name: source doc: | Specifies one or more workflow parameters that will provide input to the underlying process parameter. jsonldPredicate: "@id": "cwl:source" "@type": "@id" type: - "null" - string - type: array items: string - name: linkMerge type: ["null", LinkMergeMethod] doc: | The method to use to merge multiple inbound links into a single array. If not specified, the default method is "merge_nested". - name: default type: ["null", Any] doc: | The default value for this parameter if there is no `source` field. - type: record name: WorkflowStepOutput docParent: WorkflowStep doc: | Associate an output parameter of the underlying process with a workflow parameter. The workflow parameter (given in the `id` field) be may be used as a `source` to connect with input parameters of other workflow steps, or with an output parameter of the process. fields: - name: id type: string jsonldPredicate: "@id" doc: | A unique identifier for this workflow output parameter. This is the identifier to use in the `source` field of `WorkflowStepInput` to connect the output value to downstream parameters. - name: ScatterMethod type: enum docParent: WorkflowStep doc: The scatter method, as described in [workflow step scatter](#workflowstep). symbols: - dotproduct - nested_crossproduct - flat_crossproduct - name: WorkflowStep type: record docParent: Workflow doc: | A workflow step is an executable element of a workflow. It specifies the underlying process implementation (such as `CommandLineTool`) in the `run` field and connects the input and output parameters of the underlying process to workflow parameters. # Scatter/gather To use scatter/gather, [ScatterFeatureRequirement](#scatterfeaturerequirement) must be specified in the workflow or workflow step requirements. A "scatter" operation specifies that the associated workflow step or subworkflow should execute separately over a list of input elements. Each job making up a scatter operaution is independent and may be executed concurrently. The `scatter` field specifies one or more input parameters which will be scattered. An input parameter may be listed more than once. The declared type of each input parameter is implicitly wrapped in an array for each time it appears in the `scatter` field. As a result, upstream parameters which are connected to scattered parameters may be arrays. All output parameter types are also implicitly wrapped in arrays. Each job in the scatter results in an entry in the output array. If `scatter` declares more than one input parameter, `scatterMethod` describes how to decompose the input into a discrete set of jobs. * **dotproduct** specifies that each of the input arrays are aligned and one element taken from each array to construct each job. It is an error if all input arrays are not the same length. * **nested_crossproduct** specifies the Cartesian product of the inputs, producing a job for every combination of the scattered inputs. The output must be nested arrays for each level of scattering, in the order that the input arrays are listed in the `scatter` field. * **flat_crossproduct** specifies the Cartesian product of the inputs, producing a job for every combination of the scattered inputs. The output arrays must be flattened to a single level, but otherwise listed in the order that the input arrays are listed in the `scatter` field. # Subworkflows To specify a nested workflow as part of a workflow step, [SubworkflowFeatureRequirement](#subworkflowfeaturerequirement) must be specified in the workflow or workflow step requirements. fields: - name: id type: ["null", string] jsonldPredicate: "@id" doc: "The unique identifier for this workflow step." - name: inputs type: type: array items: WorkflowStepInput doc: | Defines the input parameters of the workflow step. The process is ready to run when all required input parameters are associated with concrete values. Input parameters include a schema for each parameter which is used to validate the input object. It may also be used build a user interface for constructing the input object. - name: outputs type: type: array items: WorkflowStepOutput doc: | Defines the parameters representing the output of the process. May be used to generate and/or validate the output object. - name: requirements type: - "null" - type: array items: ProcessRequirement doc: > Declares requirements that apply to either the runtime environment or the workflow engine that must be met in order to execute this workflow step. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. - name: hints type: - "null" - type: array items: Any doc: > Declares hints applying to either the runtime environment or the workflow engine that may be helpful in executing this workflow step. It is not an error if an implementation cannot satisfy all hints, however the implementation may report a warning. - name: label type: - "null" - string jsondldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: description type: - "null" - string jsondldPredicate: "rdfs:comment" doc: "A long, human-readable description of this process object." - name: run type: Process doc: | Specifies the process to run. - name: scatter type: - "null" - string - type: array items: string jsonldPredicate: "@id": "cwl:scatter" "@type": "@id" "@container": "@list" - name: scatterMethod doc: | Required if `scatter` is an array of more than one element. type: - "null" - ScatterMethod jsonldPredicate: "@id": "cwl:scatterMethod" "@type": "@vocab" - name: Workflow type: record docParent: Reference extends: Process specialize: OutputParameter: WorkflowOutputParameter doc: | A workflow is a process consisting of one or more `steps`. Each step has input and output parameters defined by the `inputs` and `outputs` fields. A workflow executes as described in [execution model](#workflow_graph). # Dependencies Dependencies between parameters are expressed using the `source` field on [workflow step input parameters](#workflowstepinput) and [workflow output parameters](#workflowoutputparameter). The `source` field expresses the dependency of one parameter on another such that when a value is associated with the parameter specified by `source`, that value is propagated to the destination parameter. When all data links inbound to a given step are fufilled, the step is ready to execute. # Extensions [ScatterFeatureRequirement](#scatterfeaturerequirement) and [SubworkflowFeatureRequirement](#subworkflowfeaturerequirement) are available as standard extensions to core workflow semantics. fields: - name: "class" jsonldPredicate: "@id": "@type" "@type": "@vocab" type: string - name: steps doc: | The individual steps that make up the workflow. Each step is executed when all of its input data links are fufilled. An implementation may choose to execute the steps in a different order than listed and/or execute steps concurrently, provided that dependencies between steps are met. type: - type: array items: WorkflowStep - type: record name: DockerRequirement extends: ProcessRequirement doc: | Indicates that a workflow component should be run in a [Docker](http://docker.com) container, and specifies how to fetch or build the image. If a CommandLineTool lists `DockerRequirement` under `hints` or `requirements`, it may (or must) be run in the specified Docker container. The platform must first acquire or install the correct Docker image as specified by `dockerPull`, `dockerLoad` or `dockerFile`. The platform must execute the tool in the container using `docker run` with the appropriate Docker image and tool command line. The workflow platform may provide input files and the designated output directory through the use of volume bind mounts. The platform may rewrite file paths in the input object to correspond to the Docker bind mounted locations. When running a tool contained in Docker, the workflow platform must not assume anything about the contents of the Docker container, such as the presence or absence of specific software, except to assume that the generated command line represents a valid command within the runtime environment of the container. ## Interaction with other requirements If [EnvVarRequirement](#envvarrequirement) is specified alongside a DockerRequirement, the environment variables must be provided to Docker using `--env` or `--env-file` and interact with the container's preexisting environment as defined by Docker. fields: - name: dockerPull type: ["null", "string"] doc: "Specify a Docker image to retrieve using `docker pull`." - name: "dockerLoad" type: ["null", "string"] doc: "Specify a HTTP URL from which to download a Docker image using `docker load`." - name: dockerFile type: ["null", "string"] doc: "Supply the contents of a Dockerfile which will be built using `docker build`." - name: dockerImageId type: ["null", "string"] doc: | The image id that will be used for `docker run`. May be a human-readable image name or the image identifier hash. May be skipped if `dockerPull` is specified, in which case the `dockerPull` image id must be used. - name: dockerOutputDirectory type: ["null", "string"] doc: | Set the designated output directory to a specific location inside the Docker container. - type: record name: SubworkflowFeatureRequirement extends: ProcessRequirement doc: | Indicates that the workflow platform must support nested workflows in the `run` field of (WorkflowStep)(#workflowstep). - name: CreateFileRequirement type: record extends: ProcessRequirement doc: | Define a list of files that must be created by the workflow platform in the designated output directory prior to executing the command line tool. See `FileDef` for details. fields: - name: fileDef type: type: "array" items: "FileDef" doc: The list of files. - name: EnvVarRequirement type: record extends: ProcessRequirement doc: | Define a list of environment variables which will be set in the execution environment of the tool. See `EnvironmentDef` for details. fields: - name: envDef type: type: "array" items: "EnvironmentDef" doc: The list of environment variables. - name: ScatterFeatureRequirement type: record extends: ProcessRequirement doc: | Indicates that the workflow platform must support the `scatter` and `scatterMethod` fields of [WorkflowStep](#workflowstep). - name: SchemaDefRequirement type: record extends: ProcessRequirement doc: | This field consists of an array of type definitions which must be used when interpreting the `inputs` and `outputs` fields. When a symbolic type is encountered that is not in [`Datatype`](#datatype), the implementation must check if the type is defined in `schemaDefs` and use that definition. If the type is not found in `schemaDefs`, it is an error. The entries in `schemaDefs` must be processed in the order listed such that later schema definitions may refer to earlier schema definitions. fields: - name: types type: type: array items: SchemaDef doc: The list of type definitions. - type: record name: ExpressionEngineRequirement extends: ProcessRequirement doc: | Define an expression engine, as described in [Expressions](#expressions). fields: - name: id type: string doc: "Used to identify the expression engine in the `engine` field of Expressions." jsonldPredicate: "@id" - name: requirements type: - "null" - type: array items: ProcessRequirement doc: | Requirements to run this expression engine, such as DockerRequirement for specifying a container to run the engine. - name: engineCommand type: - "null" - string - type: array items: string doc: "The command line to invoke the expression engine." - name: engineConfig type: - "null" - type: array items: string doc: | Additional configuration or code fragments that will also be passed to the expression engine. The semantics of this field are defined by the underlying expression engine. Intended for uses such as providing function definitions that will be called from CWL expressions. cwltool-1.0.20170114120503/cwltool/schemas/draft-2/CommonWorkflowLanguage.yml0000644000175000017500000021506013036412052027153 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" sld: "https://w3id.org/cwl/salad#" rdfs: "http://www.w3.org/2000/01/rdf-schema#" $graph: - name: "Common Workflow Language, Draft 2" type: documentation doc: | 7 July 2015 This version: * https://w3id.org/cwl/draft-2/ Current version: * https://w3id.org/cwl/ Authors: * Peter Amstutz , Curoverse * Nebojša Tijanić , Seven Bridges Genomics Contributers: * Luka Stojanovic , Seven Bridges Genomics * John Chilton , Galaxy Project, Pennsylvania State University * Michael R. Crusoe , Michigan State University * Hervé Ménager , Institut Pasteur * Maxim Mikheev , BioDatomics * Stian Soiland-Reyes [soiland-reyes@cs.manchester.ac.uk](mailto:soiland-reyes@cs.manchester.ac.uk), University of Manchester # Abstract A Workflow is an analysis task represented by a directed graph describing a sequence of operations that transform an input data set to output. This specification defines the Common Workflow Language (CWL), a vendor-neutral standard for representing workflows and concrete process steps intended to be portable across a variety of computing platforms. # Status of This Document This document is the product of the [Common Workflow Language working group](https://groups.google.com/forum/#!forum/common-workflow-language). The latest version of this document is available in the "specification" directory at https://github.com/common-workflow-language/common-workflow-language The products of the CWL working group (including this document) are made available under the terms of the Apache License, version 2.0. # Introduction The Common Workflow Language (CWL) working group is an informal, multi-vendor working group consisting of various organizations and individuals that have an interest in portability of data analysis workflows. The goal is to create specifications like this one that enable data scientists to describe analysis tools and workflows that are powerful, easy to use, portable, and support reproducibility. ## Introduction to draft 2 This specification represents the second milestone of the CWL group. Since draft-1, this draft introduces a number of major changes and additions: * Use of Avro schema (instead of JSON-schema) and JSON-LD for data modeling. * Significant refactoring of the Command Line Tool description. * Data and execution model for Workflows. * Extension mechanism though "hints" and "requirements". ## Purpose CWL is designed to express workflows for data-intensive science, such as Bioinformatics, Chemistry, Physics, and Astronomy. This specification is intended to define a data and execution model for Workflows and Command Line Tools that can be implemented on top of a variety of computing platforms, ranging from an individual workstation to cluster, grid, cloud, and high performance computing systems. ## References to Other Specifications * [JSON](http://json.org) * [JSON-LD](http://json-ld.org) * [JSON Pointer](https://tools.ietf.org/html/draft-ietf-appsawg-json-pointer-04) * [YAML](http://yaml.org) * [Avro](https://avro.apache.org/docs/current/spec.html) * [Uniform Resource Identifier (URI): Generic Syntax](https://tools.ietf.org/html/rfc3986) * [UTF-8](https://www.ietf.org/rfc/rfc2279.txt) * [Portable Operating System Interface (POSIX.1-2008)](http://pubs.opengroup.org/onlinepubs/9699919799/) * [Resource Description Framework (RDF)](http://www.w3.org/RDF/) ## Scope This document describes the CWL syntax, execution, and object model. It is not intended to document a specific implementation of CWL, however it may serve as a reference for the behavior of conforming implementations. ## Terminology The terminology used to describe CWL documents is defined in the Concepts section of the specification. The terms defined in the following list are used in building those definitions and in describing the actions of an CWL implementation: **may**: Conforming CWL documents and CWL implementations are permitted but not required to behave as described. **must**: Conforming CWL documents and CWL implementations are required to behave as described; otherwise they are in error. **error**: A violation of the rules of this specification; results are undefined. Conforming implementations may detect and report an error and may recover from it. **fatal error**: A violation of the rules of this specification; results are undefined. Conforming implementations must not continue to execute the current process and may report an error. **at user option**: Conforming software may or must (depending on the modal verb in the sentence) behave as described; if it does, it must provide users a means to enable or disable the behavior described. # Data model ## Data concepts An **object** is a data structure equivalent to the "object" type in JSON, consisting of a unordered set of name/value pairs (referred to here as **fields**) and where the name is a string and the value is a string, number, boolean, array, or object. A **document** is a file containing a serialized object, or an array of objects. A **process** is a basic unit of computation which accepts input data, performs some computation, and produces output data. An **input object** is an object describing the inputs to a invocation of process. An **output object** is an object describing the output of an invocation of a process. An **input schema** describes the valid format (required fields, data types) for an input object. An **output schema** describes the valid format for a output object. **Metadata** is information about workflows, tools, or input items that is not used directly in the computation. ## Syntax Documents containing CWL objects are serialized and loaded using YAML syntax and UTF-8 text encoding. A conforming implementation must accept all valid YAML documents. The CWL schema is defined using Avro Linked Data (avro-ld). Avro-ld is an extension of the Apache Avro schema language to support additional annotations mapping Avro fields to RDF predicates via JSON-LD. A CWL document may be validated by transforming the avro-ld schema to a base Apache Avro schema. An implementation may interpret a CWL document as [JSON-LD](http://json-ld.org) and convert a CWL document to a [Resource Description Framework (RDF)](http://www.w3.org/RDF/) using the CWL [JSON-LD Context](https://w3id.org/cwl/draft-2/context) (extracted from the avro-ld schema). The CWL [RDFS schema](https://w3id.org/cwl/draft-2/cwl.ttl) defines the classes and properties used by CWL as JSON-LD. The latest draft-2 schema is defined here: https://github.com/common-workflow-language/common-workflow-language/blob/master/schemas/draft-2/cwl-avro.yml ## Identifiers If an object contains an `id` field, that is used to uniquely identify the object in that document. The value of the `id` field must be unique over the entire document. The format of the `id` field is that of a [relative fragment identifier](https://tools.ietf.org/html/rfc3986#section-3.5), and must start with a hash `#` character. An implementation may choose to only honor references to object types for which the `id` field is explicitly listed in this specification. When loading a CWL document, an implementation may resolve relative identifiers to absolute URI references. For example, "my_tool.cwl" located in the directory "/home/example/work/" may be transformed to "file:///home/example/work/my_tool.cwl" and a relative fragment reference "#input" in this file may be transformed to "file:///home/example/work/my_tool.cwl#input". ## Document preprocessing An implementation must resolve `import` directives. An `import` directive is an object consisting of the field `import` specifying a URI. The URI referenced by `import` must be loaded as a CWL document (including recursive preprocessing) and then the `import` object is implicitly replaced by the external resource. URIs may include document fragments referring to objects identified by their `id` field, in which case the `import` directive is replaced by only the fragment object. An implementation must resolve `include` directives. An `include` directive is an object consisting of the field `include` specifying a URI. The URI referenced by `include` must be loaded as UTF-8 encoded text document and the `include` directive implicitly replaced by a string with the contents of the document. Because the loaded resource is unparsed, URIs used with `include` must not include fragments. ## Extensions and Metadata Implementation extensions not required for correct execution (for example, fields related to GUI rendering) may be stored in [process hints](#requirements_and_hints). Input metadata (for example, a lab sample identifier) may be explicitly represented within a workflow using input parameters which are propagated to output. Future versions of this specification may define additional facilities for working with input/output metadata. Fields for tool and workflow metadata (for example, authorship for use in citations) are not defined in this specification. Future versions of this specification may define such fields. # Execution model ## Execution concepts A **parameter** is a named symbolic input or output of process, with an associated datatype or schema. During execution, values are assigned to parameters to make the input object or output object used for concrete process invocation. A **command line tool** is a process characterized by the execution of a standalone, non-interactive program which is invoked on some input, produces output, and then terminates. A **workflow** is a process characterized by multiple subprocess steps, where step outputs are connected to the inputs of other downstream steps to form a directed graph, and independent steps may run concurrently. A **runtime environment** is the actual hardware and software environment when executing a command line tool. It includes, but is not limited to, the hardware architecture, hardware resources, operating system, software runtime (if applicable, such as the Python interpreter or the JVM), libraries, modules, packages, utilities, and data files required to run the tool. A **workflow platform** is a specific hardware and software implementation capable of interpreting a CWL document and executing the processes specified by the document. The responsibilities of the workflow platform may include scheduling process invocation, setting up the necessary runtime environment, making input data available, invoking the tool process, and collecting output. It is intended that the workflow platform has broad leeway outside of this specification to optimize use of computing resources and enforce policies not covered by this specifcation. Some areas that are currently out of scope for CWL specification but may be handled by a specific workflow platform include: * Data security and permissions. * Scheduling tool invocations on remote cluster or cloud compute nodes. * Using virtual machines or operating system containers to manage the runtime (except as described in [DockerRequirement](#dockerrequirement)). * Using remote or distributed file systems to manage input and output files. * Translating or rewriting file paths. * Determining if a process has previously been executed, skipping it and reusing previous results. * Pausing and resuming of processes or workflows. Conforming CWL processes must not assume anything about the runtime environment or workflow platform unless explicitly declared though the use of [process requirements](#processrequirement). ## Generic execution process The generic execution sequence of a CWL process (including both workflows and concrete process implementations) is as follows. 1. Load and validate CWL document, yielding a process object. 2. Load input object. 3. Validate the input object against the `inputs` schema for the process. 4. Validate that process requirements are met. 5. Perform any further setup required by the specific process type. 6. Execute the process. 7. Capture results of process execution into the output object. 8. Validate the output object against the `outputs` schema for the process. 9. Report the output object to the process caller. ## Requirements and hints A **[process requirement](#processrequirement)** modifies the semantics or runtime environment of a process. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. A **hint** is similar to a requirement, however it is not an error if an implementation cannot satisfy all hints. The implementation may report a warning if a hint cannot be satisfied. Requirements are inherited. A requirement specified in a Workflow applies to all workflow steps; a requirement specified on a workflow step will apply to the process implementation. If the same process requirement appears at different levels of the workflow, the most specific instance of the requirement is used, that is, an entry in `requirements` on a process implementation such as CommandLineTool will take precendence over an entry in `requirements` specified in a workflow step, and an entry in `requirements` on a workflow step takes precedence over the workflow. Entries in `hints` are resolved the same way. Requirements override hints. If a process implementation provides a process requirement in `hints` which is also provided in `requirements` by an enclosing workflow or workflow step, the enclosing `requirements` takes precedence. Process requirements are the primary mechanism for specifying extensions to the CWL core specification. ## Expressions An expression is a fragment of executable code which is evaluated by the workflow platform to affect the inputs, outputs, or behavior of a process. In the generic execution sequence, expressions may be evaluated during step 5 (process setup), step 6 (execute process), and/or step 7 (capture output). Expressions are distinct from regular processes in that they are intended to modify the behavior of the workflow itself rather than perform the primary work of the workflow. An implementation must provide the predefined `cwl:JsonPointer` expression engine. This expression engine specifies a [JSON Pointer](https://tools.ietf.org/html/draft-ietf-appsawg-json-pointer-04) into an expression input object consisting of the `job` and `context` fields described below. An expression engine defined with [ExpressionEngineRequirement](#expressionenginerequirement) is a command line program following the following protocol: * On standard input, receive a JSON object with the following fields: - **engineConfig**: A list of strings from the `engineConfig` field. Null if `engineConfig` is not specified. - **job**: The input object of the current Process (context dependent). - **context**: The specific value being transformed (context dependent). May be null. - **script**: The code fragment to evaluate. - **outdir**: When used in the context of a CommandLineTool, this is the designated output directory that will be used when executing the tool. Null if not applicable. - **tmpdir**: When used in the context of a CommandLineTool, this is the designated temporary directory that will be used when executing the tool. Null if not applicable. * On standard output, print a single JSON value (string, number, array, object, boolean, or null) for the return value. Expressions must be evaluated in an isolated context (a "sandbox") which permits no side effects to leak outside the context, and permit no outside data to leak into the context. Implementations may apply limits, such as process isolation, timeouts, and operating system containers/jails to minimize the security risks associated with running untrusted code. The order in which expressions are evaluated within a process or workflow is undefined. ## Workflow graph A workflow describes a set of **steps** and the **dependencies** between those processes. When a process produces output that will be consumed by a second process, the first process is a dependency of the second process. When there is a dependency, the workflow engine must execute the dependency process and wait for it to successfully produce output before executing the dependent process. If two processes are defined in the workflow graph that are not directly or indirectly dependent, these processes are **independent**, and may execute in any order or execute concurrently. A workflow is complete when all steps have been executed. ## Success and failure A completed process must result in one of `success`, `temporaryFailure` or `permanentFailure` states. An implementation may choose to retry a process execution which resulted in `temporaryFailure`. An implementation may choose to either continue running other steps of a workflow, or terminate immediately upon `permanentFailure`. * If any step of a workflow execution results in `permanentFailure`, then the workflow status is `permanentFailure`. * If one or more steps result in `temporaryFailure` and all other steps complete `success` or are not executed, then the workflow status is `temporaryFailure`. * If all workflow steps are executed and complete with `success`, then the workflow status is `success`. ## Executing CWL documents as scripts By convention, a CWL document may begin with `#!/usr/bin/env cwl-runner` and be marked as executable (the POSIX "+x" permission bits) to enable it to be executed directly. A workflow platform may support this mode of operation; if so, it must provide `cwl-runner` as an alias for the platform's CWL implementation. # Sample CWL workflow revtool.cwl: ``` #!/usr/bin/env cwl-runner # # Simplest example command line program wrapper for the Unix tool "rev". # class: CommandLineTool description: "Reverse each line using the `rev` command" # The "inputs" array defines the structure of the input object that describes # the inputs to the underlying program. Here, there is one input field # defined that will be called "input" and will contain a "File" object. # # The input binding indicates that the input value should be turned into a # command line argument. In this example inputBinding is an empty object, # which indicates that the file name should be added to the command line at # a default location. inputs: - id: "#input" type: File inputBinding: {} # The "outputs" array defines the structure of the output object that # describes the outputs of the underlying program. Here, there is one # output field defined that will be called "output", must be a "File" type, # and after the program executes, the output value will be the file # output.txt in the designated output directory. outputs: - id: "#output" type: File outputBinding: glob: output.txt # The actual program to execute. baseCommand: rev # Specify that the standard output stream must be redirected to a file called # output.txt in the designated output directory. stdout: output.txt ``` sorttool.cwl: ``` #!/usr/bin/env cwl-runner # # Example command line program wrapper for the Unix tool "sort" # demonstrating command line flags. class: CommandLineTool description: "Sort lines using the `sort` command" # This example is similar to the previous one, with an additional input # parameter called "reverse". It is a boolean parameter, which is # intepreted as a command line flag. The value of "prefix" is used for # flag to put on the command line if "reverse" is true. If "reverse" is # false, no flag is added. # # This example also introduced the "position" field. This indicates the # sorting order of items on the command line. Lower numbers are placed # before higher numbers. Here, the "--reverse" flag (if present) will be # added to the command line before the input file path. inputs: - id: "#reverse" type: boolean inputBinding: position: 1 prefix: "--reverse" - id: "#input" type: File inputBinding: position: 2 outputs: - id: "#output" type: File outputBinding: glob: output.txt baseCommand: sort stdout: output.txt ``` revsort.cwl: ``` #!/usr/bin/env cwl-runner # # This is a two-step workflow which uses "revtool" and "sorttool" defined above. # class: Workflow description: "Reverse the lines in a document, then sort those lines." # Requirements specify prerequisites and extensions to the workflow. # In this example, DockerRequirement specifies a default Docker container # in which the command line tools will execute. requirements: - class: DockerRequirement dockerPull: debian:8 # The inputs array defines the structure of the input object that describes # the inputs to the workflow. # # The "reverse_sort" input parameter demonstrates the "default" field. If the # field "reverse_sort" is not provided in the input object, the default value will # be used. inputs: - id: "#input" type: File description: "The input file to be processed." - id: "#reverse_sort" type: boolean default: true description: "If true, reverse (descending) sort" # The "outputs" array defines the structure of the output object that describes # the outputs of the workflow. # # Each output field must be connected to the output of one of the workflow # steps using the "connect" field. Here, the parameter "#output" of the # workflow comes from the "#sorted" output of the "sort" step. outputs: - id: "#output" type: File source: "#sorted.output" description: "The output with the lines reversed and sorted." # The "steps" array lists the executable steps that make up the workflow. # The tool to execute each step is listed in the "run" field. # # In the first step, the "inputs" field of the step connects the upstream # parameter "#input" of the workflow to the input parameter of the tool # "revtool.cwl#input". # # In the second step, the "inputs" field of the step connects the output # parameter "#reversed" from the first step to the input parameter of the # tool "sorttool.cwl#input". steps: - inputs: - { id: "#rev.input", source: "#input" } outputs: - { id: "#rev.output" } run: { import: revtool.cwl } - inputs: - { id: "#sorted.input", source: "#rev.output" } - { id: "#sorted.reverse", source: "#reverse_sort" } outputs: - { id: "#sorted.output" } run: { import: sorttool.cwl } ``` Sample input object: ``` { "input": { "class": "File", "path": "whale.txt" } } ``` Sample output object: ``` { "output": { "path": "/tmp/tmpdeI_p_/output.txt", "size": 1111, "class": "File", "checksum": "sha1$b9214658cc453331b62c2282b772a5c063dbd284" } } ``` - name: Reference type: documentation doc: This section specifies the core object types that make up a CWL document. - type: enum name: CWLVersions doc: "Version symbols for published CWL document versions." symbols: - cwl:draft-2 - name: Datatype type: enum docAfter: "#ProcessRequirement" symbols: - "null" - sld:boolean - sld:int - sld:long - sld:float - sld:double - sld:bytes - sld:string - sld:record - sld:enum - sld:array - sld:map - cwl:File - cwl:Any doc: | CWL data types are based on Avro schema declarations. Refer to the [Avro schema declaration documentation](https://avro.apache.org/docs/current/spec.html#schemas) for detailed information. In addition, CWL defines [`File`](#file) as a special record type. ## Primitive types * **null**: no value * **boolean**: a binary value * **int**: 32-bit signed integer * **long**: 64-bit signed integer * **float**: single precision (32-bit) IEEE 754 floating-point number * **double**: double precision (64-bit) IEEE 754 floating-point number * **bytes**: sequence of uninterpreted 8-bit unsigned bytes * **string**: Unicode character sequence ## Complex types * **record**: An object with one or more fields defined by name and type * **enum**: A value from a finite set of symbolic values * **array**: An ordered sequence of values * **map**: An unordered collection of key/value pairs ## File type See [File](#file) below. ## Any type See [Any](#any) below. - name: File type: record docParent: "#Datatype" doc: | Represents a file (or group of files if `secondaryFiles` is specified) that must be accessible by tools using standard POSIX file system call API such as open(2) and read(2). fields: - name: "class" type: type: enum name: "File_class" symbols: - cwl:File jsonldPredicate: "_id": "@type" "_type": "@vocab" doc: Must be `File` to indicate this object describes a file. - name: "path" type: "string" doc: The path to the file. - name: "checksum" type: ["null", "string"] doc: | Optional hash code for validating file integrity. Currently must be in the form "sha1$ + hexidecimal string" using the SHA-1 algorithm. - name: "size" type: ["null", "long"] doc: Optional file size. - name: "cwl:secondaryFiles" type: - "null" - type: array items: "#File" doc: | A list of additional files that are associated with the primary file and must be transferred alongside the primary file. Examples include indexes of the primary file, or external references which must be included when loading primary document. A file object listed in `secondaryFiles` may itself include `secondaryFiles` for which the same rules apply. - name: Any type: enum docParent: "#Datatype" symbols: ["cwl:Any"] doc: | The **Any** type validates for any non-null value. - name: Schema type: record doc: "A schema defines a parameter type." docParent: "#Parameter" fields: - name: type doc: "The data type of this parameter." type: - "#Datatype" - "#Schema" - "string" - type: "array" items: [ "#Datatype", "#Schema", "string" ] jsonldPredicate: "_id": "sld:type" "_type": "@vocab" - name: fields type: - "null" - type: "array" items: "#Schema" jsonldPredicate: "_id": "sld:fields" "_container": "@list" doc: "When `type` is `record`, defines the fields of the record." - name: "symbols" type: - "null" - type: "array" items: "string" jsonldPredicate: "_id": "sld:symbols" "_container": "@list" doc: "When `type` is `enum`, defines the set of valid symbols." - name: items type: - "null" - "#Datatype" - "#Schema" - "string" - type: "array" items: [ "#Datatype", "#Schema", "string" ] jsonldPredicate: "_id": "sld:items" "_container": "@list" doc: "When `type` is `array`, defines the type of the array elements." - name: "values" type: - "null" - "#Datatype" - "#Schema" - "string" - type: "array" items: [ "#Datatype", "#Schema", "string" ] jsonldPredicate: "_id": "sld:values" "_container": "@list" doc: "When `type` is `map`, defines the value type for the key/value pairs." - name: Parameter type: record docParent: "#Process" abstract: true doc: | Define an input or output parameter to a process. fields: - name: type type: - "null" - "#Datatype" - "#Schema" - string - type: array items: - "#Datatype" - "#Schema" - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" doc: | Specify valid types of data that may be assigned to this parameter. - name: label type: - "null" - string jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this parameter object." - name: description type: - "null" - string jsonldPredicate: "rdfs:comment" doc: "A long, human-readable description of this parameter object." - name: streamable type: ["null", "boolean"] doc: | Currently only applies if `type` is `File`. A value of `true` indicates that the file is read or written sequentially without seeking. An implementation may use this flag to indicate whether it is valid to stream file contents using a named pipe. Default: `false`. - name: cwl:default type: ["null", Any] doc: | The default value for this parameter if not provided in the input object. - name: JsonPointer type: enum docParent: "#Expression" symbols: - "cwl:JsonPointer" - type: record name: Expression docAfter: "#ExpressionTool" doc: | Define an expression that will be evaluated and used to modify the behavior of a tool or workflow. See [Expressions](#expressions) for more information about expressions and [ExpressionEngineRequirement](#expressionenginerequirement) for information on how to define a expression engine. fields: - name: engine type: - "#JsonPointer" - string doc: | Either `cwl:JsonPointer` or a reference to an ExpressionEngineRequirement defining which engine to use. jsonldPredicate: "_id": "cwl:engine" "_type": "@id" - name: script type: string doc: "The code to be executed by the expression engine." - name: Binding type: record docParent: "#Parameter" fields: - name: loadContents type: - "null" - boolean doc: | Only applies when `type` is `File`. Read up to the first 64 KiB of text from the file and place it in the "contents" field of the file object for manipulation by expressions. - name: cwl:secondaryFiles type: - "null" - "string" - "#Expression" - type: "array" items: ["string", "#Expression"] doc: | Only applies when `type` is `File`. Describes files that must be included alongside the primary file. If the value is an expression, the context of the expression is the input or output File parameter to which this binding applies. If the value is a string, it specifies that the following pattern should be applied to the primary file: 1. If string begins with one or more caret `^` characters, for each caret, remove the last file extension from the path (the last period `.` and all following characters). If there are no file extensions, the path is unchanged. 2. Append the remainder of the string to the end of the file path. - name: InputSchema type: record extends: "#Schema" docParent: "#InputParameter" specialize: - specializeFrom: "#Schema" specializeTo: "#InputSchema" fields: - name: cwl:inputBinding type: [ "null", "#Binding" ] doc: | Describes how to handle a value in the input object convert it into a concrete form for execution, such as command line parameters. - name: OutputSchema type: record extends: "#Schema" docParent: "#OutputParameter" specialize: - specializeFrom: "#Schema" specializeTo: "#OutputSchema" - name: InputParameter type: record extends: "#Parameter" docAfter: "#Parameter" specialize: - specializeFrom: "#Schema" specializeTo: "#InputSchema" fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this parameter object." - name: "cwl:inputBinding" type: [ "null", "#Binding" ] doc: | Describes how to handle the inputs of a process and convert them into a concrete form for execution, such as command line parameters. - name: OutputParameter type: record extends: "#Parameter" docAfter: "#Parameter" specialize: - specializeFrom: "#Schema" specializeTo: "#OutputSchema" fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this parameter object." - type: record name: "FileDef" docParent: "#CreateFileRequirement" doc: | Define a file that must be placed in the designated output directory prior to executing the command line tool. May be the result of executing an expression, such as building a configuration file from a template. fields: - name: "filename" type: ["string", "#Expression"] doc: "The name of the file to create in the output directory." - name: "fileContent" type: ["string", "#Expression"] doc: | If the value is a string literal or an expression which evaluates to a string, a new file must be created with the string as the file contents. If the value is an expression that evaluates to a File object, this indicates the referenced file should be added to the designated output directory prior to executing the tool. Files added in this way may be read-only, and may be provided by bind mounts or file system links to avoid unnecessary copying of the input file. - type: record name: EnvironmentDef docParent: "#EnvVarRequirement" doc: | Define an environment variable that will be set in the runtime environment by the workflow platform when executing the command line tool. May be the result of executing an expression, such as getting a parameter from input. fields: - name: "envName" type: "string" doc: The environment variable name - name: "envValue" type: ["string", "#Expression"] doc: The environment variable value - type: record name: SchemaDef extends: "#InputSchema" docParent: "#SchemaDefRequirement" specialize: - specializeFrom: "#InputSchema" specializeTo: "#SchemaDef" - specializeFrom: "#Binding" specializeTo: "#CommandLineBinding" fields: - name: name type: ["null", string] doc: "The type name being defined." - type: record name: ProcessRequirement docAfter: "#ExpressionTool" abstract: true doc: | A process requirement declares a prerequisite that may or must be fulfilled before executing a process. See [`Process.hints`](#process) and [`Process.requirements`](#process). Process requirements are the primary mechanism for specifying extensions to the CWL core specification. fields: - name: "class" type: "string" doc: "The specific requirement type." jsonldPredicate: "_id": "@type" "_type": "@vocab" - type: record name: Process abstract: true docAfter: "#ProcessRequirement" doc: | The base executable type in CWL is the `Process` object defined by the document. Note that the `Process` object is abstract and cannot be directly executed. fields: - name: id type: ["null", string] jsonldPredicate: "@id" doc: "The unique identifier for this process object." - name: cwl:inputs type: type: array items: "#InputParameter" doc: | Defines the input parameters of the process. The process is ready to run when all required input parameters are associated with concrete values. Input parameters include a schema for each parameter which is used to validate the input object. It may also be used build a user interface for constructing the input object. - name: cwl:outputs type: type: array items: "#OutputParameter" doc: | Defines the parameters representing the output of the process. May be used to generate and/or validate the output object. - name: cwl:requirements type: - "null" - type: array items: "#ProcessRequirement" doc: > Declares requirements that apply to either the runtime environment or the workflow engine that must be met in order to execute this process. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. - name: hints type: - "null" - type: array items: Any doc: > Declares hints applying to either the runtime environment or the workflow engine that may be helpful in executing this process. It is not an error if an implementation cannot satisfy all hints, however the implementation may report a warning. jsonldPredicate: _id: cwl:hints noLinkCheck: true - name: label type: - "null" - string jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: description type: - "null" - string jsonldPredicate: "rdfs:comment" doc: "A long, human-readable description of this process object." - name: cwlVersion type: - "null" - "#CWLVersions" doc: "CWL document version" jsonldPredicate: "_id": "cwl:cwlVersion" "_type": "@vocab" - type: record name: CommandLineBinding extends: "#Binding" docParent: "#CommandInputParameter" doc: | When listed under `inputBinding` in the input schema, the term "value" refers to the the corresponding value in the input object. For binding objects listed in `CommandLineTool.arguments`, the term "value" refers to the effective value after evaluating `valueFrom`. The binding behavior when building the command line depends on the data type of the value. If there is a mismatch between the type described by the input schema and the effective value, such as resulting from an expression evaluation, an implementation must use the data type of the effective value. - **string**: Add `prefix` and the string to the command line. - **number**: Add `prefix` and decimal representation to command line. - **boolean**: If true, add `prefix` to the command line. If false, add nothing. - **File**: Add `prefix` and the value of [`File.path`](#file) to the command line. - **array**: If `itemSeparator` is specified, add `prefix` and the join the array into a single string with `itemSeparator` separating the items. Otherwise first add `prefix`, then recursively process individual elements. - **object**: Add `prefix` only, and recursively add object fields for which `inputBinding` is specified. - **null**: Add nothing. fields: - name: "position" type: ["null", "int"] doc: "The sorting key. Default position is 0." - name: "prefix" type: [ "null", "string"] doc: "Command line prefix to add before the value." - name: "separate" type: ["null", boolean] doc: | If true (default), then the prefix and value must be added as separate command line arguments; if false, prefix and value must be concatenated into a single command line argument. - name: "itemSeparator" type: ["null", "string"] doc: | Join the array elements into a single string with the elements separated by by `itemSeparator`. - name: "valueFrom" type: - "null" - "string" - "#Expression" doc: | If `valueFrom` is a constant string value, use this as the value and apply the binding rules above. If `valueFrom` is an expression, evaluate the expression to yield the actual value to use to build the command line and apply the binding rules above. If the inputBinding is associated with an input parameter, the "context" of the expression will be the value of the input parameter. When a binding is part of the `CommandLineTool.arguments` field, the `valueFrom` field is required. - type: record name: CommandOutputBinding extends: "#Binding" docParent: "#CommandOutputParameter" doc: | Describes how to generate an output parameter based on the files produced by a CommandLineTool. The output parameter is generated by applying these operations in the following order: - glob - loadContents - outputEval fields: - name: glob type: - "null" - string - "#Expression" - type: array items: string doc: | Find files relative to the output directory, using POSIX glob(3) pathname matching. If provided an array, find files that match any pattern in the array. If provided an expression, the expression must return a string or an array of strings, which will then be evaluated as one or more glob patterns. Only files which actually exist will be matched and returned. - name: outputEval type: - "null" - "#Expression" doc: | Evaluate an expression to generate the output value. If `glob` was specified, the script `context` will be an array containing any files that were matched. Additionally, if `loadContents` is `true`, the File objects will include up to the first 64 KiB of file contents in the `contents` field. - type: record name: CommandInputSchema extends: "#InputSchema" docParent: "#CommandInputParameter" specialize: - specializeFrom: "#InputSchema" specializeTo: "#CommandInputSchema" - specializeFrom: "#Binding" specializeTo: "#CommandLineBinding" - type: record name: CommandOutputSchema extends: "#OutputSchema" docParent: "#CommandOutputParameter" specialize: - specializeFrom: "#OutputSchema" specializeTo: "#CommandOutputSchema" fields: - name: "cwl:outputBinding" type: [ "null", "#CommandOutputBinding" ] doc: | Describes how to handle the concrete outputs of a process step (such as files created by a program) and describe them in the process output parameter. - type: record name: CommandInputParameter extends: "#InputParameter" docParent: "#CommandLineTool" doc: An input parameter for a CommandLineTool. specialize: - specializeFrom: "#InputSchema" specializeTo: "#CommandInputSchema" - specializeFrom: "#Binding" specializeTo: "#CommandLineBinding" - type: record name: CommandOutputParameter extends: "#OutputParameter" docParent: "#CommandLineTool" doc: An output parameter for a CommandLineTool. specialize: - specializeFrom: "#OutputSchema" specializeTo: "#CommandOutputSchema" fields: - name: "cwl:outputBinding" type: [ "null", "#CommandOutputBinding" ] doc: | Describes how to handle the concrete outputs of a process step (such as files created by a program) and describe them in the process output parameter. - type: record name: CommandLineTool extends: "#Process" docAfter: "#Workflow" specialize: - specializeFrom: "#InputParameter" specializeTo: "#CommandInputParameter" - specializeFrom: "#OutputParameter" specializeTo: "#CommandOutputParameter" documentRoot: true doc: | A CommandLineTool process is a process implementation for executing a non-interactive application in a POSIX environment. To accommodate the enormous variety in syntax and semantics for input, runtime environment, invocation, and output of arbitrary programs, CommandLineTool uses an "input binding" that describes how to translate input parameters to an actual program invocation, and an "output binding" that describes how to generate output parameters from program output. # Input binding The tool command line is built by applying command line bindings to the input object. Bindings are listed either as part of an [input parameter](#commandinputparameter) using the `inputBinding` field, or separately using the `arguments` field of the CommandLineTool. The algorithm to build the command line is as follows. In this algorithm, the sort key is a list consisting of one or more numeric or string elements. Strings are sorted lexicographically based on UTF-8 encoding. 1. Collect `CommandLineBinding` objects from `arguments`. Assign a sorting key `[position, i]` where `position` is [`CommandLineBinding.position`](#commandlinebinding) and `i` is the index in the `arguments` list. 2. Collect `CommandLineBinding` objects from the `inputs` schema and associate them with values from the input object. Where the input type is a record, array, or map, recursively walk the schema and input object, collecting nested `CommandLineBinding` objects and associating them with values from the input object. 3. Create a sorting key by taking the value of the `position` field at each level leading to each leaf binding object. If `position` is not specified, it is not added to the sorting key. For bindings on arrays and maps, the sorting key must include the array index or map key following the position. If and only if two bindings have the same sort key, the tie must be broken using the ordering of the field or parameter name immediately containing the leaf binding. 4. Sort elements using the assigned sorting keys. Numeric entries sort before strings. 5. In the sorted order, apply the rules defined in [`CommandLineBinding`](#commandlinebinding) to convert bindings to actual command line elements. 6. Insert elements from `baseCommand` at the beginning of the command line. # Runtime environment All files listed in the input object must be made available in the runtime environment. The implementation may use a shared or distributed file system or transfer files via explicit download. Implementations may choose not to provide access to files not explicitly specified by the input object or process requirements. Output files produced by tool execution must be written to the **designated output directory**. The initial current working directory when executing the tool must be the designated output directory. When executing the tool, the child process must not inherit environment variables from the parent process. The tool must execute in a new, empty environment, containing only environment variables defined by [EnvVarRequirement](#envvarrequirement), the default environment of the Docker container specified in [DockerRequirement](#dockerrequirement) (if applicable), and `TMPDIR`. The `TMPDIR` environment variable must be set in the runtime environment to the **designated temporary directory**. Any files written to the designated temporary directory may be deleted by the workflow platform when the tool invocation is complete. An implementation may forbid the tool from writing to any location in the runtime environment file system other than the designated temporary directory and designated output directory. An implementation may provide read-only input files, and disallow in-place update of input files. The standard input stream and standard output stream may be redirected as described in the `stdin` and `stdout` fields. ## Extensions [DockerRequirement](#dockerrequirement), [CreateFileRequirement](#createfilerequirement), and [EnvVarRequirement](#envvarrequirement) are available as standard extensions to core command line tool semantics for defining the runtime environment. # Execution Once the command line is built and the runtime environment is created, the actual tool is executed. The standard error stream and standard output stream (unless redirected by setting `stdout`) may be captured by platform logging facilities for storage and reporting. Tools may be multithreaded or spawn child processes; however, when the parent process exits, the tool is considered finished regardless of whether any detached child processes are still running. Tools must not require any kind of console, GUI, or web based user interaction in order to start and run to completion. The exit code of the process indicates if the process completed successfully. By convention, an exit code of zero is treated as success and non-zero exit codes are treated as failure. This may be customized by providing the fields `successCodes`, `temporaryFailCodes`, and `permanentFailCodes`. An implementation may choose to default unspecified non-zero exit codes to either `temporaryFailure` or `permanentFailure`. # Output binding If the output directory contains a file named "cwl.output.json", that file must be loaded and used as the output object. Otherwise, the output object must be generated by walking the parameters listed in `outputs` and applying output bindings to the tool output. Output bindings are associated with output parameters using the `outputBinding` field. See [`CommandOutputBinding`](#commandoutputbinding) for details. fields: - name: "class" jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: baseCommand doc: | Specifies the program to execute. If the value is an array, the first element is the program to execute, and subsequent elements are placed at the beginning of the command line in prior to any command line bindings. If the program includes a path separator character it must be an absolute path, otherwise it is an error. If the program does not include a path separator, search the `$PATH` variable in the runtime environment of the workflow runner find the absolute path of the executable. type: - string - type: array items: string jsonldPredicate: "_id": "cwl:baseCommand" "_container": "@list" - name: arguments doc: | Command line bindings which are not directly associated with input parameters. type: - "null" - type: array items: [string, "#CommandLineBinding"] jsonldPredicate: "_id": "cwl:arguments" "_container": "@list" - name: stdin type: ["null", string, "#Expression"] doc: | A path to a file whose contents must be piped into the command's standard input stream. - name: stdout type: ["null", string, "#Expression"] doc: | Capture the command's standard output stream to a file written to the designated output directory. If `stdout` is a string, it specifies the file name to use. If `stdout` is an expression, the expression is evaluated and must return a string with the file name to use to capture stdout. If the return value is not a string, or the resulting path contains illegal characters (such as the path separator `/`) it is an error. - name: successCodes type: - "null" - type: array items: int doc: | Exit codes that indicate the process completed successfully. - name: temporaryFailCodes type: - "null" - type: array items: int doc: | Exit codes that indicate the process failed due to a possibly temporary condition, where excuting the process with the same runtime environment and inputs may produce different results. - name: permanentFailCodes type: - "null" - type: array items: int doc: Exit codes that indicate the process failed due to a permanent logic error, where excuting the process with the same runtime environment and same inputs is expected to always fail. - type: record name: ExpressionTool extends: "#Process" docAfter: "#CommandLineTool" documentRoot: true doc: | Execute an expression as a process step. fields: - name: "class" jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: expression type: "#Expression" doc: | The expression to execute. The expression must return a JSON object which matches the output parameters of the ExpressionTool. - name: LinkMergeMethod type: enum docParent: "#WorkflowStepInput" doc: The input link merge method, described in [WorkflowStepInput](#workflowstepinput). symbols: - merge_nested - merge_flattened - name: WorkflowOutputParameter type: record extends: "#OutputParameter" docParent: "#Workflow" doc: | Describe an output parameter of a workflow. The parameter must be connected to one or more parameters defined in the workflow that will provide the value of the output parameter. fields: - name: source doc: | Specifies one or more workflow parameters that will provide this output value. jsonldPredicate: "_id": "cwl:source" "_type": "@id" type: - "null" - string - type: array items: string - name: cwl:linkMerge type: ["null", "#LinkMergeMethod"] doc: | The method to use to merge multiple inbound links into a single array. If not specified, the default method is "merge_nested". - type: record name: WorkflowStepInput docParent: "#WorkflowStep" doc: | The input of a workflow step connects an upstream parameter (from the workflow inputs, or the outputs of other workflows steps) with the input parameters of the underlying process. ## Input object A WorkflowStepInput object must contain an `id` field in the form `#fieldname` or `#stepname.fieldname`. When the `id` field contains a period `.` the field name consists of the characters following the final period. This defines a field of the workflow step input object with the value of the `source` parameter(s). ## Merging If the sink parameter is an array, or named in a [workflow scatter](#workflowstep) operation, there may be multiple inbound data links listed in the `connect` field. The values from the input links are merged depending on the method specified in the `linkMerge` field. If not specified, the default method is "merge_nested". * **merge_nested** The input must be an array consisting of exactly one entry for each input link. If "merge_nested" is specified with a single link, the value from the link must be wrapped in a single-item list. * **merge_flattened** 1. The source and sink parameters must be compatible types, or the source type must be compatible with single element from the "items" type of the destination array parameter. 2. Source parameters which are arrays are concatenated. Source parameters which are single element types are appended as single elements. fields: - name: id type: string jsonldPredicate: "@id" doc: "A unique identifier for this workflow input parameter." - name: source doc: | Specifies one or more workflow parameters that will provide input to the underlying process parameter. jsonldPredicate: "_id": "cwl:source" "_type": "@id" type: - "null" - string - type: array items: string - name: cwl:linkMerge type: ["null", "#LinkMergeMethod"] doc: | The method to use to merge multiple inbound links into a single array. If not specified, the default method is "merge_nested". - name: cwl:default type: ["null", Any] doc: | The default value for this parameter if there is no `source` field. - type: record name: WorkflowStepOutput docParent: "#WorkflowStep" doc: | Associate an output parameter of the underlying process with a workflow parameter. The workflow parameter (given in the `id` field) be may be used as a `source` to connect with input parameters of other workflow steps, or with an output parameter of the process. fields: - name: id type: string jsonldPredicate: "@id" doc: | A unique identifier for this workflow output parameter. This is the identifier to use in the `source` field of `WorkflowStepInput` to connect the output value to downstream parameters. - name: ScatterMethod type: enum docParent: "#WorkflowStep" doc: The scatter method, as described in [workflow step scatter](#workflowstep). symbols: - dotproduct - nested_crossproduct - flat_crossproduct - name: WorkflowStep type: record docParent: "#Workflow" doc: | A workflow step is an executable element of a workflow. It specifies the underlying process implementation (such as `CommandLineTool`) in the `run` field and connects the input and output parameters of the underlying process to workflow parameters. # Scatter/gather To use scatter/gather, [ScatterFeatureRequirement](#scatterfeaturerequirement) must be specified in the workflow or workflow step requirements. A "scatter" operation specifies that the associated workflow step or subworkflow should execute separately over a list of input elements. Each job making up a scatter operaution is independent and may be executed concurrently. The `scatter` field specifies one or more input parameters which will be scattered. An input parameter may be listed more than once. The declared type of each input parameter is implicitly wrapped in an array for each time it appears in the `scatter` field. As a result, upstream parameters which are connected to scattered parameters may be arrays. All output parameter types are also implicitly wrapped in arrays. Each job in the scatter results in an entry in the output array. If `scatter` declares more than one input parameter, `scatterMethod` describes how to decompose the input into a discrete set of jobs. * **dotproduct** specifies that each of the input arrays are aligned and one element taken from each array to construct each job. It is an error if all input arrays are not the same length. * **nested_crossproduct** specifies the Cartesian product of the inputs, producing a job for every combination of the scattered inputs. The output must be nested arrays for each level of scattering, in the order that the input arrays are listed in the `scatter` field. * **flat_crossproduct** specifies the Cartesian product of the inputs, producing a job for every combination of the scattered inputs. The output arrays must be flattened to a single level, but otherwise listed in the order that the input arrays are listed in the `scatter` field. # Subworkflows To specify a nested workflow as part of a workflow step, [SubworkflowFeatureRequirement](#subworkflowfeaturerequirement) must be specified in the workflow or workflow step requirements. fields: - name: id type: ["null", string] jsonldPredicate: "@id" doc: "The unique identifier for this workflow step." - name: cwl:inputs type: type: array items: "#WorkflowStepInput" doc: | Defines the input parameters of the workflow step. The process is ready to run when all required input parameters are associated with concrete values. Input parameters include a schema for each parameter which is used to validate the input object. It may also be used build a user interface for constructing the input object. - name: cwl:outputs type: type: array items: "#WorkflowStepOutput" doc: | Defines the parameters representing the output of the process. May be used to generate and/or validate the output object. - name: cwl:requirements type: - "null" - type: array items: "#ProcessRequirement" doc: > Declares requirements that apply to either the runtime environment or the workflow engine that must be met in order to execute this workflow step. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. - name: hints type: - "null" - type: array items: Any doc: > Declares hints applying to either the runtime environment or the workflow engine that may be helpful in executing this workflow step. It is not an error if an implementation cannot satisfy all hints, however the implementation may report a warning. jsonldPredicate: _id: cwl:hints noLinkCheck: true - name: label type: - "null" - string jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: description type: - "null" - string jsonldPredicate: "rdfs:comment" doc: "A long, human-readable description of this process object." - name: run type: "#Process" doc: | Specifies the process to run. - name: scatter type: - "null" - string - type: array items: string jsonldPredicate: "_id": "cwl:scatter" "_type": "@id" "_container": "@list" - name: scatterMethod doc: | Required if `scatter` is an array of more than one element. type: - "null" - "#ScatterMethod" jsonldPredicate: "_id": "cwl:scatterMethod" "_type": "@vocab" - name: Workflow type: record docParent: "#Reference" extends: "#Process" specialize: - specializeFrom: "#OutputParameter" specializeTo: "#WorkflowOutputParameter" documentRoot: true doc: | A workflow is a process consisting of one or more `steps`. Each step has input and output parameters defined by the `inputs` and `outputs` fields. A workflow executes as described in [execution model](#workflow_graph). # Dependencies Dependencies between parameters are expressed using the `source` field on [workflow step input parameters](#workflowstepinput) and [workflow output parameters](#workflowoutputparameter). The `source` field expresses the dependency of one parameter on another such that when a value is associated with the parameter specified by `source`, that value is propagated to the destination parameter. When all data links inbound to a given step are fufilled, the step is ready to execute. # Extensions [ScatterFeatureRequirement](#scatterfeaturerequirement) and [SubworkflowFeatureRequirement](#subworkflowfeaturerequirement) are available as standard extensions to core workflow semantics. fields: - name: "class" jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: steps doc: | The individual steps that make up the workflow. Each step is executed when all of its input data links are fufilled. An implementation may choose to execute the steps in a different order than listed and/or execute steps concurrently, provided that dependencies between steps are met. type: - type: array items: "#WorkflowStep" - type: record name: DockerRequirement extends: "#ProcessRequirement" doc: | Indicates that a workflow component should be run in a [Docker](http://docker.com) container, and specifies how to fetch or build the image. If a CommandLineTool lists `DockerRequirement` under `hints` or `requirements`, it may (or must) be run in the specified Docker container. The platform must first acquire or install the correct Docker image as specified by `dockerPull`, `dockerLoad` or `dockerFile`. The platform must execute the tool in the container using `docker run` with the appropriate Docker image and tool command line. The workflow platform may provide input files and the designated output directory through the use of volume bind mounts. The platform may rewrite file paths in the input object to correspond to the Docker bind mounted locations. When running a tool contained in Docker, the workflow platform must not assume anything about the contents of the Docker container, such as the presence or absence of specific software, except to assume that the generated command line represents a valid command within the runtime environment of the container. ## Interaction with other requirements If [EnvVarRequirement](#envvarrequirement) is specified alongside a DockerRequirement, the environment variables must be provided to Docker using `--env` or `--env-file` and interact with the container's preexisting environment as defined by Docker. fields: - name: dockerPull type: ["null", "string"] doc: "Specify a Docker image to retrieve using `docker pull`." - name: "dockerLoad" type: ["null", "string"] doc: "Specify a HTTP URL from which to download a Docker image using `docker load`." - name: dockerFile type: ["null", "string"] doc: "Supply the contents of a Dockerfile which will be built using `docker build`." - name: dockerImageId type: ["null", "string"] doc: | The image id that will be used for `docker run`. May be a human-readable image name or the image identifier hash. May be skipped if `dockerPull` is specified, in which case the `dockerPull` image id must be used. - name: dockerOutputDirectory type: ["null", "string"] doc: | Set the designated output directory to a specific location inside the Docker container. - type: record name: SubworkflowFeatureRequirement extends: "#ProcessRequirement" doc: | Indicates that the workflow platform must support nested workflows in the `run` field of (WorkflowStep)(#workflowstep). - name: CreateFileRequirement type: record extends: "#ProcessRequirement" doc: | Define a list of files that must be created by the workflow platform in the designated output directory prior to executing the command line tool. See `FileDef` for details. fields: - name: fileDef type: type: "array" items: "#FileDef" doc: The list of files. - name: EnvVarRequirement type: record extends: "#ProcessRequirement" doc: | Define a list of environment variables which will be set in the execution environment of the tool. See `EnvironmentDef` for details. fields: - name: envDef type: type: "array" items: "#EnvironmentDef" doc: The list of environment variables. - name: ScatterFeatureRequirement type: record extends: "#ProcessRequirement" doc: | Indicates that the workflow platform must support the `scatter` and `scatterMethod` fields of [WorkflowStep](#workflowstep). - name: SchemaDefRequirement type: record extends: "#ProcessRequirement" doc: | This field consists of an array of type definitions which must be used when interpreting the `inputs` and `outputs` fields. When a symbolic type is encountered that is not in [`Datatype`](#datatype), the implementation must check if the type is defined in `schemaDefs` and use that definition. If the type is not found in `schemaDefs`, it is an error. The entries in `schemaDefs` must be processed in the order listed such that later schema definitions may refer to earlier schema definitions. fields: - name: types type: type: array items: "#SchemaDef" doc: The list of type definitions. - type: record name: ExpressionEngineRequirement extends: "#ProcessRequirement" doc: | Define an expression engine, as described in [Expressions](#expressions). fields: - name: id type: string doc: "Used to identify the expression engine in the `engine` field of Expressions." jsonldPredicate: "@id" - name: cwl:requirements type: - "null" - type: array items: "#ProcessRequirement" doc: | Requirements to run this expression engine, such as DockerRequirement for specifying a container to run the engine. - name: engineCommand type: - "null" - string - type: array items: string doc: "The command line to invoke the expression engine." - name: engineConfig type: - "null" - type: array items: string doc: | Additional configuration or code fragments that will also be passed to the expression engine. The semantics of this field are defined by the underlying expression engine. Intended for uses such as providing function definitions that will be called from CWL expressions. cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/0000755000175000017500000000000013036412107022217 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/0000755000175000017500000000000013036412107023303 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/0000755000175000017500000000000013036412107025707 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/0000755000175000017500000000000013036412107030016 5ustar michaelmichael00000000000000././@LongLink0000000000000000000000000000015300000000000011214 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name_schema.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name_sche0000644000175000017500000000040113036412053033021 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "ExampleType", "type": "record", "fields": [{ "name": "base", "type": "string", "jsonldPredicate": "http://example.com/base" }] }] } ././@LongLink0000000000000000000000000000015000000000000011211 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name_src.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name_src.0000644000175000017500000000025313036412053032751 0ustar michaelmichael00000000000000 { "base": "one", "form": { "http://example.com/base": "two", "http://example.com/three": "three", }, "acid:four": "four" } ././@LongLink0000000000000000000000000000014600000000000011216 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res_src.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res_src.ym0000644000175000017500000000047113036412053033044 0ustar michaelmichael00000000000000{ "$base": "http://example.com/base", "link": "http://example.com/base/zero", "form": { "link": "one", "things": [ { "link": "two" }, { "link": "#three", }, { "link": "four#five", }, { "link": "acid:six", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res.yml0000644000175000017500000000323413036412053032517 0ustar michaelmichael00000000000000- | ## Identifier resolution The schema may designate one or more fields as identifier fields to identify specific objects. Processing must resolve relative identifiers to absolute identifiers using the following rules: * If an identifier URI is prefixed with `#` it is a URI relative fragment identifier. It is resolved relative to the base URI by setting or replacing the fragment portion of the base URI. * If an identifier URI does not contain a scheme and is not prefixed `#` it is a parent relative fragment identifier. It is resolved relative to the base URI by the following rule: if the base URI does not contain a document fragment, set the fragment portion of the base URI. If the base URI does contain a document fragment, append a slash `/` followed by the identifier field to the fragment portion of the base URI. * If an identifier URI begins with a namespace prefix declared in `$namespaces` followed by a colon `:`, the prefix and colon must be replaced by the namespace declared in `$namespaces`. * If an identifier URI is an absolute URI consisting of a scheme and path, no processing occurs. When preprocessing visits a node containing an identifier, that identifier must be used as the base URI to process child nodes. It is an error for more than one object in a document to have the same absolute URI. ### Identifier resolution example Given the following schema: ``` - $include: ident_res_schema.yml - | ``` Process the following example: ``` - $include: ident_res_src.yml - | ``` This becomes: ``` - $include: ident_res_proc.yml - | ``` ././@LongLink0000000000000000000000000000015000000000000011211 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res_proc.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res_proc.0000644000175000017500000000036313036412053033007 0ustar michaelmichael00000000000000 { "form": { "things": [ { "voc": "red", }, { "voc": "red", }, { "voc": "http://example.com/acid#blue", } ] } } ././@LongLink0000000000000000000000000000014700000000000011217 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res_proc.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res_proc.y0000644000175000017500000000063313036412053033043 0ustar michaelmichael00000000000000{ "$base": "http://example.com/base", "link": "http://example.com/base/zero", "form": { "link": "http://example.com/one", "things": [ { "link": "http://example.com/two" }, { "link": "http://example.com/base#three" }, { "link": "http://example.com/four#five", }, { "link": "http://example.com/acid#six", } ] } } ././@LongLink0000000000000000000000000000014700000000000011217 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res_src.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res_src.y0000644000175000017500000000052213036412053033032 0ustar michaelmichael00000000000000 { "id": "http://example.com/base", "form": { "id": "one", "things": [ { "id": "two" }, { "id": "#three", }, { "id": "four#five", }, { "id": "acid:six", } ] } } ././@LongLink0000000000000000000000000000015200000000000011213 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res_schema.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res_schem0000644000175000017500000000053113036412053033062 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "Colors", "type": "enum", "symbols": ["acid:red"] }, { "name": "ExampleType", "type": "record", "fields": [{ "name": "voc", "type": "string", "jsonldPredicate": { "_type": "@vocab" } }] }] } cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name.yml0000644000175000017500000000246013036412053032626 0ustar michaelmichael00000000000000- | ## Field name resolution The document schema declares the vocabulary of known field names. During preprocessing traversal, field name in the document which are not part of the schema vocabulary must be resolved to absolute URIs. Under "strict" validation, it is an error for a document to include fields which are not part of the vocabulary and not resolvable to absolute URIs. Fields names which are not part of the vocabulary are resolved using the following rules: * If an field name URI begins with a namespace prefix declared in the document context (`@context`) followed by a colon `:`, the prefix and colon must be replaced by the namespace declared in `@context`. * If there is a vocabulary term which maps to the URI of a resolved field, the field name must be replace with the vocabulary term. * If a field name URI is an absolute URI consisting of a scheme and path and is not part of the vocabulary, no processing occurs. Field name resolution is not relative. It must not be affected by the base URI. ### Field name resolution example Given the following schema: ``` - $include: field_name_schema.yml - | ``` Process the following example: ``` - $include: field_name_src.yml - | ``` This becomes: ``` - $include: field_name_proc.yml - | ``` cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res.yml0000644000175000017500000000142313036412053032504 0ustar michaelmichael00000000000000- | ## Vocabulary resolution The schema may designate one or more vocabulary fields which use terms defined in the vocabulary. Processing must resolve vocabulary fields to either vocabulary terms or absolute URIs by first applying the link resolution rules defined above, then applying the following additional rule: * If a reference URI is a vocabulary field, and there is a vocabulary term which maps to the resolved URI, the reference must be replace with the vocabulary term. ### Vocabulary resolution example Given the following schema: ``` - $include: vocab_res_schema.yml - | ``` Process the following example: ``` - $include: vocab_res_src.yml - | ``` This becomes: ``` - $include: vocab_res_proc.yml - | ``` ././@LongLink0000000000000000000000000000015100000000000011212 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/metaschema_base.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/metaschema_base0000644000175000017500000000702413036412053033045 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/salad#" $namespaces: sld: "https://w3id.org/cwl/salad#" dct: "http://purl.org/dc/terms/" rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" rdfs: "http://www.w3.org/2000/01/rdf-schema#" xsd: "http://www.w3.org/2001/XMLSchema#" $graph: - name: PrimitiveType type: enum symbols: - "sld:null" - "xsd:boolean" - "xsd:int" - "xsd:long" - "xsd:float" - "xsd:double" - "xsd:string" doc: - | Salad data types are based on Avro schema declarations. Refer to the [Avro schema declaration documentation](https://avro.apache.org/docs/current/spec.html#schemas) for detailed information. - "null: no value" - "boolean: a binary value" - "int: 32-bit signed integer" - "long: 64-bit signed integer" - "float: single precision (32-bit) IEEE 754 floating-point number" - "double: double precision (64-bit) IEEE 754 floating-point number" - "string: Unicode character sequence" - name: Any type: enum symbols: ["#Any"] doc: | The **Any** type validates for any non-null value. - name: RecordField type: record doc: A field of a record. fields: - name: name type: string jsonldPredicate: "@id" doc: | The name of the field - name: doc type: string? doc: | A documentation string for this field jsonldPredicate: "rdfs:comment" - name: type type: - PrimitiveType - RecordSchema - EnumSchema - ArraySchema - string - type: array items: - PrimitiveType - RecordSchema - EnumSchema - ArraySchema - string jsonldPredicate: _id: sld:type _type: "@vocab" typeDSL: true refScope: 2 doc: | The field type - name: RecordSchema type: record fields: type: doc: "Must be `record`" type: name: Record_symbol type: enum symbols: - "sld:record" jsonldPredicate: _id: "sld:type" _type: "@vocab" typeDSL: true refScope: 2 fields: type: RecordField[]? jsonldPredicate: _id: sld:fields mapSubject: name mapPredicate: type doc: "Defines the fields of the record." - name: EnumSchema type: record doc: | Define an enumerated type. fields: type: doc: "Must be `enum`" type: name: Enum_symbol type: enum symbols: - "sld:enum" jsonldPredicate: _id: "sld:type" _type: "@vocab" typeDSL: true refScope: 2 symbols: type: string[] jsonldPredicate: _id: "sld:symbols" _type: "@id" identity: true doc: "Defines the set of valid symbols." - name: ArraySchema type: record fields: type: doc: "Must be `array`" type: name: Array_symbol type: enum symbols: - "sld:array" jsonldPredicate: _id: "sld:type" _type: "@vocab" typeDSL: true refScope: 2 items: type: - PrimitiveType - RecordSchema - EnumSchema - ArraySchema - string - type: array items: - PrimitiveType - RecordSchema - EnumSchema - ArraySchema - string jsonldPredicate: _id: "sld:items" _type: "@vocab" refScope: 2 doc: "Defines the type of the array elements." ././@LongLink0000000000000000000000000000015100000000000011212 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name_proc.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name_proc0000644000175000017500000000025313036412053033047 0ustar michaelmichael00000000000000 { "base": "one", "form": { "base": "two", "http://example.com/three": "three", }, "http://example.com/acid#four": "four" } ././@LongLink0000000000000000000000000000015200000000000011213 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res_schema.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res_schem0000644000175000017500000000035313036412053033075 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "ExampleType", "type": "record", "fields": [{ "name": "id", "type": "string", "jsonldPredicate": "@id" }] }] } ././@LongLink0000000000000000000000000000014700000000000011217 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/import_include.mdcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/import_include.0000644000175000017500000001076313036412053033043 0ustar michaelmichael00000000000000## Import During preprocessing traversal, an implementation must resolve `$import` directives. An `$import` directive is an object consisting of exactly one field `$import` specifying resource by URI string. It is an error if there are additional fields in the `$import` object, such additional fields must be ignored. The URI string must be resolved to an absolute URI using the link resolution rules described previously. Implementations must support loading from `file`, `http` and `https` resources. The URI referenced by `$import` must be loaded and recursively preprocessed as a Salad document. The external imported document does not inherit the context of the importing document, and the default base URI for processing the imported document must be the URI used to retrieve the imported document. If the `$import` URI includes a document fragment, the fragment must be excluded from the base URI used to preprocess the imported document. Once loaded and processed, the `$import` node is replaced in the document structure by the object or array yielded from the import operation. URIs may reference document fragments which refer to specific an object in the target document. This indicates that the `$import` node must be replaced by only the object with the appropriate fragment identifier. It is a fatal error if an import directive refers to an external resource or resource fragment which does not exist or is not accessible. ### Import example import.yml: ``` { "hello": "world" } ``` parent.yml: ``` { "form": { "bar": { "$import": "import.yml" } } } ``` This becomes: ``` { "form": { "bar": { "hello": "world" } } } ``` ## Include During preprocessing traversal, an implementation must resolve `$include` directives. An `$include` directive is an object consisting of exactly one field `$include` specifying a URI string. It is an error if there are additional fields in the `$include` object, such additional fields must be ignored. The URI string must be resolved to an absolute URI using the link resolution rules described previously. The URI referenced by `$include` must be loaded as a text data. Implementations must support loading from `file`, `http` and `https` resources. Implementations may transcode the character encoding of the text data to match that of the parent document, but must not interpret or parse the text document in any other way. Once loaded, the `$include` node is replaced in the document structure by a string containing the text data loaded from the resource. It is a fatal error if an import directive refers to an external resource which does not exist or is not accessible. ### Include example parent.yml: ``` { "form": { "bar": { "$include": "include.txt" } } } ``` include.txt: ``` hello world ``` This becomes: ``` { "form": { "bar": "hello world" } } ``` ## Mixin During preprocessing traversal, an implementation must resolve `$mixin` directives. An `$mixin` directive is an object consisting of the field `$mixin` specifying resource by URI string. If there are additional fields in the `$mixin` object, these fields override fields in the object which is loaded from the `$mixin` URI. The URI string must be resolved to an absolute URI using the link resolution rules described previously. Implementations must support loading from `file`, `http` and `https` resources. The URI referenced by `$mixin` must be loaded and recursively preprocessed as a Salad document. The external imported document must inherit the context of the importing document, however the file URI for processing the imported document must be the URI used to retrieve the imported document. The `$mixin` URI must not include a document fragment. Once loaded and processed, the `$mixin` node is replaced in the document structure by the object or array yielded from the import operation. URIs may reference document fragments which refer to specific an object in the target document. This indicates that the `$mixin` node must be replaced by only the object with the appropriate fragment identifier. It is a fatal error if an import directive refers to an external resource or resource fragment which does not exist or is not accessible. ### Mixin example mixin.yml: ``` { "hello": "world", "carrot": "orange" } ``` parent.yml: ``` { "form": { "bar": { "$mixin": "mixin.yml" "carrot": "cake" } } } ``` This becomes: ``` { "form": { "bar": { "hello": "world", "carrot": "cake" } } } ``` ././@LongLink0000000000000000000000000000014700000000000011217 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res_src.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res_src.y0000644000175000017500000000041313036412053033020 0ustar michaelmichael00000000000000 { "form": { "things": [ { "voc": "red", }, { "voc": "http://example.com/acid#red", }, { "voc": "http://example.com/acid#blue", } ] } } ././@LongLink0000000000000000000000000000015100000000000011212 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res_schema.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res_schema0000644000175000017500000000041013036412053033062 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "ExampleType", "type": "record", "fields": [{ "name": "link", "type": "string", "jsonldPredicate": { "_type": "@id" } }] }] } cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/salad.md0000644000175000017500000002435313036412053031433 0ustar michaelmichael00000000000000# Semantic Annotations for Linked Avro Data (SALAD) Author: * Peter Amstutz , Curoverse Contributors: * The developers of Apache Avro * The developers of JSON-LD * Nebojša Tijanić , Seven Bridges Genomics # Abstract Salad is a schema language for describing structured linked data documents in JSON or YAML documents. A Salad schema provides rules for preprocessing, structural validation, and link checking for documents described by a Salad schema. Salad builds on JSON-LD and the Apache Avro data serialization system, and extends Avro with features for rich data modeling such as inheritance, template specialization, object identifiers, and object references. Salad was developed to provide a bridge between the record oriented data modeling supported by Apache Avro and the Semantic Web. # Status of This Document This document is the product of the [Common Workflow Language working group](https://groups.google.com/forum/#!forum/common-workflow-language). The latest version of this document is available in the "schema_salad" directory at https://github.com/common-workflow-language/schema_salad The products of the CWL working group (including this document) are made available under the terms of the Apache License, version 2.0. # Introduction The JSON data model is an extremely popular way to represent structured data. It is attractive because of it's relative simplicity and is a natural fit with the standard types of many programming languages. However, this simplicity means that basic JSON lacks expressive features useful for working with complex data structures and document formats, such as schemas, object references, and namespaces. JSON-LD is a W3C standard providing a way to describe how to interpret a JSON document as Linked Data by means of a "context". JSON-LD provides a powerful solution for representing object references and namespaces in JSON based on standard web URIs, but is not itself a schema language. Without a schema providing a well defined structure, it is difficult to process an arbitrary JSON-LD document as idiomatic JSON because there are many ways to express the same data that are logically equivalent but structurally distinct. Several schema languages exist for describing and validating JSON data, such as the Apache Avro data serialization system, however none understand linked data. As a result, to fully take advantage of JSON-LD to build the next generation of linked data applications, one must maintain separate JSON schema, JSON-LD context, RDF schema, and human documentation, despite significant overlap of content and obvious need for these documents to stay synchronized. Schema Salad is designed to address this gap. It provides a schema language and processing rules for describing structured JSON content permitting URI resolution and strict document validation. The schema language supports linked data through annotations that describe the linked data interpretation of the content, enables generation of JSON-LD context and RDF schema, and production of RDF triples by applying the JSON-LD context. The schema language also provides for robust support of inline documentation. ## Introduction to draft 1 This is the first version of Schema Salad. It is developed concurrently with draft 3 of the Common Workflow Language for use in specifying the Common Workflow Language, however Schema Salad is intended to be useful to a broader audience. ## References to Other Specifications **Javascript Object Notation (JSON)**: http://json.org **JSON Linked Data (JSON-LD)**: http://json-ld.org **YAML**: http://yaml.org **Avro**: https://avro.apache.org/docs/current/spec.html **Uniform Resource Identifier (URI) Generic Syntax**: https://tools.ietf.org/html/rfc3986) **Resource Description Framework (RDF)**: http://www.w3.org/RDF/ **UTF-8**: https://www.ietf.org/rfc/rfc2279.txt) ## Scope This document describes the syntax, data model, algorithms, and schema language for working with Salad documents. It is not intended to document a specific implementation of Salad, however it may serve as a reference for the behavior of conforming implementations. ## Terminology The terminology used to describe Salad documents is defined in the Concepts section of the specification. The terms defined in the following list are used in building those definitions and in describing the actions of an Salad implementation: **may**: Conforming Salad documents and Salad implementations are permitted but not required to be interpreted as described. **must**: Conforming Salad documents and Salad implementations are required to be interpreted as described; otherwise they are in error. **error**: A violation of the rules of this specification; results are undefined. Conforming implementations may detect and report an error and may recover from it. **fatal error**: A violation of the rules of this specification; results are undefined. Conforming implementations must not continue to process the document and may report an error. **at user option**: Conforming software may or must (depending on the modal verb in the sentence) behave as described; if it does, it must provide users a means to enable or disable the behavior described. # Document model ## Data concepts An **object** is a data structure equivalent to the "object" type in JSON, consisting of a unordered set of name/value pairs (referred to here as **fields**) and where the name is a string and the value is a string, number, boolean, array, or object. A **document** is a file containing a serialized object, or an array of objects. A **document type** is a class of files that share a common structure and semantics. A **document schema** is a formal description of the grammar of a document type. A **base URI** is a context-dependent URI used to resolve relative references. An **identifier** is a URI that designates a single document or single object within a document. A **vocabulary** is the set of symbolic field names and enumerated symbols defined by a document schema, where each term maps to absolute URI. ## Syntax Conforming Salad documents are serialized and loaded using YAML syntax and UTF-8 text encoding. Salad documents are written using the JSON-compatible subset of YAML. Features of YAML such as headers and type tags that are not found in the standard JSON data model must not be used in conforming Salad documents. It is a fatal error if the document is not valid YAML. A Salad document must consist only of either a single root object or an array of objects. ## Document context ### Implied context The implicit context consists of the vocabulary defined by the schema and the base URI. By default, the base URI must be the URI that was used to load the document. It may be overridden by an explicit context. ### Explicit context If a document consists of a root object, this object may contain the fields `$base`, `$namespaces`, `$schemas`, and `$graph`: * `$base`: Must be a string. Set the base URI for the document used to resolve relative references. * `$namespaces`: Must be an object with strings as values. The keys of the object are namespace prefixes used in the document; the values of the object are the prefix expansions. * `$schemas`: Must be an array of strings. This field may list URI references to documents in RDF-XML format which will be queried for RDF schema data. The subjects and predicates described by the RDF schema may provide additional semantic context for the document, and may be used for validation of prefixed extension fields found in the document. Other directives beginning with `$` must be ignored. ## Document graph If a document consists of a single root object, this object may contain the field `$graph`. This field must be an array of objects. If present, this field holds the primary content of the document. A document that consists of array of objects at the root is an implicit graph. ## Document metadata If a document consists of a single root object, metadata about the document, such as authorship, may be declared in the root object. ## Document schema Document preprocessing, link validation and schema validation require a document schema. A schema may consist of: * At least one record definition object which defines valid fields that make up a record type. Record field definitions include the valid types that may be assigned to each field and annotations to indicate fields that represent identifiers and links, described below in "Semantic Annotations". * Any number of enumerated type objects which define a set of finite set of symbols that are valid value of the type. * Any number of documentation objects which allow in-line documentation of the schema. The schema for defining a salad schema (the metaschema) is described in detail in "Schema validation". ### Record field annotations In a document schema, record field definitions may include the field `jsonldPredicate`, which may be either a string or object. Implementations must use the following document preprocessing of fields by the following rules: * If the value of `jsonldPredicate` is `@id`, the field is an identifier field. * If the value of `jsonldPredicate` is an object, and contains that object contains the field `_type` with the value `@id`, the field is a link field. * If the value of `jsonldPredicate` is an object, and contains that object contains the field `_type` with the value `@vocab`, the field is a vocabulary field, which is a subtype of link field. ## Document traversal To perform document document preprocessing, link validation and schema validation, the document must be traversed starting from the fields or array items of the root object or array and recursively visiting each child item which contains an object or arrays. # Document preprocessing After processing the explicit context (if any), document preprocessing begins. Starting from the document root, object fields values or array items which contain objects or arrays are recursively traversed depth-first. For each visited object, field names, identifier fields, link fields, vocabulary fields, and `$import` and `$include` directives must be processed as described in this section. The order of traversal of child nodes within a parent node is undefined. cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/metaschema.yml0000644000175000017500000002271713036412053032661 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/salad#" $namespaces: sld: "https://w3id.org/cwl/salad#" dct: "http://purl.org/dc/terms/" rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" rdfs: "http://www.w3.org/2000/01/rdf-schema#" xsd: "http://www.w3.org/2001/XMLSchema#" $graph: - name: "Semantic_Annotations_for_Linked_Avro_Data" type: documentation doc: - $include: salad.md - $import: field_name.yml - $import: ident_res.yml - $import: link_res.yml - $import: vocab_res.yml - $include: import_include.md - name: "Link_Validation" type: documentation doc: | # Link validation Once a document has been preprocessed, an implementation may validate links. The link validation traversal may visit fields which the schema designates as link fields and check that each URI references an existing object in the current document, an imported document, file system, or network resource. Failure to validate links may be a fatal error. Link validation behavior for individual fields may be modified by `identity` and `noLinkCheck` in the `jsonldPredicate` section of the field schema. - name: "Schema_validation" type: documentation doc: "" # - name: "JSON_LD_Context" # type: documentation # doc: | # # Generating JSON-LD Context # How to generate the json-ld context... - $import: metaschema_base.yml - name: JsonldPredicate type: record doc: | Attached to a record field to define how the parent record field is handled for URI resolution and JSON-LD context generation. fields: - name: _id type: string? jsonldPredicate: _id: sld:_id _type: "@id" identity: true doc: | The predicate URI that this field corresponds to. Corresponds to JSON-LD `@id` directive. - name: _type type: string? doc: | The context type hint, corresponds to JSON-LD `@type` directive. * If the value of this field is `@id` and `identity` is false or unspecified, the parent field must be resolved using the link resolution rules. If `identity` is true, the parent field must be resolved using the identifier expansion rules. * If the value of this field is `@vocab`, the parent field must be resolved using the vocabulary resolution rules. - name: _container type: string? doc: | Structure hint, corresponds to JSON-LD `@container` directive. - name: identity type: boolean? doc: | If true and `_type` is `@id` this indicates that the parent field must be resolved according to identity resolution rules instead of link resolution rules. In addition, the field value is considered an assertion that the linked value exists; absence of an object in the loaded document with the URI is not an error. - name: noLinkCheck type: boolean? doc: | If true, this indicates that link validation traversal must stop at this field. This field (it is is a URI) or any fields under it (if it is an object or array) are not subject to link checking. - name: mapSubject type: string? doc: | If the value of the field is a JSON object, it must be transformed into an array of JSON objects, where each key-value pair from the source JSON object is a list item, the list items must be JSON objects, and the key is assigned to the field specified by `mapSubject`. - name: mapPredicate type: string? doc: | Only applies if `mapSubject` is also provided. If the value of the field is a JSON object, it is transformed as described in `mapSubject`, with the addition that when the value of a map item is not an object, the item is transformed to a JSON object with the key assigned to the field specified by `mapSubject` and the value assigned to the field specified by `mapPredicate`. - name: refScope type: int? doc: | If the field contains a relative reference, it must be resolved by searching for valid document references in each successive parent scope in the document fragment. For example, a reference of `foo` in the context `#foo/bar/baz` will first check for the existence of `#foo/bar/baz/foo`, followed by `#foo/bar/foo`, then `#foo/foo` and then finally `#foo`. The first valid URI in the search order shall be used as the fully resolved value of the identifier. The value of the refScope field is the specified number of levels from the containing identifer scope before starting the search, so if `refScope: 2` then "baz" and "bar" must be stripped to get the base `#foo` and search `#foo/foo` and the `#foo`. The last scope searched must be the top level scope before determining if the identifier cannot be resolved. - name: typeDSL type: boolean? doc: | Field must be expanded based on the the Schema Salad type DSL. - name: SpecializeDef type: record fields: - name: specializeFrom type: string doc: "The data type to be replaced" jsonldPredicate: _id: "sld:specializeFrom" _type: "@id" refScope: 1 - name: specializeTo type: string doc: "The new data type to replace with" jsonldPredicate: _id: "sld:specializeTo" _type: "@id" refScope: 1 - name: NamedType type: record abstract: true fields: - name: name type: string jsonldPredicate: "@id" doc: "The identifier for this type" - name: DocType type: record abstract: true fields: - name: doc type: - string? - string[]? doc: "A documentation string for this type, or an array of strings which should be concatenated." jsonldPredicate: "rdfs:comment" - name: docParent type: string? doc: | Hint to indicate that during documentation generation, documentation for this type should appear in a subsection under `docParent`. jsonldPredicate: _id: "sld:docParent" _type: "@id" - name: docChild type: - string? - string[]? doc: | Hint to indicate that during documentation generation, documentation for `docChild` should appear in a subsection under this type. jsonldPredicate: _id: "sld:docChild" _type: "@id" - name: docAfter type: string? doc: | Hint to indicate that during documentation generation, documentation for this type should appear after the `docAfter` section at the same level. jsonldPredicate: _id: "sld:docAfter" _type: "@id" - name: SchemaDefinedType type: record extends: DocType doc: | Abstract base for schema-defined types. abstract: true fields: - name: jsonldPredicate type: - string? - JsonldPredicate? doc: | Annotate this type with linked data context. jsonldPredicate: sld:jsonldPredicate - name: documentRoot type: boolean? doc: | If true, indicates that the type is a valid at the document root. At least one type in a schema must be tagged with `documentRoot: true`. - name: SaladRecordField type: record extends: RecordField doc: "A field of a record." fields: - name: jsonldPredicate type: - string? - JsonldPredicate? doc: | Annotate this type with linked data context. jsonldPredicate: "sld:jsonldPredicate" - name: SaladRecordSchema type: record extends: [NamedType, RecordSchema, SchemaDefinedType] documentRoot: true specialize: RecordField: SaladRecordField fields: - name: abstract type: boolean? doc: | If true, this record is abstract and may be used as a base for other records, but is not valid on its own. - name: extends type: - string? - string[]? jsonldPredicate: _id: "sld:extends" _type: "@id" refScope: 1 doc: | Indicates that this record inherits fields from one or more base records. - name: specialize type: - SpecializeDef[]? doc: | Only applies if `extends` is declared. Apply type specialization using the base record as a template. For each field inherited from the base record, replace any instance of the type `specializeFrom` with `specializeTo`. jsonldPredicate: _id: "sld:specialize" mapSubject: specializeFrom mapPredicate: specializeTo - name: SaladEnumSchema type: record extends: [EnumSchema, SchemaDefinedType] documentRoot: true doc: | Define an enumerated type. fields: - name: extends type: - string? - string[]? jsonldPredicate: _id: "sld:extends" _type: "@id" refScope: 1 doc: | Indicates that this enum inherits symbols from a base enum. - name: Documentation type: record extends: [NamedType, DocType] documentRoot: true doc: | A documentation section. This type exists to facilitate self-documenting schemas but has no role in formal validation. fields: - name: type doc: "Must be `documentation`" type: name: Documentation_symbol type: enum symbols: - "sld:documentation" jsonldPredicate: _id: "sld:type" _type: "@vocab" typeDSL: true refScope: 2 cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res.yml0000644000175000017500000000341613036412053032353 0ustar michaelmichael00000000000000- | ## Link resolution The schema may designate one or more fields as link fields reference other objects. Processing must resolve links to either absolute URIs using the following rules: * If a reference URI is prefixed with `#` it is a relative fragment identifier. It is resolved relative to the base URI by setting or replacing the fragment portion of the base URI. * If a reference URI does not contain a scheme and is not prefixed with `#` it is a path relative reference. If the reference URI contains `#` in any position other than the first character, the reference URI must be divided into a path portion and a fragment portion split on the first instance of `#`. The path portion is resolved relative to the base URI by the following rule: if the path portion of the base URI ends in a slash `/`, append the path portion of the reference URI to the path portion of the base URI. If the path portion of the base URI does not end in a slash, replace the final path segment with the path portion of the reference URI. Replace the fragment portion of the base URI with the fragment portion of the reference URI. * If a reference URI begins with a namespace prefix declared in `$namespaces` followed by a colon `:`, the prefix and colon must be replaced by the namespace declared in `$namespaces`. * If a reference URI is an absolute URI consisting of a scheme and path, no processing occurs. Link resolution must not affect the base URI used to resolve identifiers and other links. ### Link resolution example Given the following schema: ``` - $include: link_res_schema.yml - | ``` Process the following example: ``` - $include: link_res_src.yml - | ``` This becomes: ``` - $include: link_res_proc.yml - | ``` ././@LongLink0000000000000000000000000000015000000000000011211 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res_proc.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res_proc.0000644000175000017500000000056213036412053033021 0ustar michaelmichael00000000000000{ "id": "http://example.com/base", "form": { "id": "http://example.com/base#one", "things": [ { "id": "http://example.com/base#one/two" }, { "id": "http://example.com/base#three" }, { "id": "http://example.com/four#five", }, { "id": "http://example.com/acid#six", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/Process.yml0000644000175000017500000006146613036412053024375 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" sld: "https://w3id.org/cwl/salad#" $graph: - name: "Common Workflow Language, v1.1.0-dev1" type: documentation doc: {$include: concepts.md} - $import: "salad/schema_salad/metaschema/metaschema_base.yml" - name: BaseTypesDoc type: documentation doc: | ## Base types docChild: - "#CWLType" - "#Process" - type: enum name: CWLVersion doc: "Version symbols for published CWL document versions." symbols: - cwl:draft-2 - cwl:draft-3.dev1 - cwl:draft-3.dev2 - cwl:draft-3.dev3 - cwl:draft-3.dev4 - cwl:draft-3.dev5 - cwl:draft-3 - cwl:draft-4.dev1 - cwl:draft-4.dev2 - cwl:draft-4.dev3 - cwl:v1.0.dev4 - cwl:v1.0 - cwl:v1.1.0-dev1 # a dash is required by the semver 2.0 rules - name: CWLType type: enum extends: "sld:PrimitiveType" symbols: - cwl:File - cwl:Directory doc: - "Extends primitive types with the concept of a file and directory as a builtin type." - "File: A File object" - "Directory: A Directory object" - name: File type: record docParent: "#CWLType" doc: | Represents a file (or group of files if `secondaryFiles` is specified) that must be accessible by tools using standard POSIX file system call API such as open(2) and read(2). fields: - name: class type: type: enum name: File_class symbols: - cwl:File jsonldPredicate: _id: "@type" _type: "@vocab" doc: Must be `File` to indicate this object describes a file. - name: location type: string? doc: | An IRI that identifies the file resource. This may be a relative reference, in which case it must be resolved using the base IRI of the document. The location may refer to a local or remote resource; the implementation must use the IRI to retrieve file content. If an implementation is unable to retrieve the file content stored at a remote resource (due to unsupported protocol, access denied, or other issue) it must signal an error. If the `location` field is not provided, the `contents` field must be provided. The implementation must assign a unique identifier for the `location` field. If the `path` field is provided but the `location` field is not, an implementation may assign the value of the `path` field to `location`, then follow the rules above. jsonldPredicate: _id: "@id" _type: "@id" - name: path type: string? doc: | The local host path where the File is available when a CommandLineTool is executed. This field must be set by the implementation. The final path component must match the value of `basename`. This field must not be used in any other context. The command line tool being executed must be able to to access the file at `path` using the POSIX `open(2)` syscall. As a special case, if the `path` field is provided but the `location` field is not, an implementation may assign the value of the `path` field to `location`, and remove the `path` field. If the `path` contains [POSIX shell metacharacters](http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_02) (`|`,`&`, `;`, `<`, `>`, `(`,`)`, `$`,`` ` ``, `\`, `"`, `'`, ``, ``, and ``) or characters [not allowed](http://www.iana.org/assignments/idna-tables-6.3.0/idna-tables-6.3.0.xhtml) for [Internationalized Domain Names for Applications](https://tools.ietf.org/html/rfc6452) then implementations may terminate the process with a `permanentFailure`. jsonldPredicate: "_id": "cwl:path" "_type": "@id" - name: basename type: string? doc: | The base name of the file, that is, the name of the file without any leading directory path. The base name must not contain a slash `/`. If not provided, the implementation must set this field based on the `location` field by taking the final path component after parsing `location` as an IRI. If `basename` is provided, it is not required to match the value from `location`. When this file is made available to a CommandLineTool, it must be named with `basename`, i.e. the final component of the `path` field must match `basename`. jsonldPredicate: "cwl:basename" - name: dirname type: string? doc: | The name of the directory containing file, that is, the path leading up to the final slash in the path such that `dirname + '/' + basename == path`. The implementation must set this field based on the value of `path` prior to evaluating parameter references or expressions in a CommandLineTool document. This field must not be used in any other context. - name: nameroot type: string? doc: | The basename root such that `nameroot + nameext == basename`, and `nameext` is empty or begins with a period and contains at most one period. For the purposess of path splitting leading periods on the basename are ignored; a basename of `.cshrc` will have a nameroot of `.cshrc`. The implementation must set this field automatically based on the value of `basename` prior to evaluating parameter references or expressions. - name: nameext type: string? doc: | The basename extension such that `nameroot + nameext == basename`, and `nameext` is empty or begins with a period and contains at most one period. Leading periods on the basename are ignored; a basename of `.cshrc` will have an empty `nameext`. The implementation must set this field automatically based on the value of `basename` prior to evaluating parameter references or expressions. - name: checksum type: string? doc: | Optional hash code for validating file integrity. Currently must be in the form "sha1$ + hexadecimal string" using the SHA-1 algorithm. - name: size type: long? doc: Optional file size - name: "secondaryFiles" type: - "null" - type: array items: [File, Directory] jsonldPredicate: "cwl:secondaryFiles" doc: | A list of additional files that are associated with the primary file and must be transferred alongside the primary file. Examples include indexes of the primary file, or external references which must be included when loading primary document. A file object listed in `secondaryFiles` may itself include `secondaryFiles` for which the same rules apply. - name: format type: string? jsonldPredicate: _id: cwl:format _type: "@id" identity: true doc: | The format of the file: this must be an IRI of a concept node that represents the file format, preferrably defined within an ontology. If no ontology is available, file formats may be tested by exact match. Reasoning about format compatability must be done by checking that an input file format is the same, `owl:equivalentClass` or `rdfs:subClassOf` the format required by the input parameter. `owl:equivalentClass` is transitive with `rdfs:subClassOf`, e.g. if ` owl:equivalentClass ` and ` owl:subclassOf ` then infer ` owl:subclassOf `. File format ontologies may be provided in the "$schema" metadata at the root of the document. If no ontologies are specified in `$schema`, the runtime may perform exact file format matches. - name: contents type: string? doc: | File contents literal. Maximum of 64 KiB. If neither `location` nor `path` is provided, `contents` must be non-null. The implementation must assign a unique identifier for the `location` field. When the file is staged as input to CommandLineTool, the value of `contents` must be written to a file. If `loadContents` of `inputBinding` or `outputBinding` is true and `location` is valid, the implementation must read up to the first 64 KiB of text from the file and place it in the "contents" field. - name: Directory type: record docAfter: "#File" doc: | Represents a directory to present to a command line tool. fields: - name: class type: type: enum name: Directory_class symbols: - cwl:Directory jsonldPredicate: _id: "@type" _type: "@vocab" doc: Must be `Directory` to indicate this object describes a Directory. - name: location type: string? doc: | An IRI that identifies the directory resource. This may be a relative reference, in which case it must be resolved using the base IRI of the document. The location may refer to a local or remote resource. If the `listing` field is not set, the implementation must use the location IRI to retrieve directory listing. If an implementation is unable to retrieve the directory listing stored at a remote resource (due to unsupported protocol, access denied, or other issue) it must signal an error. If the `location` field is not provided, the `listing` field must be provided. The implementation must assign a unique identifier for the `location` field. If the `path` field is provided but the `location` field is not, an implementation may assign the value of the `path` field to `location`, then follow the rules above. jsonldPredicate: _id: "@id" _type: "@id" - name: path type: string? doc: | The local path where the Directory is made available prior to executing a CommandLineTool. This must be set by the implementation. This field must not be used in any other context. The command line tool being executed must be able to to access the directory at `path` using the POSIX `opendir(2)` syscall. If the `path` contains [POSIX shell metacharacters](http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_02) (`|`,`&`, `;`, `<`, `>`, `(`,`)`, `$`,`` ` ``, `\`, `"`, `'`, ``, ``, and ``) or characters [not allowed](http://www.iana.org/assignments/idna-tables-6.3.0/idna-tables-6.3.0.xhtml) for [Internationalized Domain Names for Applications](https://tools.ietf.org/html/rfc6452) then implementations may terminate the process with a `permanentFailure`. jsonldPredicate: _id: "cwl:path" _type: "@id" - name: basename type: string? doc: | The base name of the directory, that is, the name of the file without any leading directory path. The base name must not contain a slash `/`. If not provided, the implementation must set this field based on the `location` field by taking the final path component after parsing `location` as an IRI. If `basename` is provided, it is not required to match the value from `location`. When this file is made available to a CommandLineTool, it must be named with `basename`, i.e. the final component of the `path` field must match `basename`. jsonldPredicate: "cwl:basename" - name: listing type: - "null" - type: array items: [File, Directory] doc: | List of files or subdirectories contained in this directory. The name of each file or subdirectory is determined by the `basename` field of each `File` or `Directory` object. It is an error if a `File` shares a `basename` with any other entry in `listing`. If two or more `Directory` object share the same `basename`, this must be treated as equivalent to a single subdirectory with the listings recursively merged. jsonldPredicate: _id: "cwl:listing" - name: SchemaBase type: record abstract: true fields: - name: label type: - "null" - string jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this object." - name: Parameter type: record extends: SchemaBase abstract: true doc: | Define an input or output parameter to a process. fields: - name: secondaryFiles type: - "null" - string - Expression - type: array items: [string, Expression] jsonldPredicate: "cwl:secondaryFiles" doc: | Only valid when `type: File` or is an array of `items: File`. Describes files that must be included alongside the primary file(s). If the value is an expression, the value of `self` in the expression must be the primary input or output File to which this binding applies. If the value is a string, it specifies that the following pattern should be applied to the primary file: 1. If string begins with one or more caret `^` characters, for each caret, remove the last file extension from the path (the last period `.` and all following characters). If there are no file extensions, the path is unchanged. 2. Append the remainder of the string to the end of the file path. - name: format type: - "null" - string - type: array items: string - Expression jsonldPredicate: _id: cwl:format _type: "@id" identity: true doc: | Only valid when `type: File` or is an array of `items: File`. For input parameters, this must be one or more IRIs of concept nodes that represents file formats which are allowed as input to this parameter, preferrably defined within an ontology. If no ontology is available, file formats may be tested by exact match. For output parameters, this is the file format that will be assigned to the output parameter. - name: streamable type: boolean? doc: | Only valid when `type: File` or is an array of `items: File`. A value of `true` indicates that the file is read or written sequentially without seeking. An implementation may use this flag to indicate whether it is valid to stream file contents using a named pipe. Default: `false`. - name: doc type: - string? - string[]? doc: "A documentation string for this type, or an array of strings which should be concatenated." jsonldPredicate: "rdfs:comment" - type: enum name: Expression doc: | 'Expression' is not a real type. It indicates that a field must allow runtime parameter references. If [InlineJavascriptRequirement](#InlineJavascriptRequirement) is declared and supported by the platform, the field must also allow Javascript expressions. symbols: - cwl:ExpressionPlaceholder - name: InputBinding type: record abstract: true fields: - name: loadContents type: - "null" - boolean jsonldPredicate: "cwl:loadContents" doc: | Only valid when `type: File` or is an array of `items: File`. Read up to the first 64 KiB of text from the file and place it in the "contents" field of the file object for use by expressions. - name: OutputBinding type: record abstract: true - name: InputSchema extends: SchemaBase type: record abstract: true - name: OutputSchema extends: SchemaBase type: record abstract: true - name: InputRecordField type: record extends: "sld:RecordField" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: InputRecordSchema - specializeFrom: "sld:EnumSchema" specializeTo: InputEnumSchema - specializeFrom: "sld:ArraySchema" specializeTo: InputArraySchema - specializeFrom: "sld:PrimitiveType" specializeTo: CWLType fields: - name: inputBinding type: InputBinding? jsonldPredicate: "cwl:inputBinding" - name: label type: string? jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: InputRecordSchema type: record extends: ["sld:RecordSchema", InputSchema] specialize: - specializeFrom: "sld:RecordField" specializeTo: InputRecordField - name: InputEnumSchema type: record extends: ["sld:EnumSchema", InputSchema] fields: - name: inputBinding type: InputBinding? jsonldPredicate: "cwl:inputBinding" - name: InputArraySchema type: record extends: ["sld:ArraySchema", InputSchema] specialize: - specializeFrom: "sld:RecordSchema" specializeTo: InputRecordSchema - specializeFrom: "sld:EnumSchema" specializeTo: InputEnumSchema - specializeFrom: "sld:ArraySchema" specializeTo: InputArraySchema - specializeFrom: "sld:PrimitiveType" specializeTo: CWLType fields: - name: inputBinding type: InputBinding? jsonldPredicate: "cwl:inputBinding" - name: OutputRecordField type: record extends: "sld:RecordField" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: OutputRecordSchema - specializeFrom: "sld:EnumSchema" specializeTo: OutputEnumSchema - specializeFrom: "sld:ArraySchema" specializeTo: OutputArraySchema - specializeFrom: "sld:PrimitiveType" specializeTo: CWLType fields: - name: outputBinding type: OutputBinding? jsonldPredicate: "cwl:outputBinding" - name: OutputRecordSchema type: record extends: ["sld:RecordSchema", "#OutputSchema"] docParent: "#OutputParameter" specialize: - specializeFrom: "sld:RecordField" specializeTo: OutputRecordField - name: OutputEnumSchema type: record extends: ["sld:EnumSchema", OutputSchema] docParent: "#OutputParameter" fields: - name: outputBinding type: OutputBinding? jsonldPredicate: "cwl:outputBinding" - name: OutputArraySchema type: record extends: ["sld:ArraySchema", OutputSchema] docParent: "#OutputParameter" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: OutputRecordSchema - specializeFrom: "sld:EnumSchema" specializeTo: OutputEnumSchema - specializeFrom: "sld:ArraySchema" specializeTo: OutputArraySchema - specializeFrom: "sld:PrimitiveType" specializeTo: CWLType fields: - name: outputBinding type: OutputBinding? jsonldPredicate: "cwl:outputBinding" - name: InputParameter type: record extends: Parameter fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this parameter object." - name: inputBinding type: InputBinding? jsonldPredicate: "cwl:inputBinding" doc: | Describes how to handle the inputs of a process and convert them into a concrete form for execution, such as command line parameters. - name: default type: Any? jsonldPredicate: "cwl:default" doc: | The default value for this parameter if not provided in the input object. - name: RegularInputParameter type: record extends: InputParameter fields: - name: type type: - "null" - CWLType - InputRecordSchema - InputEnumSchema - InputArraySchema - string - type: array items: - CWLType - InputRecordSchema - InputEnumSchema - InputArraySchema - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" refScope: 2 typeDSL: True doc: | Specify valid types of data that may be assigned to this parameter. - name: OutputParameter type: record extends: Parameter fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this parameter object." - name: outputBinding type: OutputBinding? jsonldPredicate: "cwl:outputBinding" doc: | Describes how to handle the outputs of a process. - type: record name: ProcessRequirement abstract: true doc: | A process requirement declares a prerequisite that may or must be fulfilled before executing a process. See [`Process.hints`](#process) and [`Process.requirements`](#process). Process requirements are the primary mechanism for specifying extensions to the CWL core specification. - type: record name: Process abstract: true doc: | The base executable type in CWL is the `Process` object defined by the document. Note that the `Process` object is abstract and cannot be directly executed. fields: - name: id type: string? jsonldPredicate: "@id" doc: "The unique identifier for this process object." - name: inputs type: type: array items: InputParameter jsonldPredicate: _id: "cwl:inputs" mapSubject: id mapPredicate: type doc: | Defines the input parameters of the process. The process is ready to run when all required input parameters are associated with concrete values. Input parameters include a schema for each parameter which is used to validate the input object. It may also be used to build a user interface for constructing the input object. - name: outputs type: type: array items: OutputParameter jsonldPredicate: _id: "cwl:outputs" mapSubject: id mapPredicate: type doc: | Defines the parameters representing the output of the process. May be used to generate and/or validate the output object. - name: requirements type: ProcessRequirement[]? jsonldPredicate: _id: "cwl:requirements" mapSubject: class doc: | Declares requirements that apply to either the runtime environment or the workflow engine that must be met in order to execute this process. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. - name: hints type: Any[]? doc: | Declares hints applying to either the runtime environment or the workflow engine that may be helpful in executing this process. It is not an error if an implementation cannot satisfy all hints, however the implementation may report a warning. jsonldPredicate: _id: cwl:hints noLinkCheck: true mapSubject: class - name: label type: string? jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: doc type: string? jsonldPredicate: "rdfs:comment" doc: "A long, human-readable description of this process object." - name: cwlVersion type: CWLVersion? doc: | CWL document version. Always required at the document root. Not required for a Process embedded inside another Process. jsonldPredicate: "_id": "cwl:cwlVersion" "_type": "@vocab" - name: InlineJavascriptRequirement type: record extends: ProcessRequirement doc: | Indicates that the workflow platform must support inline Javascript expressions. If this requirement is not present, the workflow platform must not perform expression interpolatation. fields: - name: class type: string doc: "Always 'InlineJavascriptRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: expressionLib type: string[]? doc: | Additional code fragments that will also be inserted before executing the expression code. Allows for function definitions that may be called from CWL expressions. - name: SchemaDefRequirement type: record extends: ProcessRequirement doc: | This field consists of an array of type definitions which must be used when interpreting the `inputs` and `outputs` fields. When a `type` field contain a IRI, the implementation must check if the type is defined in `schemaDefs` and use that definition. If the type is not found in `schemaDefs`, it is an error. The entries in `schemaDefs` must be processed in the order listed such that later schema definitions may refer to earlier schema definitions. fields: - name: class type: string doc: "Always 'SchemaDefRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: types type: type: array items: InputSchema doc: The list of type definitions. cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/concepts.md0000644000175000017500000004332713036412053024370 0ustar michaelmichael00000000000000## References to other specifications **Javascript Object Notation (JSON)**: http://json.org **JSON Linked Data (JSON-LD)**: http://json-ld.org **YAML**: http://yaml.org **Avro**: https://avro.apache.org/docs/current/spec.html **Uniform Resource Identifier (URI) Generic Syntax**: https://tools.ietf.org/html/rfc3986) **Internationalized Resource Identifiers (IRIs)**: https://tools.ietf.org/html/rfc3987 **Portable Operating System Interface (POSIX.1-2008)**: http://pubs.opengroup.org/onlinepubs/9699919799/ **Resource Description Framework (RDF)**: http://www.w3.org/RDF/ ## Scope This document describes CWL syntax, execution, and object model. It is not intended to document a CWL specific implementation, however it may serve as a reference for the behavior of conforming implementations. ## Terminology The terminology used to describe CWL documents is defined in the Concepts section of the specification. The terms defined in the following list are used in building those definitions and in describing the actions of a CWL implementation: **may**: Conforming CWL documents and CWL implementations are permitted but not required to behave as described. **must**: Conforming CWL documents and CWL implementations are required to behave as described; otherwise they are in error. **error**: A violation of the rules of this specification; results are undefined. Conforming implementations may detect and report an error and may recover from it. **fatal error**: A violation of the rules of this specification; results are undefined. Conforming implementations must not continue to execute the current process and may report an error. **at user option**: Conforming software may or must (depending on the modal verb in the sentence) behave as described; if it does, it must provide users a means to enable or disable the behavior described. **deprecated**: Conforming software may implement a behavior for backwards compatibility. Portable CWL documents should not rely on deprecated behavior. Behavior marked as deprecated may be removed entirely from future revisions of the CWL specification. # Data model ## Data concepts An **object** is a data structure equivalent to the "object" type in JSON, consisting of a unordered set of name/value pairs (referred to here as **fields**) and where the name is a string and the value is a string, number, boolean, array, or object. A **document** is a file containing a serialized object, or an array of objects. A **process** is a basic unit of computation which accepts input data, performs some computation, and produces output data. Examples include CommandLineTools, Workflows, and ExpressionTools. An **input object** is an object describing the inputs to an invocation of a process. An **output object** is an object describing the output resulting from an invocation of a process. An **input schema** describes the valid format (required fields, data types) for an input object. An **output schema** describes the valid format for an output object. **Metadata** is information about workflows, tools, or input items. ## Syntax CWL documents must consist of an object or array of objects represented using JSON or YAML syntax. Upon loading, a CWL implementation must apply the preprocessing steps described in the [Semantic Annotations for Linked Avro Data (SALAD) Specification](SchemaSalad.html). An implementation may formally validate the structure of a CWL document using SALAD schemas located at https://github.com/common-workflow-language/common-workflow-language/tree/master/draft-4 ## Identifiers If an object contains an `id` field, that is used to uniquely identify the object in that document. The value of the `id` field must be unique over the entire document. Identifiers may be resolved relative to either the document base and/or other identifiers following the rules are described in the [Schema Salad specification](SchemaSalad.html#Identifier_resolution). An implementation may choose to only honor references to object types for which the `id` field is explicitly listed in this specification. ## Document preprocessing An implementation must resolve [$import](SchemaSalad.html#Import) and [$include](SchemaSalad.html#Import) directives as described in the [Schema Salad specification](SchemaSalad.html). Another transformation defined in Schema salad is simplification of data type definitions. Type `` ending with `?` should be transformed to `[, "null"]`. Type `` ending with `[]` should be transformed to `{"type": "array", "items": }` ## Extensions and metadata Input metadata (for example, a lab sample identifier) may be represented within a tool or workflow using input parameters which are explicitly propagated to output. Future versions of this specification may define additional facilities for working with input/output metadata. Implementation extensions not required for correct execution (for example, fields related to GUI presentation) and metadata about the tool or workflow itself (for example, authorship for use in citations) may be provided as additional fields on any object. Such extensions fields must use a namespace prefix listed in the `$namespaces` section of the document as described in the [Schema Salad specification](SchemaSalad.html#Explicit_context). Implementation extensions which modify execution semantics must be [listed in the `requirements` field](#Requirements_and_hints). # Execution model ## Execution concepts A **parameter** is a named symbolic input or output of process, with an associated datatype or schema. During execution, values are assigned to parameters to make the input object or output object used for concrete process invocation. A **CommandLineTool** is a process characterized by the execution of a standalone, non-interactive program which is invoked on some input, produces output, and then terminates. A **workflow** is a process characterized by multiple subprocess steps, where step outputs are connected to the inputs of downstream steps to form a directed acylic graph, and independent steps may run concurrently. A **runtime environment** is the actual hardware and software environment when executing a command line tool. It includes, but is not limited to, the hardware architecture, hardware resources, operating system, software runtime (if applicable, such as the specific Python interpreter or the specific Java virtual machine), libraries, modules, packages, utilities, and data files required to run the tool. A **workflow platform** is a specific hardware and software implementation capable of interpreting CWL documents and executing the processes specified by the document. The responsibilities of the workflow platform may include scheduling process invocation, setting up the necessary runtime environment, making input data available, invoking the tool process, and collecting output. A workflow platform may choose to only implement the Command Line Tool Description part of the CWL specification. It is intended that the workflow platform has broad leeway outside of this specification to optimize use of computing resources and enforce policies not covered by this specification. Some areas that are currently out of scope for CWL specification but may be handled by a specific workflow platform include: * Data security and permissions * Scheduling tool invocations on remote cluster or cloud compute nodes. * Using virtual machines or operating system containers to manage the runtime (except as described in [DockerRequirement](CommandLineTool.html#DockerRequirement)). * Using remote or distributed file systems to manage input and output files. * Transforming file paths. * Determining if a process has previously been executed, and if so skipping it and reusing previous results. * Pausing, resuming or checkpointing processes or workflows. Conforming CWL processes must not assume anything about the runtime environment or workflow platform unless explicitly declared though the use of [process requirements](#Requirements_and_hints). ## Generic execution process The generic execution sequence of a CWL process (including workflows and command line line tools) is as follows. 1. Load, process and validate a CWL document, yielding a process object. 2. Load input object. 3. Validate the input object against the `inputs` schema for the process. 4. Validate process requirements are met. 5. Perform any further setup required by the specific process type. 6. Execute the process. 7. Capture results of process execution into the output object. 8. Validate the output object against the `outputs` schema for the process. 9. Report the output object to the process caller. ## Requirements and hints A **process requirement** modifies the semantics or runtime environment of a process. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. A **hint** is similar to a requirement; however, it is not an error if an implementation cannot satisfy all hints. The implementation may report a warning if a hint cannot be satisfied. Requirements are inherited. A requirement specified in a Workflow applies to all workflow steps; a requirement specified on a workflow step will apply to the process implementation of that step and any of its substeps. If the same process requirement appears at different levels of the workflow, the most specific instance of the requirement is used, that is, an entry in `requirements` on a process implementation such as CommandLineTool will take precedence over an entry in `requirements` specified in a workflow step, and an entry in `requirements` on a workflow step takes precedence over the workflow. Entries in `hints` are resolved the same way. Requirements override hints. If a process implementation provides a process requirement in `hints` which is also provided in `requirements` by an enclosing workflow or workflow step, the enclosing `requirements` takes precedence. ## Parameter references Parameter references are denoted by the syntax `$(...)` and may be used in any field permitting the pseudo-type `Expression`, as specified by this document. Conforming implementations must support parameter references. Parameter references use the following subset of [Javascript/ECMAScript 5.1](http://www.ecma-international.org/ecma-262/5.1/) syntax, but they are designed to not require a Javascript engine for evaluation. In the following [BNF grammar](https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_Form), character classes, and grammar rules are denoted in '{}', '-' denotes exclusion from a character class, '(())' denotes grouping, '|' denotes alternates, trailing '*' denotes zero or more repeats, '+' denote one or more repeats, '/' escapes these special characters, and all other characters are literal values.

symbol:: {Unicode alphanumeric}+
singleq:: [' (( {character - '} | \' ))* ']
doubleq:: [" (( {character - "} | \" ))* "]
index:: [ {decimal digit}+ ]
segment:: . {symbol} | {singleq} | {doubleq} | {index}
parameter reference::$( {symbol} {segment}*)

Use the following algorithm to resolve a parameter reference: 1. Match the leading symbol as the key 2. Look up the key in the parameter context (described below) to get the current value. It is an error if the key is not found in the parameter context. 3. If there are no subsequent segments, terminate and return current value 4. Else, match the next segment 5. Extract the symbol, string, or index from the segment as the key 6. Look up the key in current value and assign as new current value. If the key is a symbol or string, the current value must be an object. If the key is an index, the current value must be an array or string. It is an error if the key does not match the required type, or the key is not found or out of range. 7. Repeat steps 3-6 The root namespace is the parameter context. The following parameters must be provided: * `inputs`: The input object to the current Process. * `self`: A context-specific value. The contextual values for 'self' are documented for specific fields elsewhere in this specification. If a contextual value of 'self' is not documented for a field, it must be 'null'. * `runtime`: An object containing configuration details. Specific to the process type. An implementation may provide opaque strings for any or all fields of `runtime`. These must be filled in by the platform after processing the Tool but before actual execution. Parameter references and expressions may only use the literal string value of the field and must not perform computation on the contents, except where noted otherwise. If the value of a field has no leading or trailing non-whitespace characters around a parameter reference, the effective value of the field becomes the value of the referenced parameter, preserving the return type. If the value of a field has non-whitespace leading or trailing characters around a parameter reference, it is subject to string interpolation. The effective value of the field is a string containing the leading characters, followed by the string value of the parameter reference, followed by the trailing characters. The string value of the parameter reference is its textual JSON representation with the following rules: * Leading and trailing quotes are stripped from strings * Objects entries are sorted by key Multiple parameter references may appear in a single field. This case must be treated as a string interpolation. After interpolating the first parameter reference, interpolation must be recursively applied to the trailing characters to yield the final string value. ## Expressions An expression is a fragment of [Javascript/ECMAScript 5.1](http://www.ecma-international.org/ecma-262/5.1/) code evaluated by the workflow platform to affect the inputs, outputs, or behavior of a process. In the generic execution sequence, expressions may be evaluated during step 5 (process setup), step 6 (execute process), and/or step 7 (capture output). Expressions are distinct from regular processes in that they are intended to modify the behavior of the workflow itself rather than perform the primary work of the workflow. To declare the use of expressions, the document must include the process requirement `InlineJavascriptRequirement`. Expressions may be used in any field permitting the pseudo-type `Expression`, as specified by this document. Expressions are denoted by the syntax `$(...)` or `${...}`. A code fragment wrapped in the `$(...)` syntax must be evaluated as a [ECMAScript expression](http://www.ecma-international.org/ecma-262/5.1/#sec-11). A code fragment wrapped in the `${...}` syntax must be evaluated as a [EMACScript function body](http://www.ecma-international.org/ecma-262/5.1/#sec-13) for an anonymous, zero-argument function. Expressions must return a valid JSON data type: one of null, string, number, boolean, array, object. Other return values must result in a `permanentFailure`. Implementations must permit any syntactically valid Javascript and account for nesting of parenthesis or braces and that strings that may contain parenthesis or braces when scanning for expressions. The runtime must include any code defined in the ["expressionLib" field of InlineJavascriptRequirement](#InlineJavascriptRequirement) prior to executing the actual expression. Before executing the expression, the runtime must initialize as global variables the fields of the parameter context described above. The effective value of the field after expression evaluation follows the same rules as parameter references discussed above. Multiple expressions may appear in a single field. Expressions must be evaluated in an isolated context (a "sandbox") which permits no side effects to leak outside the context. Expressions also must be evaluated in [Javascript strict mode](http://www.ecma-international.org/ecma-262/5.1/#sec-4.2.2). The order in which expressions are evaluated is undefined except where otherwise noted in this document. An implementation may choose to implement parameter references by evaluating as a Javascript expression. The results of evaluating parameter references must be identical whether implemented by Javascript evaluation or some other means. Implementations may apply other limits, such as process isolation, timeouts, and operating system containers/jails to minimize the security risks associated with running untrusted code embedded in a CWL document. Exceptions thrown from an exception must result in a `permanentFailure` of the process. ## Executing CWL documents as scripts By convention, a CWL document may begin with `#!/usr/bin/env cwl-runner` and be marked as executable (the POSIX "+x" permission bits) to enable it to be executed directly. A workflow platform may support this mode of operation; if so, it must provide `cwl-runner` as an alias for the platform's CWL implementation. A CWL input object document may similarly begin with `#!/usr/bin/env cwl-runner` and be marked as executable. In this case, the input object must include the field `cwl:tool` supplying an IRI to the default CWL document that should be executed using the fields of the input object as input parameters. ## Discovering CWL documents on a local filesystem To discover CWL documents look in the following locations: `/usr/share/commonwl/` `/usr/local/share/commonwl/` `$XDG_DATA_HOME/commonwl/` (usually `$HOME/.local/share/commonwl`) `$XDF_DATA_HOME` is from the [XDG Base Directory Specification](http://standards.freedesktop.org/basedir-spec/basedir-spec-0.6.html) cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/intro.md0000644000175000017500000000157313036412053023702 0ustar michaelmichael00000000000000# Status of this document This document is the product of the [Common Workflow Language working group](https://groups.google.com/forum/#!forum/common-workflow-language). The latest stable version of this document is available in the "v1.0" directory at https://github.com/common-workflow-language/common-workflow-language The products of the CWL working group (including this document) are made available under the terms of the Apache License, version 2.0. # Introduction The Common Workflow Language (CWL) working group is an informal, multi-vendor working group consisting of various organizations and individuals that have an interest in portability of data analysis workflows. The goal is to create specifications like this one that enable data scientists to describe analysis tools and workflows that are powerful, easy to use, portable, and support reproducibility. cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/UserGuide.yml0000644000175000017500000006715113036412053024650 0ustar michaelmichael00000000000000- name: userguide type: documentation doc: - $include: userguide-intro.md - | # Wrapping Command Line Tools - | ## First example The simplest "hello world" program. This accepts one input parameter, writes a message to the terminal or job log, and produces no permanent output. CWL documents are written in [JSON](http://json.org) or [YAML](http://yaml.org), or a mix of the two. *1st-tool.cwl* ``` - $include: examples/1st-tool.cwl - | ``` Use a YAML object in a separate file to describe the input of a run: *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwl-runner 1st-tool.cwl echo-job.yml [job 140199012414352] $ echo 'Hello world!' Hello world! Final process status is success ``` What's going on here? Let's break it down: ``` cwlVersion: v1.1.0-dev1 class: CommandLineTool ``` The `cwlVersion` field indicates the version of the CWL spec used by the document. The `class` field indicates this document describes a command line tool. ``` baseCommand: echo ``` The `baseCommand` provides the name of program that will actually run (echo) ``` inputs: message: type: string inputBinding: position: 1 ``` The `inputs` section describes the inputs of the tool. This is a list of input parameters and each parameter includes an identifier, a data type, and optionally an `inputBinding` which describes how this input parameter should appear on the command line. In this example, the `position` field indicates where it should appear on the command line. ``` outputs: [] ``` This tool has no formal output, so the `outputs` section is an empty list. - | ## Essential input parameters The `inputs` of a tool is a list of input parameters that control how to run the tool. Each parameter has an `id` for the name of parameter, and `type` describing what types of values are valid for that parameter. Available primitive types are *string*, *int*, *long*, *float*, *double*, and *null*; complex types are *array* and *record*; in addition there are special types *File*, *Directory* and *Any*. The following example demonstrates some input parameters with different types and appearing on the command line in different ways: *inp.cwl* ``` - $include: examples/inp.cwl - | ``` *inp-job.yml* ``` - $include: examples/inp-job.yml - | ``` Notice that "example_file", as a `File` type, must be provided as an object with the fields `class: File` and `path`. Next, create a whale.txt and invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ touch whale.txt $ cwl-runner inp.cwl inp-job.yml [job 140020149614160] /home/example$ echo -f -i42 --example-string hello --file=/home/example/whale.txt -f -i42 --example-string hello --file=/home/example/whale.txt Final process status is success ``` The field `inputBinding` is optional and indicates whether and how the input parameter should be appear on the tool's command line. If `inputBinding` is missing, the parameter does not appear on the command line. Let's look at each example in detail. ``` example_flag: type: boolean inputBinding: position: 1 prefix: -f ``` Boolean types are treated as a flag. If the input parameter "example_flag" is "true", then `prefix` will be added to the command line. If false, no flag is added. ``` example_string: type: string inputBinding: position: 3 prefix: --example-string ``` String types appear on the command line as literal values. The `prefix` is optional, if provided, it appears as a separate argument on the command line before the parameter . In the example above, this is rendered as `--example-string hello`. ``` example_int: type: int inputBinding: position: 2 prefix: -i separate: false ``` Integer (and floating point) types appear on the command line with decimal text representation. When the option `separate` is false (the default value is true), the prefix and value are combined into a single argument. In the example above, this is rendered as `-i42`. ``` example_file: type: File? inputBinding: prefix: --file= separate: false position: 4 ``` File types appear on the command line as the path to the file. When the parameter type ends with a question mark `?` it indicates that the parameter is optional. In the example above, this is rendered as `--file=/home/example/whale.txt`. However, if the "example_file" parameter were not provided in the input, nothing would appear on the command line. Input files are read-only. If you wish to update an input file, you must first copy it to the output directory. The value of `position` is used to determine where parameter should appear on the command line. Positions are relative to one another, not abosolute. As a result, positions do not have to be sequential, three parameters with positions `[1, 3, 5]` will result in the same command line as `[1, 2, 3]`. More than one parameter can have the same position (ties are broken using the parameter name), and the position field itself is optional. the default position is 0. The `baseCommand` field always comes before parameters. - | ## Returning output files The `outputs` of a tool is a list of output parameters that should be returned after running the tool. Each parameter has an `id` for the name of parameter, and `type` describing what types of values are valid for that parameter. When a tool runs under CWL, the starting working directory is the designated output directory. The underlying tool or script must record its results in the form of files created in the output directory. The output parameters returned by the CWL tool are either the output files themselves, or come from examining the content of those files. *tar.cwl* ``` - $include: examples/tar.cwl - | ``` *tar-job.yml* ``` - $include: examples/tar-job.yml - | ``` Next, create a tar file for the example and invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ touch hello.txt && tar -cvf hello.tar hello.txt $ cwl-runner tar.cwl tar-job.yml [job 139868145165200] $ tar xf /home/example/hello.tar Final process status is success { "example_out": { "location": "hello.txt", "size": 13, "class": "File", "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b" } } ``` The field `outputBinding` describes how to to set the value of each output parameter. ``` outputs: - id: example_out type: File outputBinding: glob: hello.txt ``` The `glob` field consists of the name of a file in the output directory. If you don't know name of the file in advance, you can use a wildcard pattern. - | ## Capturing a tool's standard output stream To capture a tool's standard output stream, add the `stdout` field with the name of the file where the output stream should go. Then add `type: stdout` on the corresponding output parameter. *stdout.cwl* ``` - $include: examples/stdout.cwl - | ``` *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ cwl-runner stdout.cwl echo-job.yml [job 140199012414352] $ echo 'Hello world!' > output.txt Final process status is success { "output": { "location": "output.txt", "size": 13, "class": "File", "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b" } } $ cat output.txt Hello world! ``` - | ## Parameter references In a previous example, we used extracted a file using the "tar" program. However, that example was very limited becuase it assumed that the file we were interested in was called "hello.txt". In this example, you will see how to reference the value of input parameters dynamically from other fields. *tar-param.cwl* ``` - $include: examples/tar-param.cwl - | ``` *tar-param-job.yml* ``` - $include: examples/tar-param-job.yml - | ``` Create your input files and invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ rm hello.tar || true && touch goodbye.txt && tar -cvf hello.tar goodbye.txt $ cwl-runner tar-param.cwl tar-param-job.yml [job 139868145165200] $ tar xf /home/example/hello.tar goodbye.txt Final process status is success { "example_out": { "location": "goodbye.txt", "size": 24, "class": "File", "checksum": "sha1$dd0a4c4c49ba43004d6611771972b6cf969c1c01" } } ``` Certain fields permit parameter references which are enclosed in `$(...)`. These are evaluated and replaced with value being referenced. ``` outputs: example_out: type: File outputBinding: glob: $(inputs.extractfile) ``` References are written using a subset of Javascript syntax. In this example, `$(inputs.extractfile)`, `$(inputs["extractfile"])`, and `$(inputs['extractfile'])` are equivalent. The value of the "inputs" variable is the input object provided when the CWL tool was invoked. Note that because File parameters are objects, to get the path to an input file you must reference the path field on a file object; to reference the path to the tar file in the above example you would write `$(inputs.tarfile.path)`. - | ## Running tools inside Docker [Docker](http://docker.io) containers simplify software installation by providing a complete known-good runtime for software and its dependencies. However, containers are also purposefully isolated from the host system, so in order to run a tool inside a Docker container there is additional work to ensure that input files are available inside the container and output files can be recovered from the contianer. CWL can perform this work automatically, allowing you to use Docker to simplify your software management while avoiding the complexity of invoking and managing Docker containers. This example runs a simple Node.js script inside a Docker container. *docker.cwl* ``` - $include: examples/docker.cwl - | ``` *docker-job.yml* ``` - $include: examples/docker-job.yml - | ``` Provide a hello.js and invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ echo "console.log(\"Hello World\");" > hello.js $ cwl-runner docker.cwl docker-job.yml [job 140259721854416] /home/example$ docker run -i --volume=/home/example/hello.js:/var/lib/cwl/job369354770_examples/hello.js:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpDLs5hm:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp node:slim node /var/lib/cwl/job369354770_examples/hello.js Hello world! Final process status is success ``` Notice the CWL runner has constructed a Docker command line to run the script. One of the responsibilies of the CWL runner is to the paths of input files to reflect the location where they appear inside the container. In this example, the path to the script `hello.js` is `/home/example/hello.js` outside the container but `/var/lib/cwl/job369354770_examples/hello.js` inside the container, as reflected in the invocation of the `node` command. - | ## Additional command line arguments and runtime parameters Sometimes tools require additional command line options that don't correspond exactly to input parameters. In this example, we will wrap the Java compiler to compile a java source file to a class file. By default, `javac` will create the class files in the same directory as the source file. However, CWL input files (and the directories in which they appear) may be read-only, so we need to instruct javac to write the class file to the designated output directory instead. *arguments.cwl* ``` - $include: examples/arguments.cwl - | ``` *arguments-job.yml* ``` - $include: examples/arguments-job.yml - | ``` Now create a sample Java file and invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ echo "public class Hello {}" > Hello.java $ cwl-runner arguments.cwl arguments-job.yml [job arguments.cwl] /tmp/tmpwYALo1$ docker \ run \ -i \ --volume=/home/peter/work/common-workflow-language/v1.1.0-dev1/examples/Hello.java:/var/lib/cwl/stg8939ac04-7443-4990-a518-1855b2322141/Hello.java:ro \ --volume=/tmp/tmpwYALo1:/var/spool/cwl:rw \ --volume=/tmp/tmpptIAJ8:/tmp:rw \ --workdir=/var/spool/cwl \ --read-only=true \ --user=1001 \ --rm \ --env=TMPDIR=/tmp \ --env=HOME=/var/spool/cwl \ java:7 \ javac \ -d \ /var/spool/cwl \ /var/lib/cwl/stg8939ac04-7443-4990-a518-1855b2322141/Hello.java Final process status is success { "classfile": { "size": 416, "location": "/home/example/Hello.class", "checksum": "sha1$2f7ac33c1f3aac3f1fec7b936b6562422c85b38a", "class": "File" } } ``` Here we use the `arguments` field to add an additional argument to the command line that isn't tied to a specific input parameter. ``` arguments: ["-d", $(runtime.outdir)] ``` This example references a runtime parameter. Runtime parameters provide information about the hardware or software environment when the tool is actually executed. The `$(runtime.outdir)` parameter is the path to the designated output directory. Other parameters include `$(runtime.tmpdir)`, `$(runtime.ram)`, `$(runtime.cores)`, `$(runtime.ram)`, `$(runtime.outdirSize)`, and `$(runtime.tmpdirSize)`. See the [Runtime Environment](CommandLineTool.html#Runtime_environment) section of the CWL specification for details. - | ## Array inputs It is easy to add arrays of input parameters represented to the command line. To specify an array parameter, the array definition is nested under the `type` field with `type: array` and `items` defining the valid data types that may appear in the array. *array-inputs.cwl* ``` - $include: examples/array-inputs.cwl - | ``` *array-inputs-job.yml* ``` - $include: examples/array-inputs-job.yml - | ``` Now invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ cwl-runner array-inputs.cwl array-inputs-job.yml [job 140334923640912] /home/example$ echo -A one two three -B=four -B=five -B=six -C=seven,eight,nine -A one two three -B=four -B=five -B=six -C=seven,eight,nine Final process status is success {} ``` The `inputBinding` can appear either on the outer array parameter definition or the inner array element definition, and these produce different behavior when constructing the command line, as shown above. In addition, the `itemSeperator` field, if provided, specifies that array values should be concatenated into a single argument separated by the item separator string. You can specify arrays of arrays, arrays of records, and other complex types. - | ## Array outputs You can also capture multiple output files into an array of files using `glob`. *array-outputs.cwl* ``` - $include: examples/array-outputs.cwl - | ``` *array-outpust-job.yml* ``` - $include: examples/array-outputs-job.yml - | ``` Now invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ cwl-runner array-outputs.cwl array-outputs-job.yml [job 140190876078160] /home/example$ touch foo.txt bar.dat baz.txt Final process status is success { "output": [ { "size": 0, "location": "/home/peter/work/common-workflow-language/draft-3/examples/foo.txt", "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", "class": "File" }, { "size": 0, "location": "/home/peter/work/common-workflow-language/draft-3/examples/baz.txt", "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", "class": "File" } ] } ``` - | ## Record inputs, dependent and mutually exclusive parameters Sometimes an underlying tool has several arguments that must be provided together (they are dependent) or several arguments that cannot be provided together (they are exclusive). You can use records and type unions to group parameters together to describe these two conditions. *record.cwl* ``` - $include: examples/record.cwl - | ``` *record-job1.yml* ``` - $include: examples/record-job1.yml - | ``` ``` $ cwl-runner record.cwl record-job1.yml Workflow error: Error validating input record, could not validate field `dependent_parameters` because missing required field `itemB` ``` In the first example, you can't provide `itemA` without also providing `itemB`. *record-job2.yml* ``` - $include: examples/record-job2.yml - | ``` ``` $ cwl-runner record.cwl record-job2.yml [job 140566927111376] /home/example$ echo -A one -B two -C three -A one -B two -C three Final process status is success {} ``` In the second example, `itemC` and `itemD` are exclusive, so only `itemC` is added to the command line and `itemD` is ignored. *record-job3.yml* ``` - $include: examples/record-job3.yml - | ``` ``` $ cwl-runner record.cwl record-job3.yml [job 140606932172880] /home/example$ echo -A one -B two -D four -A one -B two -D four Final process status is success {} ``` In the third example, only `itemD` is provided, so it appears on the command line. - | ## Environment variables Tools run in a restricted environment and do not inherit most environment variables from the parent process. You can set environment variables for the tool using `EnvVarRequirement`. *env.cwl* ``` - $include: examples/env.cwl - | ``` *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwl-runner env.cwl echo-job.yml [job 140710387785808] /home/example$ env PATH=/bin:/usr/bin:/usr/local/bin HELLO=Hello world! TMPDIR=/tmp/tmp63Obpk Final process status is success {} ``` - | ## Javascript expressions If you need to manipulate input parameters, include the requirement `InlineJavascriptRequirement` and then anywhere a parameter reference is legal you can provide a fragment of Javascript that will be evaluated by the CWL runner. *expression.cwl* ``` - $include: examples/expression.cwl - | ``` ``` $ cwl-runner expression.cwl empty.yml [job 140000594593168] /home/example$ echo -A 2 -B baz -C 10 9 8 7 6 5 4 3 2 1 -A 2 -B baz -C 10 9 8 7 6 5 4 3 2 1 Final process status is success {} ``` You can only use expressions in certain fields. These are: `filename`, `fileContent`, `envValue`, `valueFrom`, `glob`, `outputEval`, `stdin`, `stdout`, `coresMin`, `coresMax`, `ramMin`, `ramMax`, `tmpdirMin`, `tmpdirMax`, `outdirMin`, and `outdirMax`. - | ## Creating files at runtime Sometimes you need to create a file on the fly from input parameters, such as tools which expect to read their input configuration from a file rather than the command line parameters. To do this, use `InitialWorkDirRequirement`. *createfile.cwl* ``` - $include: examples/createfile.cwl - | ``` *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwltool createfile.cwl echo-job.yml [job 140528604979344] /home/example$ cat example.conf CONFIGVAR=Hello world! Final process status is success {} ``` - | ## Staging input files in the output directory Normally, input files are located in a read-only directory separate from the output directory. This causes problems if the underlying tool expects to write its output files alongside the input file in the same directory. You use `InitialWorkDirRequirement` to stage input files into the output directory. In this example, we use a Javascript expression to extract the base name of the input file from its leading directory path. *linkfile.cwl* ``` - $include: examples/linkfile.cwl - | ``` *arguments-job.yml* ``` - $include: examples/arguments-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwl-runner linkfile.cwl arguments-job.yml [job 139928309171664] /home/example$ docker run -i --volume=/home/example/Hello.java:/var/lib/cwl/job557617295_examples/Hello.java:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpmNbApw:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp java:7 javac Hello.java Final process status is success { "classfile": { "size": 416, "location": "/home/example/Hello.class", "checksum": "sha1$2f7ac33c1f3aac3f1fec7b936b6562422c85b38a", "class": "File" } } ``` - | # Writing Workflows ## First workflow This workflow extracts a java source file from a tar file and then compiles it. *1st-workflow.cwl* ``` - $include: examples/1st-workflow.cwl - | ``` Use a JSON object in a separate file to describe the input of a run: *1st-workflow-job.yml* ``` - $include: examples/1st-workflow-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ echo "public class Hello {}" > Hello.java && tar -cvf hello.tar Hello.java $ cwl-runner 1st-workflow.cwl 1st-workflow-job.yml [job untar] /tmp/tmp94qFiM$ tar xf /home/example/hello.tar Hello.java [step untar] completion status is success [job compile] /tmp/tmpu1iaKL$ docker run -i --volume=/tmp/tmp94qFiM/Hello.java:/var/lib/cwl/job301600808_tmp94qFiM/Hello.java:ro --volume=/tmp/tmpu1iaKL:/var/spool/cwl:rw --volume=/tmp/tmpfZnNdR:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp java:7 javac -d /var/spool/cwl /var/lib/cwl/job301600808_tmp94qFiM/Hello.java [step compile] completion status is success [workflow 1st-workflow.cwl] outdir is /home/example Final process status is success { "classout": { "location": "/home/example/Hello.class", "checksum": "sha1$e68df795c0686e9aa1a1195536bd900f5f417b18", "class": "File", "size": 416 } } ``` What's going on here? Let's break it down: ``` cwlVersion: v1.1.0-dev1 class: Workflow ``` The 'cwlVersion` field indicates the version of the CWL spec used by the document. The `class` field indicates this document describes a workflow. ``` inputs: inp: File ex: string ``` The `inputs` section describes the inputs of the workflow. This is a list of input parameters where each parameter consists of an identifier and a data type. These parameters can be used as sources for input to specific workflows steps. ``` outputs: classout: type: File outputSource: compile/classfile ``` The `outputs` section describes the outputs of the workflow. This is a list of output parameters where each parameter consists of an identifier and a data type. The `source` connects the output parameter `classfile` of the `compile` step to the workflow output parameter `classout`. ``` steps: untar: run: tar-param.cwl in: tarfile: inp extractfile: ex outputs: [example_out] ``` The `steps` section describes the actual steps of the workflow. In this example, the first step extracts a file from a tar file, and the second step compiles the file from the first step using the java compiler. Workflow steps are not necessarily run in the order they are listed, instead the order is determined by the dependencies between steps (using `source`). In addition, workflow steps which do not depend on one another may run in parallel. The first step, `untar` runs `tar-param.cwl` (described previously in [Parameter references](#Parameter_references)). This tool has two input parameters, `tarfile` and `extractfile` and one output parameter `example_out`. The `inputs` section of the workflow step connects these two input parameters to the inputs of the workflow, `inp` and `ex` using `source`. This means that when the workflow step is executed, the values assigned to `inp` and `ex` will be used for the parameters `tarfile` and `extractfile` in order to run the tool. The `outputs` section of the workflow step lists the output parameters that are expected from the tool. ``` compile: run: arguments.cwl in: src: untar/example_out outputs: [classfile] ``` The second step `compile` depends on the results from the first step by connecting the input parameter `src` to the output parameter of `untar` using `untar/example_out`. The output of this step `classfile` is connected to the `outputs` section for the Workflow, described above. cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/Workflow.yml0000644000175000017500000004752213036412053024566 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" $graph: - name: "WorkflowDoc" type: documentation doc: - | # Common Workflow Language (CWL) Workflow Description, v1.1.0-dev1 This version: * https://w3id.org/cwl/v1.1.0-dev1/ Current version: * https://w3id.org/cwl/ - "\n\n" - {$include: contrib.md} - "\n\n" - | # Abstract One way to define a workflow is: an analysis task represented by a directed graph describing a sequence of operations that transform an input data set to output. This specification defines the Common Workflow Language (CWL) Workflow description, a vendor-neutral standard for representing workflows intended to be portable across a variety of computing platforms. - {$include: intro.md} - | ## Introduction to v1.1.0-dev1 This is the first development release of the 1.1.0 version of the CWL Workflow specification. ## Introduction to v1.0 This specification represents the first full release from the CWL group. Since draft-3, this draft introduces the following changes and additions: * The `inputs` and `outputs` fields have been renamed `in` and `out`. * Syntax simplifcations: denoted by the `map<>` syntax. Example: `in` contains a list of items, each with an id. Now one can specify a mapping of that identifier to the corresponding `InputParameter`. ``` in: - id: one type: string doc: First input parameter - id: two type: int doc: Second input parameter ``` can be ``` in: one: type: string doc: First input parameter two: type: int doc: Second input parameter ``` * The common field `description` has been renamed to `doc`. ## Purpose The Common Workflow Language Command Line Tool Description express workflows for data-intensive science, such as Bioinformatics, Chemistry, Physics, and Astronomy. This specification is intended to define a data and execution model for Workflows that can be implemented on top of a variety of computing platforms, ranging from an individual workstation to cluster, grid, cloud, and high performance computing systems. - {$include: concepts.md} - name: ExpressionToolOutputParameter type: record extends: OutputParameter fields: - name: type type: - "null" - CWLType - OutputRecordSchema - OutputEnumSchema - OutputArraySchema - string - type: array items: - CWLType - OutputRecordSchema - OutputEnumSchema - OutputArraySchema - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" refScope: 2 typeDSL: True doc: | Specify valid types of data that may be assigned to this parameter. - type: record name: ExpressionTool extends: Process specialize: - specializeFrom: InputParameter specializeTo: RegularInputParameter - specializeFrom: OutputParameter specializeTo: ExpressionToolOutputParameter documentRoot: true doc: | Execute an expression as a Workflow step. fields: - name: class jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: expression type: [string, Expression] doc: | The expression to execute. The expression must return a JSON object which matches the output parameters of the ExpressionTool. - name: LinkMergeMethod type: enum docParent: "#WorkflowStepInput" doc: The input link merge method, described in [WorkflowStepInput](#WorkflowStepInput). symbols: - merge_nested - merge_flattened - name: WorkflowOutputParameter type: record extends: OutputParameter docParent: "#Workflow" doc: | Describe an output parameter of a workflow. The parameter must be connected to one or more parameters defined in the workflow that will provide the value of the output parameter. fields: - name: outputSource doc: | Specifies one or more workflow parameters that supply the value of to the output parameter. jsonldPredicate: "_id": "cwl:outputSource" "_type": "@id" refScope: 0 type: - string? - string[]? - name: linkMerge type: ["null", LinkMergeMethod] jsonldPredicate: "cwl:linkMerge" doc: | The method to use to merge multiple sources into a single array. If not specified, the default method is "merge_nested". - name: type type: - "null" - CWLType - OutputRecordSchema - OutputEnumSchema - OutputArraySchema - string - type: array items: - CWLType - OutputRecordSchema - OutputEnumSchema - OutputArraySchema - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" refScope: 2 typeDSL: True doc: | Specify valid types of data that may be assigned to this parameter. - name: Sink type: record abstract: true fields: - name: source doc: | Specifies one or more workflow parameters that will provide input to the underlying step parameter. jsonldPredicate: "_id": "cwl:source" "_type": "@id" refScope: 2 type: - string? - string[]? - name: linkMerge type: LinkMergeMethod? jsonldPredicate: "cwl:linkMerge" doc: | The method to use to merge multiple inbound links into a single array. If not specified, the default method is "merge_nested". - type: record name: WorkflowStepInput extends: Sink docParent: "#WorkflowStep" doc: | The input of a workflow step connects an upstream parameter (from the workflow inputs, or the outputs of other workflows steps) with the input parameters of the underlying step. ## Input object A WorkflowStepInput object must contain an `id` field in the form `#fieldname` or `#stepname.fieldname`. When the `id` field contains a period `.` the field name consists of the characters following the final period. This defines a field of the workflow step input object with the value of the `source` parameter(s). ## Merging To merge multiple inbound data links, [MultipleInputFeatureRequirement](#MultipleInputFeatureRequirement) must be specified in the workflow or workflow step requirements. If the sink parameter is an array, or named in a [workflow scatter](#WorkflowStep) operation, there may be multiple inbound data links listed in the `source` field. The values from the input links are merged depending on the method specified in the `linkMerge` field. If not specified, the default method is "merge_nested". * **merge_nested** The input must be an array consisting of exactly one entry for each input link. If "merge_nested" is specified with a single link, the value from the link must be wrapped in a single-item list. * **merge_flattened** 1. The source and sink parameters must be compatible types, or the source type must be compatible with single element from the "items" type of the destination array parameter. 2. Source parameters which are arrays are concatenated. Source parameters which are single element types are appended as single elements. fields: - name: id type: string jsonldPredicate: "@id" doc: "A unique identifier for this workflow input parameter." - name: default type: ["null", Any] doc: | The default value for this parameter if there is no `source` field. jsonldPredicate: "cwl:default" - name: valueFrom type: - "null" - string - Expression jsonldPredicate: "cwl:valueFrom" doc: | To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must be specified in the workflow or workflow step requirements. If `valueFrom` is a constant string value, use this as the value for this input parameter. If `valueFrom` is a parameter reference or expression, it must be evaluated to yield the actual value to be assiged to the input field. The `self` value of in the parameter reference or expression must be the value of the parameter(s) specified in the `source` field, or null if there is no `source` field. The value of `inputs` in the parameter reference or expression must be the input object to the workflow step after assigning the `source` values and then scattering. The order of evaluating `valueFrom` among step input parameters is undefined and the result of evaluating `valueFrom` on a parameter must not be visible to evaluation of `valueFrom` on other parameters. - type: record name: WorkflowStepOutput docParent: "#WorkflowStep" doc: | Associate an output parameter of the underlying process with a workflow parameter. The workflow parameter (given in the `id` field) be may be used as a `source` to connect with input parameters of other workflow steps, or with an output parameter of the process. fields: - name: id type: string jsonldPredicate: "@id" doc: | A unique identifier for this workflow output parameter. This is the identifier to use in the `source` field of `WorkflowStepInput` to connect the output value to downstream parameters. - name: ScatterMethod type: enum docParent: "#WorkflowStep" doc: The scatter method, as described in [workflow step scatter](#WorkflowStep). symbols: - dotproduct - nested_crossproduct - flat_crossproduct - name: WorkflowStep type: record docParent: "#Workflow" doc: | A workflow step is an executable element of a workflow. It specifies the underlying process implementation (such as `CommandLineTool` or another `Workflow`) in the `run` field and connects the input and output parameters of the underlying process to workflow parameters. # Scatter/gather To use scatter/gather, [ScatterFeatureRequirement](#ScatterFeatureRequirement) must be specified in the workflow or workflow step requirements. A "scatter" operation specifies that the associated workflow step or subworkflow should execute separately over a list of input elements. Each job making up a scatter operation is independent and may be executed concurrently. The `scatter` field specifies one or more input parameters which will be scattered. An input parameter may be listed more than once. The declared type of each input parameter is implicitly wrapped in an array for each time it appears in the `scatter` field. As a result, upstream parameters which are connected to scattered parameters may be arrays. All output parameter types are also implicitly wrapped in arrays. Each job in the scatter results in an entry in the output array. If `scatter` declares more than one input parameter, `scatterMethod` describes how to decompose the input into a discrete set of jobs. * **dotproduct** specifies that each of the input arrays are aligned and one element taken from each array to construct each job. It is an error if all input arrays are not the same length. * **nested_crossproduct** specifies the Cartesian product of the inputs, producing a job for every combination of the scattered inputs. The output must be nested arrays for each level of scattering, in the order that the input arrays are listed in the `scatter` field. * **flat_crossproduct** specifies the Cartesian product of the inputs, producing a job for every combination of the scattered inputs. The output arrays must be flattened to a single level, but otherwise listed in the order that the input arrays are listed in the `scatter` field. # Subworkflows To specify a nested workflow as part of a workflow step, [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) must be specified in the workflow or workflow step requirements. fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this workflow step." - name: in type: WorkflowStepInput[] jsonldPredicate: _id: "cwl:in" mapSubject: id mapPredicate: source doc: | Defines the input parameters of the workflow step. The process is ready to run when all required input parameters are associated with concrete values. Input parameters include a schema for each parameter which is used to validate the input object. It may also be used build a user interface for constructing the input object. - name: out type: - type: array items: [string, WorkflowStepOutput] jsonldPredicate: _id: "cwl:out" _type: "@id" identity: true doc: | Defines the parameters representing the output of the process. May be used to generate and/or validate the output object. - name: requirements type: ProcessRequirement[]? jsonldPredicate: _id: "cwl:requirements" mapSubject: class doc: | Declares requirements that apply to either the runtime environment or the workflow engine that must be met in order to execute this workflow step. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. - name: hints type: Any[]? jsonldPredicate: _id: "cwl:hints" noLinkCheck: true mapSubject: class doc: | Declares hints applying to either the runtime environment or the workflow engine that may be helpful in executing this workflow step. It is not an error if an implementation cannot satisfy all hints, however the implementation may report a warning. - name: label type: string? jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: doc type: string? jsonldPredicate: "rdfs:comment" doc: "A long, human-readable description of this process object." - name: run type: [string, Process] jsonldPredicate: "_id": "cwl:run" "_type": "@id" doc: | Specifies the process to run. - name: scatter type: - string? - string[]? jsonldPredicate: "_id": "cwl:scatter" "_type": "@id" "_container": "@list" refScope: 0 - name: scatterMethod doc: | Required if `scatter` is an array of more than one element. type: ScatterMethod? jsonldPredicate: "_id": "cwl:scatterMethod" "_type": "@vocab" - name: Workflow type: record extends: "#Process" documentRoot: true specialize: - specializeFrom: InputParameter specializeTo: RegularInputParameter - specializeFrom: OutputParameter specializeTo: WorkflowOutputParameter doc: | A workflow describes a set of **steps** and the **dependencies** between those steps. When a step produces output that will be consumed by a second step, the first step is a dependency of the second step. When there is a dependency, the workflow engine must execute the preceeding step and wait for it to successfully produce output before executing the dependent step. If two steps are defined in the workflow graph that are not directly or indirectly dependent, these steps are **independent**, and may execute in any order or execute concurrently. A workflow is complete when all steps have been executed. Dependencies between parameters are expressed using the `source` field on [workflow step input parameters](#WorkflowStepInput) and [workflow output parameters](#WorkflowOutputParameter). The `source` field expresses the dependency of one parameter on another such that when a value is associated with the parameter specified by `source`, that value is propagated to the destination parameter. When all data links inbound to a given step are fufilled, the step is ready to execute. ## Workflow success and failure A completed step must result in one of `success`, `temporaryFailure` or `permanentFailure` states. An implementation may choose to retry a step execution which resulted in `temporaryFailure`. An implementation may choose to either continue running other steps of a workflow, or terminate immediately upon `permanentFailure`. * If any step of a workflow execution results in `permanentFailure`, then the workflow status is `permanentFailure`. * If one or more steps result in `temporaryFailure` and all other steps complete `success` or are not executed, then the workflow status is `temporaryFailure`. * If all workflow steps are executed and complete with `success`, then the workflow status is `success`. # Extensions [ScatterFeatureRequirement](#ScatterFeatureRequirement) and [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) are available as standard [extensions](#Extensions_and_Metadata) to core workflow semantics. fields: - name: "class" jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: steps doc: | The individual steps that make up the workflow. Each step is executed when all of its input data links are fufilled. An implementation may choose to execute the steps in a different order than listed and/or execute steps concurrently, provided that dependencies between steps are met. type: - type: array items: "#WorkflowStep" jsonldPredicate: mapSubject: id - type: record name: SubworkflowFeatureRequirement extends: ProcessRequirement doc: | Indicates that the workflow platform must support nested workflows in the `run` field of [WorkflowStep](#WorkflowStep). fields: - name: "class" type: "string" doc: "Always 'SubworkflowFeatureRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: ScatterFeatureRequirement type: record extends: ProcessRequirement doc: | Indicates that the workflow platform must support the `scatter` and `scatterMethod` fields of [WorkflowStep](#WorkflowStep). fields: - name: "class" type: "string" doc: "Always 'ScatterFeatureRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: MultipleInputFeatureRequirement type: record extends: ProcessRequirement doc: | Indicates that the workflow platform must support multiple inbound data links listed in the `source` field of [WorkflowStepInput](#WorkflowStepInput). fields: - name: "class" type: "string" doc: "Always 'MultipleInputFeatureRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - type: record name: StepInputExpressionRequirement extends: ProcessRequirement doc: | Indicate that the workflow platform must support the `valueFrom` field of [WorkflowStepInput](#WorkflowStepInput). fields: - name: "class" type: "string" doc: "Always 'StepInputExpressionRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/contrib.md0000644000175000017500000000200313036412053024174 0ustar michaelmichael00000000000000Authors: * Peter Amstutz , Arvados Project, Curoverse * Michael R. Crusoe , Common Workflow Language project * Nebojša Tijanić , Seven Bridges Genomics Contributors: * Brad Chapman , Harvard Chan School of Public Health * John Chilton , Galaxy Project, Pennsylvania State University * Michael Heuer ,UC Berkeley AMPLab * Andrey Kartashov , Cincinnati Children's Hospital * Dan Leehr , Duke University * Hervé Ménager , Institut Pasteur * Maya Nedeljkovich , Seven Bridges Genomics * Matt Scales , Institute of Cancer Research, London * Stian Soiland-Reyes [soiland-reyes@cs.manchester.ac.uk](mailto:soiland-reyes@cs.manchester.ac.uk), University of Manchester * Luka Stojanovic , Seven Bridges Genomics cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/README.md0000644000175000017500000000151013036412053023473 0ustar michaelmichael00000000000000# Common Workflow Language Specifications, v1.1.0-dev1 The CWL specifications are divided up into several documents. The [User Guide](UserGuide.html) provides a gentle introduction to writing CWL command line tools and workflows. The [Command Line Tool Description Specification](CommandLineTool.html) specifies the document schema and execution semantics for wrapping and executing command line tools. The [Workflow Description Specification](Workflow.html) specifies the document schema and execution semantics for composing workflows from components such as command line tools and other workflows. The [Semantic Annotations for Linked Avro Data (SALAD) Specification](SchemaSalad.html) specifies the preprocessing steps that must be applied when loading CWL documents and the schema language used to write the above specifications. cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/CommandLineTool.yml0000644000175000017500000007607013036412053026000 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" $graph: - name: CommandLineToolDoc type: documentation doc: - | # Common Workflow Language (CWL) Command Line Tool Description, v1.1.0-dev1 This version: * https://w3id.org/cwl/v1.1.0-dev1/ Current version: * https://w3id.org/cwl/ - "\n\n" - {$include: contrib.md} - "\n\n" - | # Abstract A Command Line Tool is a non-interactive executable program that reads some input, performs a computation, and terminates after producing some output. Command line programs are a flexible unit of code sharing and reuse, unfortunately the syntax and input/output semantics among command line programs is extremely heterogeneous. A common layer for describing the syntax and semantics of programs can reduce this incidental complexity by providing a consistent way to connect programs together. This specification defines the Common Workflow Language (CWL) Command Line Tool Description, a vendor-neutral standard for describing the syntax and input/output semantics of command line programs. - {$include: intro.md} - | ## Introduction to v1.1.0-dev1 This is the in progress first development version of the first maintenance release of the CWL CommandLineTool specification. Version 1.1 introduces the followings additions: * Addition of `stdin` type shortcut for `CommandInputParamater`s. ## Introduction to v1.0 This specification represents the first full release from the CWL group. Since draft-3, version 1.0 introduces the following changes and additions: * The [Directory](#Directory) type. * Syntax simplifcations: denoted by the `map<>` syntax. Example: inputs contains a list of items, each with an id. Now one can specify a mapping of that identifier to the corresponding `CommandInputParamater`. ``` inputs: - id: one type: string doc: First input parameter - id: two type: int doc: Second input parameter ``` can be ``` inputs: one: type: string doc: First input parameter two: type: int doc: Second input parameter ``` * [InitialWorkDirRequirement](#InitialWorkDirRequirement): list of files and subdirectories to be present in the output directory prior to execution. * Shortcuts for specifying the standard [output](#stdout) and/or [error](#stderr) streams as a (streamable) File output. * [SoftwareRequirement](#SoftwareRequirement) for describing software dependencies of a tool. * The common `description` field has been renamed to `doc`. ## Errata Post v1.0 release changes to the spec. * 13 July 2016: Mark `baseCommand` as optional and update descriptive text. ## Purpose Standalone programs are a flexible and interoperable form of code reuse. Unlike monolithic applications, applications and analysis workflows which are composed of multiple separate programs can be written in multiple languages and execute concurrently on multiple hosts. However, POSIX does not dictate computer-readable grammar or semantics for program input and output, resulting in extremely heterogeneous command line grammar and input/output semantics among program. This is a particular problem in distributed computing (multi-node compute clusters) and virtualized environments (such as Docker containers) where it is often necessary to provision resources such as input files before executing the program. Often this gap is filled by hard coding program invocation and implicitly assuming requirements will be met, or abstracting program invocation with wrapper scripts or descriptor documents. Unfortunately, where these approaches are application or platform specific it creates a significant barrier to reproducibility and portability, as methods developed for one platform must be manually ported to be used on new platforms. Similarly it creates redundant work, as wrappers for popular tools must be rewritten for each application or platform in use. The Common Workflow Language Command Line Tool Description is designed to provide a common standard description of grammar and semantics for invoking programs used in data-intensive fields such as Bioinformatics, Chemistry, Physics, Astronomy, and Statistics. This specification defines a precise data and execution model for Command Line Tools that can be implemented on a variety of computing platforms, ranging from a single workstation to cluster, grid, cloud, and high performance computing platforms. - {$include: concepts.md} - {$include: invocation.md} - type: record name: EnvironmentDef doc: | Define an environment variable that will be set in the runtime environment by the workflow platform when executing the command line tool. May be the result of executing an expression, such as getting a parameter from input. fields: - name: envName type: string doc: The environment variable name - name: envValue type: [string, Expression] doc: The environment variable value - type: record name: CommandLineBinding extends: InputBinding doc: | When listed under `inputBinding` in the input schema, the term "value" refers to the the corresponding value in the input object. For binding objects listed in `CommandLineTool.arguments`, the term "value" refers to the effective value after evaluating `valueFrom`. The binding behavior when building the command line depends on the data type of the value. If there is a mismatch between the type described by the input schema and the effective value, such as resulting from an expression evaluation, an implementation must use the data type of the effective value. - **string**: Add `prefix` and the string to the command line. - **number**: Add `prefix` and decimal representation to command line. - **boolean**: If true, add `prefix` to the command line. If false, add nothing. - **File**: Add `prefix` and the value of [`File.path`](#File) to the command line. - **array**: If `itemSeparator` is specified, add `prefix` and the join the array into a single string with `itemSeparator` separating the items. Otherwise first add `prefix`, then recursively process individual elements. - **object**: Add `prefix` only, and recursively add object fields for which `inputBinding` is specified. - **null**: Add nothing. fields: - name: position type: int? doc: "The sorting key. Default position is 0." - name: prefix type: string? doc: "Command line prefix to add before the value." - name: separate type: boolean? doc: | If true (default), then the prefix and value must be added as separate command line arguments; if false, prefix and value must be concatenated into a single command line argument. - name: itemSeparator type: string? doc: | Join the array elements into a single string with the elements separated by by `itemSeparator`. - name: valueFrom type: - "null" - string - Expression jsonldPredicate: "cwl:valueFrom" doc: | If `valueFrom` is a constant string value, use this as the value and apply the binding rules above. If `valueFrom` is an expression, evaluate the expression to yield the actual value to use to build the command line and apply the binding rules above. If the inputBinding is associated with an input parameter, the value of `self` in the expression will be the value of the input parameter. When a binding is part of the `CommandLineTool.arguments` field, the `valueFrom` field is required. - name: shellQuote type: boolean? doc: | If `ShellCommandRequirement` is in the requirements for the current command, this controls whether the value is quoted on the command line (default is true). Use `shellQuote: false` to inject metacharacters for operations such as pipes. - type: record name: CommandOutputBinding extends: OutputBinding doc: | Describes how to generate an output parameter based on the files produced by a CommandLineTool. The output parameter is generated by applying these operations in the following order: - glob - loadContents - outputEval fields: - name: glob type: - "null" - string - Expression - type: array items: string doc: | Find files relative to the output directory, using POSIX glob(3) pathname matching. If an array is provided, find files that match any pattern in the array. If an expression is provided, the expression must return a string or an array of strings, which will then be evaluated as one or more glob patterns. Must only match and return files which actually exist. - name: loadContents type: - "null" - boolean jsonldPredicate: "cwl:loadContents" doc: | For each file matched in `glob`, read up to the first 64 KiB of text from the file and place it in the `contents` field of the file object for manipulation by `outputEval`. - name: outputEval type: - "null" - string - Expression doc: | Evaluate an expression to generate the output value. If `glob` was specified, the value of `self` must be an array containing file objects that were matched. If no files were matched, `self` must be a zero length array; if a single file was matched, the value of `self` is an array of a single element. Additionally, if `loadContents` is `true`, the File objects must include up to the first 64 KiB of file contents in the `contents` field. - name: CommandInputRecordField type: record extends: InputRecordField specialize: - specializeFrom: InputRecordSchema specializeTo: CommandInputRecordSchema - specializeFrom: InputEnumSchema specializeTo: CommandInputEnumSchema - specializeFrom: InputArraySchema specializeTo: CommandInputArraySchema - specializeFrom: InputBinding specializeTo: CommandLineBinding - name: CommandInputRecordSchema type: record extends: InputRecordSchema specialize: - specializeFrom: InputRecordField specializeTo: CommandInputRecordField - name: CommandInputEnumSchema type: record extends: InputEnumSchema specialize: - specializeFrom: InputBinding specializeTo: CommandLineBinding - name: CommandInputArraySchema type: record extends: InputArraySchema specialize: - specializeFrom: InputRecordSchema specializeTo: CommandInputRecordSchema - specializeFrom: InputEnumSchema specializeTo: CommandInputEnumSchema - specializeFrom: InputArraySchema specializeTo: CommandInputArraySchema - specializeFrom: InputBinding specializeTo: CommandLineBinding - name: CommandOutputRecordField type: record extends: OutputRecordField specialize: - specializeFrom: OutputRecordSchema specializeTo: CommandOutputRecordSchema - specializeFrom: OutputEnumSchema specializeTo: CommandOutputEnumSchema - specializeFrom: OutputArraySchema specializeTo: CommandOutputArraySchema - specializeFrom: OutputBinding specializeTo: CommandOutputBinding - name: CommandOutputRecordSchema type: record extends: OutputRecordSchema specialize: - specializeFrom: OutputRecordField specializeTo: CommandOutputRecordField - name: CommandOutputEnumSchema type: record extends: OutputEnumSchema specialize: - specializeFrom: OutputRecordSchema specializeTo: CommandOutputRecordSchema - specializeFrom: OutputEnumSchema specializeTo: CommandOutputEnumSchema - specializeFrom: OutputArraySchema specializeTo: CommandOutputArraySchema - specializeFrom: OutputBinding specializeTo: CommandOutputBinding - name: CommandOutputArraySchema type: record extends: OutputArraySchema specialize: - specializeFrom: OutputRecordSchema specializeTo: CommandOutputRecordSchema - specializeFrom: OutputEnumSchema specializeTo: CommandOutputEnumSchema - specializeFrom: OutputArraySchema specializeTo: CommandOutputArraySchema - specializeFrom: OutputBinding specializeTo: CommandOutputBinding - type: record name: CommandInputParameter extends: InputParameter doc: An input parameter for a CommandLineTool. specialize: - specializeFrom: InputBinding specializeTo: CommandLineBinding fields: - name: type type: - "null" - CWLType - stdin - CommandInputRecordSchema - CommandInputEnumSchema - CommandInputArraySchema - string - type: array items: - CWLType - CommandInputRecordSchema - CommandInputEnumSchema - CommandInputArraySchema - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" refScope: 2 typeDSL: True doc: | Specify valid types of data that may be assigned to this parameter. - type: record name: CommandOutputParameter extends: OutputParameter doc: An output parameter for a CommandLineTool. specialize: - specializeFrom: OutputBinding specializeTo: CommandOutputBinding fields: - name: type type: - "null" - CWLType - stdin - stdout - stderr - CommandOutputRecordSchema - CommandOutputEnumSchema - CommandOutputArraySchema - string - type: array items: - CWLType - CommandOutputRecordSchema - CommandOutputEnumSchema - CommandOutputArraySchema - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" refScope: 2 typeDSL: True doc: | Specify valid types of data that may be assigned to this parameter. - name: stdin type: enum symbols: [ "cwl:stdin" ] docParent: "#CommandOutputParameter" doc: | Only valid as a `type` for a `CommandLineTool` input with no `inputBinding` set. `stdin` must not be specified at the `CommandLineTool` level. The following ``` inputs: an_input_name: type: stdin ``` is equivalent to ``` inputs: an_input_name: type: File streamable: true stdin: ${inputs.an_input_name.path} ``` - name: stdout type: enum symbols: [ "cwl:stdout" ] docParent: "#CommandOutputParameter" doc: | Only valid as a `type` for a `CommandLineTool` output with no `outputBinding` set. The following ``` outputs: an_output_name: type: stdout stdout: a_stdout_file ``` is equivalent to ``` outputs: an_output_name: type: File streamable: true outputBinding: glob: a_stdout_file stdout: a_stdout_file ``` If there is no `stdout` name provided, a random filename will be created. For example, the following ``` outputs: an_output_name: type: stdout ``` is equivalent to ``` outputs: an_output_name: type: File streamable: true outputBinding: glob: random_stdout_filenameABCDEFG stdout: random_stdout_filenameABCDEFG ``` - name: stderr type: enum symbols: [ "cwl:stderr" ] docParent: "#CommandOutputParameter" doc: | Only valid as a `type` for a `CommandLineTool` output with no `outputBinding` set. The following ``` outputs: an_output_name: type: stderr stderr: a_stderr_file ``` is equivalent to ``` outputs: an_output_name: type: File streamable: true outputBinding: glob: a_stderr_file stderr: a_stderr_file ``` If there is no `stderr` name provided, a random filename will be created. For example, the following ``` outputs: an_output_name: type: stderr ``` is equivalent to ``` outputs: an_output_name: type: File streamable: true outputBinding: glob: random_stderr_filenameABCDEFG stderr: random_stderr_filenameABCDEFG ``` - type: record name: CommandLineTool extends: Process documentRoot: true specialize: - specializeFrom: InputParameter specializeTo: CommandInputParameter - specializeFrom: OutputParameter specializeTo: CommandOutputParameter doc: | This defines the schema of the CWL Command Line Tool Description document. fields: - name: class jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: baseCommand doc: | Specifies the program to execute. If an array, the first element of the array is the command to execute, and subsequent elements are mandatory command line arguments. The elements in `baseCommand` must appear before any command line bindings from `inputBinding` or `arguments`. If `baseCommand` is not provided or is an empty array, the first element of the command line produced after processing `inputBinding` or `arguments` must be used as the program to execute. If the program includes a path separator character it must be an absolute path, otherwise it is an error. If the program does not include a path separator, search the `$PATH` variable in the runtime environment of the workflow runner find the absolute path of the executable. type: - string? - string[]? jsonldPredicate: "_id": "cwl:baseCommand" "_container": "@list" - name: arguments doc: | Command line bindings which are not directly associated with input parameters. type: - "null" - type: array items: [string, Expression, CommandLineBinding] jsonldPredicate: "_id": "cwl:arguments" "_container": "@list" - name: stdin type: ["null", string, Expression] jsonldPredicate: "https://w3id.org/cwl/cwl#stdin" doc: | A path to a file whose contents must be piped into the command's standard input stream. - name: stderr type: ["null", string, Expression] jsonldPredicate: "https://w3id.org/cwl/cwl#stderr" doc: | Capture the command's standard error stream to a file written to the designated output directory. If `stderr` is a string, it specifies the file name to use. If `stderr` is an expression, the expression is evaluated and must return a string with the file name to use to capture stderr. If the return value is not a string, or the resulting path contains illegal characters (such as the path separator `/`) it is an error. - name: stdout type: ["null", string, Expression] jsonldPredicate: "https://w3id.org/cwl/cwl#stdout" doc: | Capture the command's standard output stream to a file written to the designated output directory. If `stdout` is a string, it specifies the file name to use. If `stdout` is an expression, the expression is evaluated and must return a string with the file name to use to capture stdout. If the return value is not a string, or the resulting path contains illegal characters (such as the path separator `/`) it is an error. - name: successCodes type: int[]? doc: | Exit codes that indicate the process completed successfully. - name: temporaryFailCodes type: int[]? doc: | Exit codes that indicate the process failed due to a possibly temporary condition, where executing the process with the same runtime environment and inputs may produce different results. - name: permanentFailCodes type: int[]? doc: Exit codes that indicate the process failed due to a permanent logic error, where executing the process with the same runtime environment and same inputs is expected to always fail. - type: record name: DockerRequirement extends: ProcessRequirement doc: | Indicates that a workflow component should be run in a [Docker](http://docker.com) container, and specifies how to fetch or build the image. If a CommandLineTool lists `DockerRequirement` under `hints` (or `requirements`), it may (or must) be run in the specified Docker container. The platform must first acquire or install the correct Docker image as specified by `dockerPull`, `dockerImport`, `dockerLoad` or `dockerFile`. The platform must execute the tool in the container using `docker run` with the appropriate Docker image and tool command line. The workflow platform may provide input files and the designated output directory through the use of volume bind mounts. The platform may rewrite file paths in the input object to correspond to the Docker bind mounted locations. When running a tool contained in Docker, the workflow platform must not assume anything about the contents of the Docker container, such as the presence or absence of specific software, except to assume that the generated command line represents a valid command within the runtime environment of the container. ## Interaction with other requirements If [EnvVarRequirement](#EnvVarRequirement) is specified alongside a DockerRequirement, the environment variables must be provided to Docker using `--env` or `--env-file` and interact with the container's preexisting environment as defined by Docker. fields: - name: class type: string doc: "Always 'DockerRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: dockerPull type: string? doc: "Specify a Docker image to retrieve using `docker pull`." - name: dockerLoad type: string? doc: "Specify a HTTP URL from which to download a Docker image using `docker load`." - name: dockerFile type: string? doc: "Supply the contents of a Dockerfile which will be built using `docker build`." - name: dockerImport type: string? doc: "Provide HTTP URL to download and gunzip a Docker images using `docker import." - name: dockerImageId type: string? doc: | The image id that will be used for `docker run`. May be a human-readable image name or the image identifier hash. May be skipped if `dockerPull` is specified, in which case the `dockerPull` image id must be used. - name: dockerOutputDirectory type: string? doc: | Set the designated output directory to a specific location inside the Docker container. - type: record name: SoftwareRequirement extends: ProcessRequirement doc: | A list of software packages that should be configured in the environment of the defined process. fields: - name: class type: string doc: "Always 'SoftwareRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: packages type: SoftwarePackage[] doc: "The list of software to be configured." jsonldPredicate: mapSubject: package mapPredicate: specs - name: SoftwarePackage type: record fields: - name: package type: string doc: "The common name of the software to be configured." jsonldPredicate: "@id" - name: version type: string[]? doc: "The (optional) version of the software to configured." - name: specs type: string[]? doc: | Must be one or more IRIs identifying resources for installing or enabling the software. Implementations may provide resolvers which map well-known software spec IRIs to some configuration action. For example, an IRI `https://packages.debian.org/jessie/bowtie` could be resolved with `apt-get install bowtie`. An IRI `https://anaconda.org/bioconda/bowtie` could be resolved with `conda install -c bioconda bowtie`. Tools may also provide IRIs to index entries such as [RRID](http://www.identifiers.org/rrid/), such as `http://identifiers.org/rrid/RRID:SCR_005476` - name: Dirent type: record doc: | Define a file or subdirectory that must be placed in the designated output directory prior to executing the command line tool. May be the result of executing an expression, such as building a configuration file from a template. fields: - name: entryname type: ["null", string, Expression] jsonldPredicate: _id: cwl:entryname doc: | The name of the file or subdirectory to create in the output directory. If `entry` is a File or Directory, this overrides `basename`. Optional. - name: entry type: [string, Expression] jsonldPredicate: _id: cwl:entry doc: | If the value is a string literal or an expression which evaluates to a string, a new file must be created with the string as the file contents. If the value is an expression that evaluates to a `File` object, this indicates the referenced file should be added to the designated output directory prior to executing the tool. If the value is an expression that evaluates to a `Dirent` object, this indicates that the File or Directory in `entry` should be added to the designated output directory with the name in `entryname`. If `writable` is false, the file may be made available using a bind mount or file system link to avoid unnecessary copying of the input file. - name: writable type: boolean? doc: | If true, the file or directory must be writable by the tool. Changes to the file or directory must be isolated and not visible by any other CommandLineTool process. This may be implemented by making a copy of the original file or directory. Default false (files and directories read-only by default). - name: InitialWorkDirRequirement type: record extends: ProcessRequirement doc: Define a list of files and subdirectories that must be created by the workflow platform in the designated output directory prior to executing the command line tool. fields: - name: class type: string doc: InitialWorkDirRequirement jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: listing type: - type: array items: [File, Directory, Dirent, string, Expression] - string - Expression jsonldPredicate: _id: "cwl:listing" doc: | The list of files or subdirectories that must be placed in the designated output directory prior to executing the command line tool. May be an expression. If so, the expression return value must validate as `{type: array, items: [File, Directory]}`. - name: EnvVarRequirement type: record extends: ProcessRequirement doc: | Define a list of environment variables which will be set in the execution environment of the tool. See `EnvironmentDef` for details. fields: - name: class type: string doc: "Always 'EnvVarRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: envDef type: EnvironmentDef[] doc: The list of environment variables. jsonldPredicate: mapSubject: envName mapPredicate: envValue - type: record name: ShellCommandRequirement extends: ProcessRequirement doc: | Modify the behavior of CommandLineTool to generate a single string containing a shell command line. Each item in the argument list must be joined into a string separated by single spaces and quoted to prevent intepretation by the shell, unless `CommandLineBinding` for that argument contains `shellQuote: false`. If `shellQuote: false` is specified, the argument is joined into the command string without quoting, which allows the use of shell metacharacters such as `|` for pipes. fields: - name: class type: string doc: "Always 'ShellCommandRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - type: record name: ResourceRequirement extends: ProcessRequirement doc: | Specify basic hardware resource requirements. "min" is the minimum amount of a resource that must be reserved to schedule a job. If "min" cannot be satisfied, the job should not be run. "max" is the maximum amount of a resource that the job shall be permitted to use. If a node has sufficient resources, multiple jobs may be scheduled on a single node provided each job's "max" resource requirements are met. If a job attempts to exceed its "max" resource allocation, an implementation may deny additional resources, which may result in job failure. If "min" is specified but "max" is not, then "max" == "min" If "max" is specified by "min" is not, then "min" == "max". It is an error if max < min. It is an error if the value of any of these fields is negative. If neither "min" nor "max" is specified for a resource, an implementation may provide a default. fields: - name: class type: string doc: "Always 'ResourceRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: coresMin type: ["null", long, string, Expression] doc: Minimum reserved number of CPU cores - name: coresMax type: ["null", int, string, Expression] doc: Maximum reserved number of CPU cores - name: ramMin type: ["null", long, string, Expression] doc: Minimum reserved RAM in mebibytes (2**20) - name: ramMax type: ["null", long, string, Expression] doc: Maximum reserved RAM in mebibytes (2**20) - name: tmpdirMin type: ["null", long, string, Expression] doc: Minimum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20) - name: tmpdirMax type: ["null", long, string, Expression] doc: Maximum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20) - name: outdirMin type: ["null", long, string, Expression] doc: Minimum reserved filesystem based storage for the designated output directory, in mebibytes (2**20) - name: outdirMax type: ["null", long, string, Expression] doc: Maximum reserved filesystem based storage for the designated output directory, in mebibytes (2**20) cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/CommonWorkflowLanguage.yml0000644000175000017500000000032113036412053027365 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" sld: "https://w3id.org/cwl/salad#" $graph: - $import: Process.yml - $import: CommandLineTool.yml - $import: Workflow.yml cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/invocation.md0000644000175000017500000001732113036412053024716 0ustar michaelmichael00000000000000# Running a Command To accommodate the enormous variety in syntax and semantics for input, runtime environment, invocation, and output of arbitrary programs, a CommandLineTool defines an "input binding" that describes how to translate abstract input parameters to an concrete program invocation, and an "output binding" that describes how to generate output parameters from program output. ## Input binding The tool command line is built by applying command line bindings to the input object. Bindings are listed either as part of an [input parameter](#CommandInputParameter) using the `inputBinding` field, or separately using the `arguments` field of the CommandLineTool. The algorithm to build the command line is as follows. In this algorithm, the sort key is a list consisting of one or more numeric or string elements. Strings are sorted lexicographically based on UTF-8 encoding. 1. Collect `CommandLineBinding` objects from `arguments`. Assign a sorting key `[position, i]` where `position` is [`CommandLineBinding.position`](#CommandLineBinding) and `i` is the index in the `arguments` list. 2. Collect `CommandLineBinding` objects from the `inputs` schema and associate them with values from the input object. Where the input type is a record, array, or map, recursively walk the schema and input object, collecting nested `CommandLineBinding` objects and associating them with values from the input object. 3. Create a sorting key by taking the value of the `position` field at each level leading to each leaf binding object. If `position` is not specified, it is not added to the sorting key. For bindings on arrays and maps, the sorting key must include the array index or map key following the position. If and only if two bindings have the same sort key, the tie must be broken using the ordering of the field or parameter name immediately containing the leaf binding. 4. Sort elements using the assigned sorting keys. Numeric entries sort before strings. 5. In the sorted order, apply the rules defined in [`CommandLineBinding`](#CommandLineBinding) to convert bindings to actual command line elements. 6. Insert elements from `baseCommand` at the beginning of the command line. ## Runtime environment All files listed in the input object must be made available in the runtime environment. The implementation may use a shared or distributed file system or transfer files via explicit download to the host. Implementations may choose not to provide access to files not explicitly specified in the input object or process requirements. Output files produced by tool execution must be written to the **designated output directory**. The initial current working directory when executing the tool must be the designated output directory. Files may also be written to the **designated temporary directory**. This directory must be isolated and not shared with other processes. Any files written to the designated temporary directory may be automatically deleted by the workflow platform immediately after the tool terminates. For compatibility, files may be written to the **system temporary directory** which must be located at `/tmp`. Because the system temporary directory may be shared with other processes on the system, files placed in the system temporary directory are not guaranteed to be deleted automatically. A tool must not use the system temporary directory as a backchannel communication with other tools. It is valid for the system temporary directory to be the same as the designated temporary directory. When executing the tool, the tool must execute in a new, empty environment with only the environment variables described below; the child process must not inherit environment variables from the parent process except as specified or at user option. * `HOME` must be set to the designated output directory. * `TMPDIR` must be set to the designated temporary directory. * `PATH` may be inherited from the parent process, except when run in a container that provides its own `PATH`. * Variables defined by [EnvVarRequirement](#EnvVarRequirement) * The default environment of the container, such as when using [DockerRequirement](#DockerRequirement) An implementation may forbid the tool from writing to any location in the runtime environment file system other than the designated temporary directory, system temporary directory, and designated output directory. An implementation may provide read-only input files, and disallow in-place update of input files. The designated temporary directory, system temporary directory and designated output directory may each reside on different mount points on different file systems. An implementation may forbid the tool from directly accessing network resources. Correct tools must not assume any network access. Future versions of the specification may incorporate optional process requirements that describe the networking needs of a tool. The `runtime` section available in [parameter references](#Parameter_references) and [expressions](#Expressions) contains the following fields. As noted earlier, an implementation may perform deferred resolution of runtime fields by providing opaque strings for any or all of the following fields; parameter references and expressions may only use the literal string value of the field and must not perform computation on the contents. * `runtime.outdir`: an absolute path to the designated output directory * `runtime.tmpdir`: an absolute path to the designated temporary directory * `runtime.cores`: number of CPU cores reserved for the tool process * `runtime.ram`: amount of RAM in mebibytes (2\*\*20) reserved for the tool process * `runtime.outdirSize`: reserved storage space available in the designated output directory * `runtime.tmpdirSize`: reserved storage space available in the designated temporary directory For `cores`, `ram`, `outdirSize` and `tmpdirSize`, if an implementation can't provide the actual number of reserved cores during the expression evaluation time, it should report back the minimal requested amount. See [ResourceRequirement](#ResourceRequirement) for details on how to describe the hardware resources required by a tool. The standard input stream and standard output stream may be redirected as described in the `stdin` and `stdout` fields. ## Execution Once the command line is built and the runtime environment is created, the actual tool is executed. The standard error stream and standard output stream (unless redirected by setting `stdout` or `stderr`) may be captured by platform logging facilities for storage and reporting. Tools may be multithreaded or spawn child processes; however, when the parent process exits, the tool is considered finished regardless of whether any detached child processes are still running. Tools must not require any kind of console, GUI, or web based user interaction in order to start and run to completion. The exit code of the process indicates if the process completed successfully. By convention, an exit code of zero is treated as success and non-zero exit codes are treated as failure. This may be customized by providing the fields `successCodes`, `temporaryFailCodes`, and `permanentFailCodes`. An implementation may choose to default unspecified non-zero exit codes to either `temporaryFailure` or `permanentFailure`. ## Output binding If the output directory contains a file named "cwl.output.json", that file must be loaded and used as the output object. Otherwise, the output object must be generated by walking the parameters listed in `outputs` and applying output bindings to the tool output. Output bindings are associated with output parameters using the `outputBinding` field. See [`CommandOutputBinding`](#CommandOutputBinding) for details. cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/userguide-intro.md0000644000175000017500000000240013036412053025662 0ustar michaelmichael00000000000000# A Gentle Introduction to the Common Workflow Language Hello! This guide will introduce you to writing tool wrappers and workflows using the Common Workflow Language (CWL). This guide describes the current development specification, version 1.1.0-dev1. Note: This document is a work in progress. Not all features are covered, yet. # Introduction CWL is a way to describe command line tools and connect them together to create workflows. Because CWL is a specification and not a specific piece of software, tools and workflows described using CWL are portable across a variety of platforms that support the CWL standard. CWL has roots in "make" and many similar tools that determine order of execution based on dependencies between tasks. However unlike "make", CWL tasks are isolated and you must be explicit about your inputs and outputs. The benefit of explicitness and isolation are flexibility, portability, and scalability: tools and workflows described with CWL can transparently leverage technologies such as Docker, be used with CWL implementations from different vendors, and is well suited for describing large-scale workflows in cluster, cloud and high performance computing environments where tasks are scheduled in parallel across many nodes. cwltool-1.0.20170114120503/cwltool/schemas/v1.1.0-dev1/CommandLineTool-standalone.yml0000644000175000017500000000006513036412053030115 0ustar michaelmichael00000000000000- $import: Process.yml - $import: CommandLineTool.ymlcwltool-1.0.20170114120503/cwltool/schemas/v1.0/0000755000175000017500000000000013036412107021223 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/0000755000175000017500000000000013036412107022307 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/0000755000175000017500000000000013036412107024713 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/0000755000175000017500000000000013036412107027022 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/field_name_schema.yml0000644000175000017500000000040113036412052033142 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "ExampleType", "type": "record", "fields": [{ "name": "base", "type": "string", "jsonldPredicate": "http://example.com/base" }] }] } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/field_name_src.yml0000644000175000017500000000025313036412052032476 0ustar michaelmichael00000000000000 { "base": "one", "form": { "http://example.com/base": "two", "http://example.com/three": "three", }, "acid:four": "four" } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/link_res_src.yml0000644000175000017500000000047113036412052032223 0ustar michaelmichael00000000000000{ "$base": "http://example.com/base", "link": "http://example.com/base/zero", "form": { "link": "one", "things": [ { "link": "two" }, { "link": "#three", }, { "link": "four#five", }, { "link": "acid:six", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/ident_res.yml0000644000175000017500000000323413036412052031522 0ustar michaelmichael00000000000000- | ## Identifier resolution The schema may designate one or more fields as identifier fields to identify specific objects. Processing must resolve relative identifiers to absolute identifiers using the following rules: * If an identifier URI is prefixed with `#` it is a URI relative fragment identifier. It is resolved relative to the base URI by setting or replacing the fragment portion of the base URI. * If an identifier URI does not contain a scheme and is not prefixed `#` it is a parent relative fragment identifier. It is resolved relative to the base URI by the following rule: if the base URI does not contain a document fragment, set the fragment portion of the base URI. If the base URI does contain a document fragment, append a slash `/` followed by the identifier field to the fragment portion of the base URI. * If an identifier URI begins with a namespace prefix declared in `$namespaces` followed by a colon `:`, the prefix and colon must be replaced by the namespace declared in `$namespaces`. * If an identifier URI is an absolute URI consisting of a scheme and path, no processing occurs. When preprocessing visits a node containing an identifier, that identifier must be used as the base URI to process child nodes. It is an error for more than one object in a document to have the same absolute URI. ### Identifier resolution example Given the following schema: ``` - $include: ident_res_schema.yml - | ``` Process the following example: ``` - $include: ident_res_src.yml - | ``` This becomes: ``` - $include: ident_res_proc.yml - | ``` cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/vocab_res_proc.yml0000644000175000017500000000036313036412052032534 0ustar michaelmichael00000000000000 { "form": { "things": [ { "voc": "red", }, { "voc": "red", }, { "voc": "http://example.com/acid#blue", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/link_res_proc.yml0000644000175000017500000000063313036412052032377 0ustar michaelmichael00000000000000{ "$base": "http://example.com/base", "link": "http://example.com/base/zero", "form": { "link": "http://example.com/one", "things": [ { "link": "http://example.com/two" }, { "link": "http://example.com/base#three" }, { "link": "http://example.com/four#five", }, { "link": "http://example.com/acid#six", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/ident_res_src.yml0000644000175000017500000000052213036412052032366 0ustar michaelmichael00000000000000 { "id": "http://example.com/base", "form": { "id": "one", "things": [ { "id": "two" }, { "id": "#three", }, { "id": "four#five", }, { "id": "acid:six", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/vocab_res_schema.yml0000644000175000017500000000053113036412052033026 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "Colors", "type": "enum", "symbols": ["acid:red"] }, { "name": "ExampleType", "type": "record", "fields": [{ "name": "voc", "type": "string", "jsonldPredicate": { "_type": "@vocab" } }] }] } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/field_name.yml0000644000175000017500000000246013036412052031631 0ustar michaelmichael00000000000000- | ## Field name resolution The document schema declares the vocabulary of known field names. During preprocessing traversal, field name in the document which are not part of the schema vocabulary must be resolved to absolute URIs. Under "strict" validation, it is an error for a document to include fields which are not part of the vocabulary and not resolvable to absolute URIs. Fields names which are not part of the vocabulary are resolved using the following rules: * If an field name URI begins with a namespace prefix declared in the document context (`@context`) followed by a colon `:`, the prefix and colon must be replaced by the namespace declared in `@context`. * If there is a vocabulary term which maps to the URI of a resolved field, the field name must be replace with the vocabulary term. * If a field name URI is an absolute URI consisting of a scheme and path and is not part of the vocabulary, no processing occurs. Field name resolution is not relative. It must not be affected by the base URI. ### Field name resolution example Given the following schema: ``` - $include: field_name_schema.yml - | ``` Process the following example: ``` - $include: field_name_src.yml - | ``` This becomes: ``` - $include: field_name_proc.yml - | ``` cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/vocab_res.yml0000644000175000017500000000142313036412052031507 0ustar michaelmichael00000000000000- | ## Vocabulary resolution The schema may designate one or more vocabulary fields which use terms defined in the vocabulary. Processing must resolve vocabulary fields to either vocabulary terms or absolute URIs by first applying the link resolution rules defined above, then applying the following additional rule: * If a reference URI is a vocabulary field, and there is a vocabulary term which maps to the resolved URI, the reference must be replace with the vocabulary term. ### Vocabulary resolution example Given the following schema: ``` - $include: vocab_res_schema.yml - | ``` Process the following example: ``` - $include: vocab_res_src.yml - | ``` This becomes: ``` - $include: vocab_res_proc.yml - | ``` cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/metaschema_base.yml0000644000175000017500000000702413036412052032650 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/salad#" $namespaces: sld: "https://w3id.org/cwl/salad#" dct: "http://purl.org/dc/terms/" rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" rdfs: "http://www.w3.org/2000/01/rdf-schema#" xsd: "http://www.w3.org/2001/XMLSchema#" $graph: - name: PrimitiveType type: enum symbols: - "sld:null" - "xsd:boolean" - "xsd:int" - "xsd:long" - "xsd:float" - "xsd:double" - "xsd:string" doc: - | Salad data types are based on Avro schema declarations. Refer to the [Avro schema declaration documentation](https://avro.apache.org/docs/current/spec.html#schemas) for detailed information. - "null: no value" - "boolean: a binary value" - "int: 32-bit signed integer" - "long: 64-bit signed integer" - "float: single precision (32-bit) IEEE 754 floating-point number" - "double: double precision (64-bit) IEEE 754 floating-point number" - "string: Unicode character sequence" - name: Any type: enum symbols: ["#Any"] doc: | The **Any** type validates for any non-null value. - name: RecordField type: record doc: A field of a record. fields: - name: name type: string jsonldPredicate: "@id" doc: | The name of the field - name: doc type: string? doc: | A documentation string for this field jsonldPredicate: "rdfs:comment" - name: type type: - PrimitiveType - RecordSchema - EnumSchema - ArraySchema - string - type: array items: - PrimitiveType - RecordSchema - EnumSchema - ArraySchema - string jsonldPredicate: _id: sld:type _type: "@vocab" typeDSL: true refScope: 2 doc: | The field type - name: RecordSchema type: record fields: type: doc: "Must be `record`" type: name: Record_symbol type: enum symbols: - "sld:record" jsonldPredicate: _id: "sld:type" _type: "@vocab" typeDSL: true refScope: 2 fields: type: RecordField[]? jsonldPredicate: _id: sld:fields mapSubject: name mapPredicate: type doc: "Defines the fields of the record." - name: EnumSchema type: record doc: | Define an enumerated type. fields: type: doc: "Must be `enum`" type: name: Enum_symbol type: enum symbols: - "sld:enum" jsonldPredicate: _id: "sld:type" _type: "@vocab" typeDSL: true refScope: 2 symbols: type: string[] jsonldPredicate: _id: "sld:symbols" _type: "@id" identity: true doc: "Defines the set of valid symbols." - name: ArraySchema type: record fields: type: doc: "Must be `array`" type: name: Array_symbol type: enum symbols: - "sld:array" jsonldPredicate: _id: "sld:type" _type: "@vocab" typeDSL: true refScope: 2 items: type: - PrimitiveType - RecordSchema - EnumSchema - ArraySchema - string - type: array items: - PrimitiveType - RecordSchema - EnumSchema - ArraySchema - string jsonldPredicate: _id: "sld:items" _type: "@vocab" refScope: 2 doc: "Defines the type of the array elements." cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/field_name_proc.yml0000644000175000017500000000025313036412052032652 0ustar michaelmichael00000000000000 { "base": "one", "form": { "base": "two", "http://example.com/three": "three", }, "http://example.com/acid#four": "four" } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/ident_res_schema.yml0000644000175000017500000000035313036412052033041 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "ExampleType", "type": "record", "fields": [{ "name": "id", "type": "string", "jsonldPredicate": "@id" }] }] } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/import_include.md0000644000175000017500000001076313036412052032367 0ustar michaelmichael00000000000000## Import During preprocessing traversal, an implementation must resolve `$import` directives. An `$import` directive is an object consisting of exactly one field `$import` specifying resource by URI string. It is an error if there are additional fields in the `$import` object, such additional fields must be ignored. The URI string must be resolved to an absolute URI using the link resolution rules described previously. Implementations must support loading from `file`, `http` and `https` resources. The URI referenced by `$import` must be loaded and recursively preprocessed as a Salad document. The external imported document does not inherit the context of the importing document, and the default base URI for processing the imported document must be the URI used to retrieve the imported document. If the `$import` URI includes a document fragment, the fragment must be excluded from the base URI used to preprocess the imported document. Once loaded and processed, the `$import` node is replaced in the document structure by the object or array yielded from the import operation. URIs may reference document fragments which refer to specific an object in the target document. This indicates that the `$import` node must be replaced by only the object with the appropriate fragment identifier. It is a fatal error if an import directive refers to an external resource or resource fragment which does not exist or is not accessible. ### Import example import.yml: ``` { "hello": "world" } ``` parent.yml: ``` { "form": { "bar": { "$import": "import.yml" } } } ``` This becomes: ``` { "form": { "bar": { "hello": "world" } } } ``` ## Include During preprocessing traversal, an implementation must resolve `$include` directives. An `$include` directive is an object consisting of exactly one field `$include` specifying a URI string. It is an error if there are additional fields in the `$include` object, such additional fields must be ignored. The URI string must be resolved to an absolute URI using the link resolution rules described previously. The URI referenced by `$include` must be loaded as a text data. Implementations must support loading from `file`, `http` and `https` resources. Implementations may transcode the character encoding of the text data to match that of the parent document, but must not interpret or parse the text document in any other way. Once loaded, the `$include` node is replaced in the document structure by a string containing the text data loaded from the resource. It is a fatal error if an import directive refers to an external resource which does not exist or is not accessible. ### Include example parent.yml: ``` { "form": { "bar": { "$include": "include.txt" } } } ``` include.txt: ``` hello world ``` This becomes: ``` { "form": { "bar": "hello world" } } ``` ## Mixin During preprocessing traversal, an implementation must resolve `$mixin` directives. An `$mixin` directive is an object consisting of the field `$mixin` specifying resource by URI string. If there are additional fields in the `$mixin` object, these fields override fields in the object which is loaded from the `$mixin` URI. The URI string must be resolved to an absolute URI using the link resolution rules described previously. Implementations must support loading from `file`, `http` and `https` resources. The URI referenced by `$mixin` must be loaded and recursively preprocessed as a Salad document. The external imported document must inherit the context of the importing document, however the file URI for processing the imported document must be the URI used to retrieve the imported document. The `$mixin` URI must not include a document fragment. Once loaded and processed, the `$mixin` node is replaced in the document structure by the object or array yielded from the import operation. URIs may reference document fragments which refer to specific an object in the target document. This indicates that the `$mixin` node must be replaced by only the object with the appropriate fragment identifier. It is a fatal error if an import directive refers to an external resource or resource fragment which does not exist or is not accessible. ### Mixin example mixin.yml: ``` { "hello": "world", "carrot": "orange" } ``` parent.yml: ``` { "form": { "bar": { "$mixin": "mixin.yml" "carrot": "cake" } } } ``` This becomes: ``` { "form": { "bar": { "hello": "world", "carrot": "cake" } } } ``` cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/vocab_res_src.yml0000644000175000017500000000041313036412052032354 0ustar michaelmichael00000000000000 { "form": { "things": [ { "voc": "red", }, { "voc": "http://example.com/acid#red", }, { "voc": "http://example.com/acid#blue", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/link_res_schema.yml0000644000175000017500000000041013036412052032665 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "ExampleType", "type": "record", "fields": [{ "name": "link", "type": "string", "jsonldPredicate": { "_type": "@id" } }] }] } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/salad.md0000644000175000017500000002435313036412052030436 0ustar michaelmichael00000000000000# Semantic Annotations for Linked Avro Data (SALAD) Author: * Peter Amstutz , Curoverse Contributors: * The developers of Apache Avro * The developers of JSON-LD * Nebojša Tijanić , Seven Bridges Genomics # Abstract Salad is a schema language for describing structured linked data documents in JSON or YAML documents. A Salad schema provides rules for preprocessing, structural validation, and link checking for documents described by a Salad schema. Salad builds on JSON-LD and the Apache Avro data serialization system, and extends Avro with features for rich data modeling such as inheritance, template specialization, object identifiers, and object references. Salad was developed to provide a bridge between the record oriented data modeling supported by Apache Avro and the Semantic Web. # Status of This Document This document is the product of the [Common Workflow Language working group](https://groups.google.com/forum/#!forum/common-workflow-language). The latest version of this document is available in the "schema_salad" directory at https://github.com/common-workflow-language/schema_salad The products of the CWL working group (including this document) are made available under the terms of the Apache License, version 2.0. # Introduction The JSON data model is an extremely popular way to represent structured data. It is attractive because of it's relative simplicity and is a natural fit with the standard types of many programming languages. However, this simplicity means that basic JSON lacks expressive features useful for working with complex data structures and document formats, such as schemas, object references, and namespaces. JSON-LD is a W3C standard providing a way to describe how to interpret a JSON document as Linked Data by means of a "context". JSON-LD provides a powerful solution for representing object references and namespaces in JSON based on standard web URIs, but is not itself a schema language. Without a schema providing a well defined structure, it is difficult to process an arbitrary JSON-LD document as idiomatic JSON because there are many ways to express the same data that are logically equivalent but structurally distinct. Several schema languages exist for describing and validating JSON data, such as the Apache Avro data serialization system, however none understand linked data. As a result, to fully take advantage of JSON-LD to build the next generation of linked data applications, one must maintain separate JSON schema, JSON-LD context, RDF schema, and human documentation, despite significant overlap of content and obvious need for these documents to stay synchronized. Schema Salad is designed to address this gap. It provides a schema language and processing rules for describing structured JSON content permitting URI resolution and strict document validation. The schema language supports linked data through annotations that describe the linked data interpretation of the content, enables generation of JSON-LD context and RDF schema, and production of RDF triples by applying the JSON-LD context. The schema language also provides for robust support of inline documentation. ## Introduction to draft 1 This is the first version of Schema Salad. It is developed concurrently with draft 3 of the Common Workflow Language for use in specifying the Common Workflow Language, however Schema Salad is intended to be useful to a broader audience. ## References to Other Specifications **Javascript Object Notation (JSON)**: http://json.org **JSON Linked Data (JSON-LD)**: http://json-ld.org **YAML**: http://yaml.org **Avro**: https://avro.apache.org/docs/current/spec.html **Uniform Resource Identifier (URI) Generic Syntax**: https://tools.ietf.org/html/rfc3986) **Resource Description Framework (RDF)**: http://www.w3.org/RDF/ **UTF-8**: https://www.ietf.org/rfc/rfc2279.txt) ## Scope This document describes the syntax, data model, algorithms, and schema language for working with Salad documents. It is not intended to document a specific implementation of Salad, however it may serve as a reference for the behavior of conforming implementations. ## Terminology The terminology used to describe Salad documents is defined in the Concepts section of the specification. The terms defined in the following list are used in building those definitions and in describing the actions of an Salad implementation: **may**: Conforming Salad documents and Salad implementations are permitted but not required to be interpreted as described. **must**: Conforming Salad documents and Salad implementations are required to be interpreted as described; otherwise they are in error. **error**: A violation of the rules of this specification; results are undefined. Conforming implementations may detect and report an error and may recover from it. **fatal error**: A violation of the rules of this specification; results are undefined. Conforming implementations must not continue to process the document and may report an error. **at user option**: Conforming software may or must (depending on the modal verb in the sentence) behave as described; if it does, it must provide users a means to enable or disable the behavior described. # Document model ## Data concepts An **object** is a data structure equivalent to the "object" type in JSON, consisting of a unordered set of name/value pairs (referred to here as **fields**) and where the name is a string and the value is a string, number, boolean, array, or object. A **document** is a file containing a serialized object, or an array of objects. A **document type** is a class of files that share a common structure and semantics. A **document schema** is a formal description of the grammar of a document type. A **base URI** is a context-dependent URI used to resolve relative references. An **identifier** is a URI that designates a single document or single object within a document. A **vocabulary** is the set of symbolic field names and enumerated symbols defined by a document schema, where each term maps to absolute URI. ## Syntax Conforming Salad documents are serialized and loaded using YAML syntax and UTF-8 text encoding. Salad documents are written using the JSON-compatible subset of YAML. Features of YAML such as headers and type tags that are not found in the standard JSON data model must not be used in conforming Salad documents. It is a fatal error if the document is not valid YAML. A Salad document must consist only of either a single root object or an array of objects. ## Document context ### Implied context The implicit context consists of the vocabulary defined by the schema and the base URI. By default, the base URI must be the URI that was used to load the document. It may be overridden by an explicit context. ### Explicit context If a document consists of a root object, this object may contain the fields `$base`, `$namespaces`, `$schemas`, and `$graph`: * `$base`: Must be a string. Set the base URI for the document used to resolve relative references. * `$namespaces`: Must be an object with strings as values. The keys of the object are namespace prefixes used in the document; the values of the object are the prefix expansions. * `$schemas`: Must be an array of strings. This field may list URI references to documents in RDF-XML format which will be queried for RDF schema data. The subjects and predicates described by the RDF schema may provide additional semantic context for the document, and may be used for validation of prefixed extension fields found in the document. Other directives beginning with `$` must be ignored. ## Document graph If a document consists of a single root object, this object may contain the field `$graph`. This field must be an array of objects. If present, this field holds the primary content of the document. A document that consists of array of objects at the root is an implicit graph. ## Document metadata If a document consists of a single root object, metadata about the document, such as authorship, may be declared in the root object. ## Document schema Document preprocessing, link validation and schema validation require a document schema. A schema may consist of: * At least one record definition object which defines valid fields that make up a record type. Record field definitions include the valid types that may be assigned to each field and annotations to indicate fields that represent identifiers and links, described below in "Semantic Annotations". * Any number of enumerated type objects which define a set of finite set of symbols that are valid value of the type. * Any number of documentation objects which allow in-line documentation of the schema. The schema for defining a salad schema (the metaschema) is described in detail in "Schema validation". ### Record field annotations In a document schema, record field definitions may include the field `jsonldPredicate`, which may be either a string or object. Implementations must use the following document preprocessing of fields by the following rules: * If the value of `jsonldPredicate` is `@id`, the field is an identifier field. * If the value of `jsonldPredicate` is an object, and contains that object contains the field `_type` with the value `@id`, the field is a link field. * If the value of `jsonldPredicate` is an object, and contains that object contains the field `_type` with the value `@vocab`, the field is a vocabulary field, which is a subtype of link field. ## Document traversal To perform document document preprocessing, link validation and schema validation, the document must be traversed starting from the fields or array items of the root object or array and recursively visiting each child item which contains an object or arrays. # Document preprocessing After processing the explicit context (if any), document preprocessing begins. Starting from the document root, object fields values or array items which contain objects or arrays are recursively traversed depth-first. For each visited object, field names, identifier fields, link fields, vocabulary fields, and `$import` and `$include` directives must be processed as described in this section. The order of traversal of child nodes within a parent node is undefined. cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/metaschema.yml0000644000175000017500000002271713036412052031664 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/salad#" $namespaces: sld: "https://w3id.org/cwl/salad#" dct: "http://purl.org/dc/terms/" rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" rdfs: "http://www.w3.org/2000/01/rdf-schema#" xsd: "http://www.w3.org/2001/XMLSchema#" $graph: - name: "Semantic_Annotations_for_Linked_Avro_Data" type: documentation doc: - $include: salad.md - $import: field_name.yml - $import: ident_res.yml - $import: link_res.yml - $import: vocab_res.yml - $include: import_include.md - name: "Link_Validation" type: documentation doc: | # Link validation Once a document has been preprocessed, an implementation may validate links. The link validation traversal may visit fields which the schema designates as link fields and check that each URI references an existing object in the current document, an imported document, file system, or network resource. Failure to validate links may be a fatal error. Link validation behavior for individual fields may be modified by `identity` and `noLinkCheck` in the `jsonldPredicate` section of the field schema. - name: "Schema_validation" type: documentation doc: "" # - name: "JSON_LD_Context" # type: documentation # doc: | # # Generating JSON-LD Context # How to generate the json-ld context... - $import: metaschema_base.yml - name: JsonldPredicate type: record doc: | Attached to a record field to define how the parent record field is handled for URI resolution and JSON-LD context generation. fields: - name: _id type: string? jsonldPredicate: _id: sld:_id _type: "@id" identity: true doc: | The predicate URI that this field corresponds to. Corresponds to JSON-LD `@id` directive. - name: _type type: string? doc: | The context type hint, corresponds to JSON-LD `@type` directive. * If the value of this field is `@id` and `identity` is false or unspecified, the parent field must be resolved using the link resolution rules. If `identity` is true, the parent field must be resolved using the identifier expansion rules. * If the value of this field is `@vocab`, the parent field must be resolved using the vocabulary resolution rules. - name: _container type: string? doc: | Structure hint, corresponds to JSON-LD `@container` directive. - name: identity type: boolean? doc: | If true and `_type` is `@id` this indicates that the parent field must be resolved according to identity resolution rules instead of link resolution rules. In addition, the field value is considered an assertion that the linked value exists; absence of an object in the loaded document with the URI is not an error. - name: noLinkCheck type: boolean? doc: | If true, this indicates that link validation traversal must stop at this field. This field (it is is a URI) or any fields under it (if it is an object or array) are not subject to link checking. - name: mapSubject type: string? doc: | If the value of the field is a JSON object, it must be transformed into an array of JSON objects, where each key-value pair from the source JSON object is a list item, the list items must be JSON objects, and the key is assigned to the field specified by `mapSubject`. - name: mapPredicate type: string? doc: | Only applies if `mapSubject` is also provided. If the value of the field is a JSON object, it is transformed as described in `mapSubject`, with the addition that when the value of a map item is not an object, the item is transformed to a JSON object with the key assigned to the field specified by `mapSubject` and the value assigned to the field specified by `mapPredicate`. - name: refScope type: int? doc: | If the field contains a relative reference, it must be resolved by searching for valid document references in each successive parent scope in the document fragment. For example, a reference of `foo` in the context `#foo/bar/baz` will first check for the existence of `#foo/bar/baz/foo`, followed by `#foo/bar/foo`, then `#foo/foo` and then finally `#foo`. The first valid URI in the search order shall be used as the fully resolved value of the identifier. The value of the refScope field is the specified number of levels from the containing identifer scope before starting the search, so if `refScope: 2` then "baz" and "bar" must be stripped to get the base `#foo` and search `#foo/foo` and the `#foo`. The last scope searched must be the top level scope before determining if the identifier cannot be resolved. - name: typeDSL type: boolean? doc: | Field must be expanded based on the the Schema Salad type DSL. - name: SpecializeDef type: record fields: - name: specializeFrom type: string doc: "The data type to be replaced" jsonldPredicate: _id: "sld:specializeFrom" _type: "@id" refScope: 1 - name: specializeTo type: string doc: "The new data type to replace with" jsonldPredicate: _id: "sld:specializeTo" _type: "@id" refScope: 1 - name: NamedType type: record abstract: true fields: - name: name type: string jsonldPredicate: "@id" doc: "The identifier for this type" - name: DocType type: record abstract: true fields: - name: doc type: - string? - string[]? doc: "A documentation string for this type, or an array of strings which should be concatenated." jsonldPredicate: "rdfs:comment" - name: docParent type: string? doc: | Hint to indicate that during documentation generation, documentation for this type should appear in a subsection under `docParent`. jsonldPredicate: _id: "sld:docParent" _type: "@id" - name: docChild type: - string? - string[]? doc: | Hint to indicate that during documentation generation, documentation for `docChild` should appear in a subsection under this type. jsonldPredicate: _id: "sld:docChild" _type: "@id" - name: docAfter type: string? doc: | Hint to indicate that during documentation generation, documentation for this type should appear after the `docAfter` section at the same level. jsonldPredicate: _id: "sld:docAfter" _type: "@id" - name: SchemaDefinedType type: record extends: DocType doc: | Abstract base for schema-defined types. abstract: true fields: - name: jsonldPredicate type: - string? - JsonldPredicate? doc: | Annotate this type with linked data context. jsonldPredicate: sld:jsonldPredicate - name: documentRoot type: boolean? doc: | If true, indicates that the type is a valid at the document root. At least one type in a schema must be tagged with `documentRoot: true`. - name: SaladRecordField type: record extends: RecordField doc: "A field of a record." fields: - name: jsonldPredicate type: - string? - JsonldPredicate? doc: | Annotate this type with linked data context. jsonldPredicate: "sld:jsonldPredicate" - name: SaladRecordSchema type: record extends: [NamedType, RecordSchema, SchemaDefinedType] documentRoot: true specialize: RecordField: SaladRecordField fields: - name: abstract type: boolean? doc: | If true, this record is abstract and may be used as a base for other records, but is not valid on its own. - name: extends type: - string? - string[]? jsonldPredicate: _id: "sld:extends" _type: "@id" refScope: 1 doc: | Indicates that this record inherits fields from one or more base records. - name: specialize type: - SpecializeDef[]? doc: | Only applies if `extends` is declared. Apply type specialization using the base record as a template. For each field inherited from the base record, replace any instance of the type `specializeFrom` with `specializeTo`. jsonldPredicate: _id: "sld:specialize" mapSubject: specializeFrom mapPredicate: specializeTo - name: SaladEnumSchema type: record extends: [EnumSchema, SchemaDefinedType] documentRoot: true doc: | Define an enumerated type. fields: - name: extends type: - string? - string[]? jsonldPredicate: _id: "sld:extends" _type: "@id" refScope: 1 doc: | Indicates that this enum inherits symbols from a base enum. - name: Documentation type: record extends: [NamedType, DocType] documentRoot: true doc: | A documentation section. This type exists to facilitate self-documenting schemas but has no role in formal validation. fields: - name: type doc: "Must be `documentation`" type: name: Documentation_symbol type: enum symbols: - "sld:documentation" jsonldPredicate: _id: "sld:type" _type: "@vocab" typeDSL: true refScope: 2 cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/link_res.yml0000644000175000017500000000341613036412052031356 0ustar michaelmichael00000000000000- | ## Link resolution The schema may designate one or more fields as link fields reference other objects. Processing must resolve links to either absolute URIs using the following rules: * If a reference URI is prefixed with `#` it is a relative fragment identifier. It is resolved relative to the base URI by setting or replacing the fragment portion of the base URI. * If a reference URI does not contain a scheme and is not prefixed with `#` it is a path relative reference. If the reference URI contains `#` in any position other than the first character, the reference URI must be divided into a path portion and a fragment portion split on the first instance of `#`. The path portion is resolved relative to the base URI by the following rule: if the path portion of the base URI ends in a slash `/`, append the path portion of the reference URI to the path portion of the base URI. If the path portion of the base URI does not end in a slash, replace the final path segment with the path portion of the reference URI. Replace the fragment portion of the base URI with the fragment portion of the reference URI. * If a reference URI begins with a namespace prefix declared in `$namespaces` followed by a colon `:`, the prefix and colon must be replaced by the namespace declared in `$namespaces`. * If a reference URI is an absolute URI consisting of a scheme and path, no processing occurs. Link resolution must not affect the base URI used to resolve identifiers and other links. ### Link resolution example Given the following schema: ``` - $include: link_res_schema.yml - | ``` Process the following example: ``` - $include: link_res_src.yml - | ``` This becomes: ``` - $include: link_res_proc.yml - | ``` cwltool-1.0.20170114120503/cwltool/schemas/v1.0/salad/schema_salad/metaschema/ident_res_proc.yml0000644000175000017500000000056213036412052032546 0ustar michaelmichael00000000000000{ "id": "http://example.com/base", "form": { "id": "http://example.com/base#one", "things": [ { "id": "http://example.com/base#one/two" }, { "id": "http://example.com/base#three" }, { "id": "http://example.com/four#five", }, { "id": "http://example.com/acid#six", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/v1.0/Process.yml0000644000175000017500000006123213036412052023367 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" sld: "https://w3id.org/cwl/salad#" $graph: - name: "Common Workflow Language, v1.0" type: documentation doc: {$include: concepts.md} - $import: "salad/schema_salad/metaschema/metaschema_base.yml" - name: BaseTypesDoc type: documentation doc: | ## Base types docChild: - "#CWLType" - "#Process" - type: enum name: CWLVersion doc: "Version symbols for published CWL document versions." symbols: - cwl:draft-2 - cwl:draft-3.dev1 - cwl:draft-3.dev2 - cwl:draft-3.dev3 - cwl:draft-3.dev4 - cwl:draft-3.dev5 - cwl:draft-3 - cwl:draft-4.dev1 - cwl:draft-4.dev2 - cwl:draft-4.dev3 - cwl:v1.0.dev4 - cwl:v1.0 - name: CWLType type: enum extends: "sld:PrimitiveType" symbols: - cwl:File - cwl:Directory doc: - "Extends primitive types with the concept of a file and directory as a builtin type." - "File: A File object" - "Directory: A Directory object" - name: File type: record docParent: "#CWLType" doc: | Represents a file (or group of files if `secondaryFiles` is specified) that must be accessible by tools using standard POSIX file system call API such as open(2) and read(2). fields: - name: class type: type: enum name: File_class symbols: - cwl:File jsonldPredicate: _id: "@type" _type: "@vocab" doc: Must be `File` to indicate this object describes a file. - name: location type: string? doc: | An IRI that identifies the file resource. This may be a relative reference, in which case it must be resolved using the base IRI of the document. The location may refer to a local or remote resource; the implementation must use the IRI to retrieve file content. If an implementation is unable to retrieve the file content stored at a remote resource (due to unsupported protocol, access denied, or other issue) it must signal an error. If the `location` field is not provided, the `contents` field must be provided. The implementation must assign a unique identifier for the `location` field. If the `path` field is provided but the `location` field is not, an implementation may assign the value of the `path` field to `location`, then follow the rules above. jsonldPredicate: _id: "@id" _type: "@id" - name: path type: string? doc: | The local host path where the File is available when a CommandLineTool is executed. This field must be set by the implementation. The final path component must match the value of `basename`. This field must not be used in any other context. The command line tool being executed must be able to to access the file at `path` using the POSIX `open(2)` syscall. As a special case, if the `path` field is provided but the `location` field is not, an implementation may assign the value of the `path` field to `location`, and remove the `path` field. If the `path` contains [POSIX shell metacharacters](http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_02) (`|`,`&`, `;`, `<`, `>`, `(`,`)`, `$`,`` ` ``, `\`, `"`, `'`, ``, ``, and ``) or characters [not allowed](http://www.iana.org/assignments/idna-tables-6.3.0/idna-tables-6.3.0.xhtml) for [Internationalized Domain Names for Applications](https://tools.ietf.org/html/rfc6452) then implementations may terminate the process with a `permanentFailure`. jsonldPredicate: "_id": "cwl:path" "_type": "@id" - name: basename type: string? doc: | The base name of the file, that is, the name of the file without any leading directory path. The base name must not contain a slash `/`. If not provided, the implementation must set this field based on the `location` field by taking the final path component after parsing `location` as an IRI. If `basename` is provided, it is not required to match the value from `location`. When this file is made available to a CommandLineTool, it must be named with `basename`, i.e. the final component of the `path` field must match `basename`. jsonldPredicate: "cwl:basename" - name: dirname type: string? doc: | The name of the directory containing file, that is, the path leading up to the final slash in the path such that `dirname + '/' + basename == path`. The implementation must set this field based on the value of `path` prior to evaluating parameter references or expressions in a CommandLineTool document. This field must not be used in any other context. - name: nameroot type: string? doc: | The basename root such that `nameroot + nameext == basename`, and `nameext` is empty or begins with a period and contains at most one period. For the purposess of path splitting leading periods on the basename are ignored; a basename of `.cshrc` will have a nameroot of `.cshrc`. The implementation must set this field automatically based on the value of `basename` prior to evaluating parameter references or expressions. - name: nameext type: string? doc: | The basename extension such that `nameroot + nameext == basename`, and `nameext` is empty or begins with a period and contains at most one period. Leading periods on the basename are ignored; a basename of `.cshrc` will have an empty `nameext`. The implementation must set this field automatically based on the value of `basename` prior to evaluating parameter references or expressions. - name: checksum type: string? doc: | Optional hash code for validating file integrity. Currently must be in the form "sha1$ + hexadecimal string" using the SHA-1 algorithm. - name: size type: long? doc: Optional file size - name: "secondaryFiles" type: - "null" - type: array items: [File, Directory] jsonldPredicate: "cwl:secondaryFiles" doc: | A list of additional files that are associated with the primary file and must be transferred alongside the primary file. Examples include indexes of the primary file, or external references which must be included when loading primary document. A file object listed in `secondaryFiles` may itself include `secondaryFiles` for which the same rules apply. - name: format type: string? jsonldPredicate: _id: cwl:format _type: "@id" identity: true doc: | The format of the file: this must be an IRI of a concept node that represents the file format, preferrably defined within an ontology. If no ontology is available, file formats may be tested by exact match. Reasoning about format compatability must be done by checking that an input file format is the same, `owl:equivalentClass` or `rdfs:subClassOf` the format required by the input parameter. `owl:equivalentClass` is transitive with `rdfs:subClassOf`, e.g. if ` owl:equivalentClass ` and ` owl:subclassOf ` then infer ` owl:subclassOf `. File format ontologies may be provided in the "$schema" metadata at the root of the document. If no ontologies are specified in `$schema`, the runtime may perform exact file format matches. - name: contents type: string? doc: | File contents literal. Maximum of 64 KiB. If neither `location` nor `path` is provided, `contents` must be non-null. The implementation must assign a unique identifier for the `location` field. When the file is staged as input to CommandLineTool, the value of `contents` must be written to a file. If `loadContents` of `inputBinding` or `outputBinding` is true and `location` is valid, the implementation must read up to the first 64 KiB of text from the file and place it in the "contents" field. - name: Directory type: record docAfter: "#File" doc: | Represents a directory to present to a command line tool. fields: - name: class type: type: enum name: Directory_class symbols: - cwl:Directory jsonldPredicate: _id: "@type" _type: "@vocab" doc: Must be `Directory` to indicate this object describes a Directory. - name: location type: string? doc: | An IRI that identifies the directory resource. This may be a relative reference, in which case it must be resolved using the base IRI of the document. The location may refer to a local or remote resource. If the `listing` field is not set, the implementation must use the location IRI to retrieve directory listing. If an implementation is unable to retrieve the directory listing stored at a remote resource (due to unsupported protocol, access denied, or other issue) it must signal an error. If the `location` field is not provided, the `listing` field must be provided. The implementation must assign a unique identifier for the `location` field. If the `path` field is provided but the `location` field is not, an implementation may assign the value of the `path` field to `location`, then follow the rules above. jsonldPredicate: _id: "@id" _type: "@id" - name: path type: string? doc: | The local path where the Directory is made available prior to executing a CommandLineTool. This must be set by the implementation. This field must not be used in any other context. The command line tool being executed must be able to to access the directory at `path` using the POSIX `opendir(2)` syscall. If the `path` contains [POSIX shell metacharacters](http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_02) (`|`,`&`, `;`, `<`, `>`, `(`,`)`, `$`,`` ` ``, `\`, `"`, `'`, ``, ``, and ``) or characters [not allowed](http://www.iana.org/assignments/idna-tables-6.3.0/idna-tables-6.3.0.xhtml) for [Internationalized Domain Names for Applications](https://tools.ietf.org/html/rfc6452) then implementations may terminate the process with a `permanentFailure`. jsonldPredicate: _id: "cwl:path" _type: "@id" - name: basename type: string? doc: | The base name of the directory, that is, the name of the file without any leading directory path. The base name must not contain a slash `/`. If not provided, the implementation must set this field based on the `location` field by taking the final path component after parsing `location` as an IRI. If `basename` is provided, it is not required to match the value from `location`. When this file is made available to a CommandLineTool, it must be named with `basename`, i.e. the final component of the `path` field must match `basename`. jsonldPredicate: "cwl:basename" - name: listing type: - "null" - type: array items: [File, Directory] doc: | List of files or subdirectories contained in this directory. The name of each file or subdirectory is determined by the `basename` field of each `File` or `Directory` object. It is an error if a `File` shares a `basename` with any other entry in `listing`. If two or more `Directory` object share the same `basename`, this must be treated as equivalent to a single subdirectory with the listings recursively merged. jsonldPredicate: _id: "cwl:listing" - name: SchemaBase type: record abstract: true fields: - name: label type: - "null" - string jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this object." - name: Parameter type: record extends: SchemaBase abstract: true doc: | Define an input or output parameter to a process. fields: - name: secondaryFiles type: - "null" - string - Expression - type: array items: [string, Expression] jsonldPredicate: "cwl:secondaryFiles" doc: | Only valid when `type: File` or is an array of `items: File`. Describes files that must be included alongside the primary file(s). If the value is an expression, the value of `self` in the expression must be the primary input or output File to which this binding applies. If the value is a string, it specifies that the following pattern should be applied to the primary file: 1. If string begins with one or more caret `^` characters, for each caret, remove the last file extension from the path (the last period `.` and all following characters). If there are no file extensions, the path is unchanged. 2. Append the remainder of the string to the end of the file path. - name: format type: - "null" - string - type: array items: string - Expression jsonldPredicate: _id: cwl:format _type: "@id" identity: true doc: | Only valid when `type: File` or is an array of `items: File`. For input parameters, this must be one or more IRIs of concept nodes that represents file formats which are allowed as input to this parameter, preferrably defined within an ontology. If no ontology is available, file formats may be tested by exact match. For output parameters, this is the file format that will be assigned to the output parameter. - name: streamable type: boolean? doc: | Only valid when `type: File` or is an array of `items: File`. A value of `true` indicates that the file is read or written sequentially without seeking. An implementation may use this flag to indicate whether it is valid to stream file contents using a named pipe. Default: `false`. - name: doc type: - string? - string[]? doc: "A documentation string for this type, or an array of strings which should be concatenated." jsonldPredicate: "rdfs:comment" - type: enum name: Expression doc: | 'Expression' is not a real type. It indicates that a field must allow runtime parameter references. If [InlineJavascriptRequirement](#InlineJavascriptRequirement) is declared and supported by the platform, the field must also allow Javascript expressions. symbols: - cwl:ExpressionPlaceholder - name: InputBinding type: record abstract: true fields: - name: loadContents type: - "null" - boolean jsonldPredicate: "cwl:loadContents" doc: | Only valid when `type: File` or is an array of `items: File`. Read up to the first 64 KiB of text from the file and place it in the "contents" field of the file object for use by expressions. - name: OutputBinding type: record abstract: true - name: InputSchema extends: SchemaBase type: record abstract: true - name: OutputSchema extends: SchemaBase type: record abstract: true - name: InputRecordField type: record extends: "sld:RecordField" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: InputRecordSchema - specializeFrom: "sld:EnumSchema" specializeTo: InputEnumSchema - specializeFrom: "sld:ArraySchema" specializeTo: InputArraySchema - specializeFrom: "sld:PrimitiveType" specializeTo: CWLType fields: - name: inputBinding type: InputBinding? jsonldPredicate: "cwl:inputBinding" - name: label type: string? jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: InputRecordSchema type: record extends: ["sld:RecordSchema", InputSchema] specialize: - specializeFrom: "sld:RecordField" specializeTo: InputRecordField - name: InputEnumSchema type: record extends: ["sld:EnumSchema", InputSchema] fields: - name: inputBinding type: InputBinding? jsonldPredicate: "cwl:inputBinding" - name: InputArraySchema type: record extends: ["sld:ArraySchema", InputSchema] specialize: - specializeFrom: "sld:RecordSchema" specializeTo: InputRecordSchema - specializeFrom: "sld:EnumSchema" specializeTo: InputEnumSchema - specializeFrom: "sld:ArraySchema" specializeTo: InputArraySchema - specializeFrom: "sld:PrimitiveType" specializeTo: CWLType fields: - name: inputBinding type: InputBinding? jsonldPredicate: "cwl:inputBinding" - name: OutputRecordField type: record extends: "sld:RecordField" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: OutputRecordSchema - specializeFrom: "sld:EnumSchema" specializeTo: OutputEnumSchema - specializeFrom: "sld:ArraySchema" specializeTo: OutputArraySchema - specializeFrom: "sld:PrimitiveType" specializeTo: CWLType fields: - name: outputBinding type: OutputBinding? jsonldPredicate: "cwl:outputBinding" - name: OutputRecordSchema type: record extends: ["sld:RecordSchema", "#OutputSchema"] docParent: "#OutputParameter" specialize: - specializeFrom: "sld:RecordField" specializeTo: OutputRecordField - name: OutputEnumSchema type: record extends: ["sld:EnumSchema", OutputSchema] docParent: "#OutputParameter" fields: - name: outputBinding type: OutputBinding? jsonldPredicate: "cwl:outputBinding" - name: OutputArraySchema type: record extends: ["sld:ArraySchema", OutputSchema] docParent: "#OutputParameter" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: OutputRecordSchema - specializeFrom: "sld:EnumSchema" specializeTo: OutputEnumSchema - specializeFrom: "sld:ArraySchema" specializeTo: OutputArraySchema - specializeFrom: "sld:PrimitiveType" specializeTo: CWLType fields: - name: outputBinding type: OutputBinding? jsonldPredicate: "cwl:outputBinding" - name: InputParameter type: record extends: Parameter fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this parameter object." - name: inputBinding type: InputBinding? jsonldPredicate: "cwl:inputBinding" doc: | Describes how to handle the inputs of a process and convert them into a concrete form for execution, such as command line parameters. - name: default type: Any? jsonldPredicate: "cwl:default" doc: | The default value for this parameter if not provided in the input object. - name: type type: - "null" - CWLType - InputRecordSchema - InputEnumSchema - InputArraySchema - string - type: array items: - CWLType - InputRecordSchema - InputEnumSchema - InputArraySchema - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" refScope: 2 typeDSL: True doc: | Specify valid types of data that may be assigned to this parameter. - name: OutputParameter type: record extends: Parameter fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this parameter object." - name: outputBinding type: OutputBinding? jsonldPredicate: "cwl:outputBinding" doc: | Describes how to handle the outputs of a process. - type: record name: ProcessRequirement abstract: true doc: | A process requirement declares a prerequisite that may or must be fulfilled before executing a process. See [`Process.hints`](#process) and [`Process.requirements`](#process). Process requirements are the primary mechanism for specifying extensions to the CWL core specification. - type: record name: Process abstract: true doc: | The base executable type in CWL is the `Process` object defined by the document. Note that the `Process` object is abstract and cannot be directly executed. fields: - name: id type: string? jsonldPredicate: "@id" doc: "The unique identifier for this process object." - name: inputs type: type: array items: InputParameter jsonldPredicate: _id: "cwl:inputs" mapSubject: id mapPredicate: type doc: | Defines the input parameters of the process. The process is ready to run when all required input parameters are associated with concrete values. Input parameters include a schema for each parameter which is used to validate the input object. It may also be used to build a user interface for constructing the input object. - name: outputs type: type: array items: OutputParameter jsonldPredicate: _id: "cwl:outputs" mapSubject: id mapPredicate: type doc: | Defines the parameters representing the output of the process. May be used to generate and/or validate the output object. - name: requirements type: ProcessRequirement[]? jsonldPredicate: _id: "cwl:requirements" mapSubject: class doc: | Declares requirements that apply to either the runtime environment or the workflow engine that must be met in order to execute this process. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. - name: hints type: Any[]? doc: | Declares hints applying to either the runtime environment or the workflow engine that may be helpful in executing this process. It is not an error if an implementation cannot satisfy all hints, however the implementation may report a warning. jsonldPredicate: _id: cwl:hints noLinkCheck: true mapSubject: class - name: label type: string? jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: doc type: string? jsonldPredicate: "rdfs:comment" doc: "A long, human-readable description of this process object." - name: cwlVersion type: CWLVersion? doc: | CWL document version. Always required at the document root. Not required for a Process embedded inside another Process. jsonldPredicate: "_id": "cwl:cwlVersion" "_type": "@vocab" - name: InlineJavascriptRequirement type: record extends: ProcessRequirement doc: | Indicates that the workflow platform must support inline Javascript expressions. If this requirement is not present, the workflow platform must not perform expression interpolatation. fields: - name: class type: string doc: "Always 'InlineJavascriptRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: expressionLib type: string[]? doc: | Additional code fragments that will also be inserted before executing the expression code. Allows for function definitions that may be called from CWL expressions. - name: SchemaDefRequirement type: record extends: ProcessRequirement doc: | This field consists of an array of type definitions which must be used when interpreting the `inputs` and `outputs` fields. When a `type` field contain a IRI, the implementation must check if the type is defined in `schemaDefs` and use that definition. If the type is not found in `schemaDefs`, it is an error. The entries in `schemaDefs` must be processed in the order listed such that later schema definitions may refer to earlier schema definitions. fields: - name: class type: string doc: "Always 'SchemaDefRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: types type: type: array items: InputSchema doc: The list of type definitions. cwltool-1.0.20170114120503/cwltool/schemas/v1.0/concepts.md0000644000175000017500000004323213036412052023366 0ustar michaelmichael00000000000000## References to other specifications **Javascript Object Notation (JSON)**: http://json.org **JSON Linked Data (JSON-LD)**: http://json-ld.org **YAML**: http://yaml.org **Avro**: https://avro.apache.org/docs/current/spec.html **Uniform Resource Identifier (URI) Generic Syntax**: https://tools.ietf.org/html/rfc3986) **Internationalized Resource Identifiers (IRIs)**: https://tools.ietf.org/html/rfc3987 **Portable Operating System Interface (POSIX.1-2008)**: http://pubs.opengroup.org/onlinepubs/9699919799/ **Resource Description Framework (RDF)**: http://www.w3.org/RDF/ ## Scope This document describes CWL syntax, execution, and object model. It is not intended to document a CWL specific implementation, however it may serve as a reference for the behavior of conforming implementations. ## Terminology The terminology used to describe CWL documents is defined in the Concepts section of the specification. The terms defined in the following list are used in building those definitions and in describing the actions of a CWL implementation: **may**: Conforming CWL documents and CWL implementations are permitted but not required to behave as described. **must**: Conforming CWL documents and CWL implementations are required to behave as described; otherwise they are in error. **error**: A violation of the rules of this specification; results are undefined. Conforming implementations may detect and report an error and may recover from it. **fatal error**: A violation of the rules of this specification; results are undefined. Conforming implementations must not continue to execute the current process and may report an error. **at user option**: Conforming software may or must (depending on the modal verb in the sentence) behave as described; if it does, it must provide users a means to enable or disable the behavior described. **deprecated**: Conforming software may implement a behavior for backwards compatibility. Portable CWL documents should not rely on deprecated behavior. Behavior marked as deprecated may be removed entirely from future revisions of the CWL specification. # Data model ## Data concepts An **object** is a data structure equivalent to the "object" type in JSON, consisting of a unordered set of name/value pairs (referred to here as **fields**) and where the name is a string and the value is a string, number, boolean, array, or object. A **document** is a file containing a serialized object, or an array of objects. A **process** is a basic unit of computation which accepts input data, performs some computation, and produces output data. Examples include CommandLineTools, Workflows, and ExpressionTools. An **input object** is an object describing the inputs to an invocation of a process. An **output object** is an object describing the output resulting from an invocation of a process. An **input schema** describes the valid format (required fields, data types) for an input object. An **output schema** describes the valid format for an output object. **Metadata** is information about workflows, tools, or input items. ## Syntax CWL documents must consist of an object or array of objects represented using JSON or YAML syntax. Upon loading, a CWL implementation must apply the preprocessing steps described in the [Semantic Annotations for Linked Avro Data (SALAD) Specification](SchemaSalad.html). An implementation may formally validate the structure of a CWL document using SALAD schemas located at https://github.com/common-workflow-language/common-workflow-language/tree/master/v1.0 ## Identifiers If an object contains an `id` field, that is used to uniquely identify the object in that document. The value of the `id` field must be unique over the entire document. Identifiers may be resolved relative to either the document base and/or other identifiers following the rules are described in the [Schema Salad specification](SchemaSalad.html#Identifier_resolution). An implementation may choose to only honor references to object types for which the `id` field is explicitly listed in this specification. ## Document preprocessing An implementation must resolve [$import](SchemaSalad.html#Import) and [$include](SchemaSalad.html#Import) directives as described in the [Schema Salad specification](SchemaSalad.html). Another transformation defined in Schema salad is simplification of data type definitions. Type `` ending with `?` should be transformed to `[, "null"]`. Type `` ending with `[]` should be transformed to `{"type": "array", "items": }` ## Extensions and metadata Input metadata (for example, a lab sample identifier) may be represented within a tool or workflow using input parameters which are explicitly propagated to output. Future versions of this specification may define additional facilities for working with input/output metadata. Implementation extensions not required for correct execution (for example, fields related to GUI presentation) and metadata about the tool or workflow itself (for example, authorship for use in citations) may be provided as additional fields on any object. Such extensions fields must use a namespace prefix listed in the `$namespaces` section of the document as described in the [Schema Salad specification](SchemaSalad.html#Explicit_context). Implementation extensions which modify execution semantics must be [listed in the `requirements` field](#Requirements_and_hints). # Execution model ## Execution concepts A **parameter** is a named symbolic input or output of process, with an associated datatype or schema. During execution, values are assigned to parameters to make the input object or output object used for concrete process invocation. A **CommandLineTool** is a process characterized by the execution of a standalone, non-interactive program which is invoked on some input, produces output, and then terminates. A **workflow** is a process characterized by multiple subprocess steps, where step outputs are connected to the inputs of downstream steps to form a directed acylic graph, and independent steps may run concurrently. A **runtime environment** is the actual hardware and software environment when executing a command line tool. It includes, but is not limited to, the hardware architecture, hardware resources, operating system, software runtime (if applicable, such as the specific Python interpreter or the specific Java virtual machine), libraries, modules, packages, utilities, and data files required to run the tool. A **workflow platform** is a specific hardware and software implementation capable of interpreting CWL documents and executing the processes specified by the document. The responsibilities of the workflow platform may include scheduling process invocation, setting up the necessary runtime environment, making input data available, invoking the tool process, and collecting output. A workflow platform may choose to only implement the Command Line Tool Description part of the CWL specification. It is intended that the workflow platform has broad leeway outside of this specification to optimize use of computing resources and enforce policies not covered by this specification. Some areas that are currently out of scope for CWL specification but may be handled by a specific workflow platform include: * Data security and permissions * Scheduling tool invocations on remote cluster or cloud compute nodes. * Using virtual machines or operating system containers to manage the runtime (except as described in [DockerRequirement](CommandLineTool.html#DockerRequirement)). * Using remote or distributed file systems to manage input and output files. * Transforming file paths. * Determining if a process has previously been executed, and if so skipping it and reusing previous results. * Pausing, resuming or checkpointing processes or workflows. Conforming CWL processes must not assume anything about the runtime environment or workflow platform unless explicitly declared though the use of [process requirements](#Requirements_and_hints). ## Generic execution process The generic execution sequence of a CWL process (including workflows and command line line tools) is as follows. 1. Load, process and validate a CWL document, yielding a process object. 2. Load input object. 3. Validate the input object against the `inputs` schema for the process. 4. Validate process requirements are met. 5. Perform any further setup required by the specific process type. 6. Execute the process. 7. Capture results of process execution into the output object. 8. Validate the output object against the `outputs` schema for the process. 9. Report the output object to the process caller. ## Requirements and hints A **process requirement** modifies the semantics or runtime environment of a process. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. A **hint** is similar to a requirement; however, it is not an error if an implementation cannot satisfy all hints. The implementation may report a warning if a hint cannot be satisfied. Requirements are inherited. A requirement specified in a Workflow applies to all workflow steps; a requirement specified on a workflow step will apply to the process implementation of that step and any of its substeps. If the same process requirement appears at different levels of the workflow, the most specific instance of the requirement is used, that is, an entry in `requirements` on a process implementation such as CommandLineTool will take precedence over an entry in `requirements` specified in a workflow step, and an entry in `requirements` on a workflow step takes precedence over the workflow. Entries in `hints` are resolved the same way. Requirements override hints. If a process implementation provides a process requirement in `hints` which is also provided in `requirements` by an enclosing workflow or workflow step, the enclosing `requirements` takes precedence. ## Parameter references Parameter references are denoted by the syntax `$(...)` and may be used in any field permitting the pseudo-type `Expression`, as specified by this document. Conforming implementations must support parameter references. Parameter references use the following subset of [Javascript/ECMAScript 5.1](http://www.ecma-international.org/ecma-262/5.1/) syntax, but they are designed to not require a Javascript engine for evaluation. In the following BNF grammar, character classes, and grammar rules are denoted in '{}', '-' denotes exclusion from a character class, '(())' denotes grouping, '|' denotes alternates, trailing '*' denotes zero or more repeats, '+' denote one or more repeats, '/' escapes these special characters, and all other characters are literal values.

symbol:: {Unicode alphanumeric}+
singleq:: [' (( {character - '} | \' ))* ']
doubleq:: [" (( {character - "} | \" ))* "]
index:: [ {decimal digit}+ ]
segment:: . {symbol} | {singleq} | {doubleq} | {index}
parameter reference::$( {symbol} {segment}*)

Use the following algorithm to resolve a parameter reference: 1. Match the leading symbol as the key 2. Look up the key in the parameter context (described below) to get the current value. It is an error if the key is not found in the parameter context. 3. If there are no subsequent segments, terminate and return current value 4. Else, match the next segment 5. Extract the symbol, string, or index from the segment as the key 6. Look up the key in current value and assign as new current value. If the key is a symbol or string, the current value must be an object. If the key is an index, the current value must be an array or string. It is an error if the key does not match the required type, or the key is not found or out of range. 7. Repeat steps 3-6 The root namespace is the parameter context. The following parameters must be provided: * `inputs`: The input object to the current Process. * `self`: A context-specific value. The contextual values for 'self' are documented for specific fields elsewhere in this specification. If a contextual value of 'self' is not documented for a field, it must be 'null'. * `runtime`: An object containing configuration details. Specific to the process type. An implementation may provide opaque strings for any or all fields of `runtime`. These must be filled in by the platform after processing the Tool but before actual execution. Parameter references and expressions may only use the literal string value of the field and must not perform computation on the contents, except where noted otherwise. If the value of a field has no leading or trailing non-whitespace characters around a parameter reference, the effective value of the field becomes the value of the referenced parameter, preserving the return type. If the value of a field has non-whitespace leading or trailing characters around a parameter reference, it is subject to string interpolation. The effective value of the field is a string containing the leading characters, followed by the string value of the parameter reference, followed by the trailing characters. The string value of the parameter reference is its textual JSON representation with the following rules: * Leading and trailing quotes are stripped from strings * Objects entries are sorted by key Multiple parameter references may appear in a single field. This case must be treated as a string interpolation. After interpolating the first parameter reference, interpolation must be recursively applied to the trailing characters to yield the final string value. ## Expressions An expression is a fragment of [Javascript/ECMAScript 5.1](http://www.ecma-international.org/ecma-262/5.1/) code evaluated by the workflow platform to affect the inputs, outputs, or behavior of a process. In the generic execution sequence, expressions may be evaluated during step 5 (process setup), step 6 (execute process), and/or step 7 (capture output). Expressions are distinct from regular processes in that they are intended to modify the behavior of the workflow itself rather than perform the primary work of the workflow. To declare the use of expressions, the document must include the process requirement `InlineJavascriptRequirement`. Expressions may be used in any field permitting the pseudo-type `Expression`, as specified by this document. Expressions are denoted by the syntax `$(...)` or `${...}`. A code fragment wrapped in the `$(...)` syntax must be evaluated as a [ECMAScript expression](http://www.ecma-international.org/ecma-262/5.1/#sec-11). A code fragment wrapped in the `${...}` syntax must be evaluated as a [EMACScript function body](http://www.ecma-international.org/ecma-262/5.1/#sec-13) for an anonymous, zero-argument function. Expressions must return a valid JSON data type: one of null, string, number, boolean, array, object. Other return values must result in a `permanentFailure`. Implementations must permit any syntactically valid Javascript and account for nesting of parenthesis or braces and that strings that may contain parenthesis or braces when scanning for expressions. The runtime must include any code defined in the ["expressionLib" field of InlineJavascriptRequirement](#InlineJavascriptRequirement) prior to executing the actual expression. Before executing the expression, the runtime must initialize as global variables the fields of the parameter context described above. The effective value of the field after expression evaluation follows the same rules as parameter references discussed above. Multiple expressions may appear in a single field. Expressions must be evaluated in an isolated context (a "sandbox") which permits no side effects to leak outside the context. Expressions also must be evaluated in [Javascript strict mode](http://www.ecma-international.org/ecma-262/5.1/#sec-4.2.2). The order in which expressions are evaluated is undefined except where otherwise noted in this document. An implementation may choose to implement parameter references by evaluating as a Javascript expression. The results of evaluating parameter references must be identical whether implemented by Javascript evaluation or some other means. Implementations may apply other limits, such as process isolation, timeouts, and operating system containers/jails to minimize the security risks associated with running untrusted code embedded in a CWL document. Exceptions thrown from an exception must result in a `permanentFailure` of the process. ## Executing CWL documents as scripts By convention, a CWL document may begin with `#!/usr/bin/env cwl-runner` and be marked as executable (the POSIX "+x" permission bits) to enable it to be executed directly. A workflow platform may support this mode of operation; if so, it must provide `cwl-runner` as an alias for the platform's CWL implementation. A CWL input object document may similarly begin with `#!/usr/bin/env cwl-runner` and be marked as executable. In this case, the input object must include the field `cwl:tool` supplying an IRI to the default CWL document that should be executed using the fields of the input object as input parameters. ## Discovering CWL documents on a local filesystem To discover CWL documents look in the following locations: `/usr/share/commonwl/` `/usr/local/share/commonwl/` `$XDG_DATA_HOME/commonwl/` (usually `$HOME/.local/share/commonwl`) `$XDF_DATA_HOME` is from the [XDG Base Directory Specification](http://standards.freedesktop.org/basedir-spec/basedir-spec-0.6.html) cwltool-1.0.20170114120503/cwltool/schemas/v1.0/intro.md0000644000175000017500000000156413036412052022705 0ustar michaelmichael00000000000000# Status of this document This document is the product of the [Common Workflow Language working group](https://groups.google.com/forum/#!forum/common-workflow-language). The latest version of this document is available in the "v1.0" directory at https://github.com/common-workflow-language/common-workflow-language The products of the CWL working group (including this document) are made available under the terms of the Apache License, version 2.0. # Introduction The Common Workflow Language (CWL) working group is an informal, multi-vendor working group consisting of various organizations and individuals that have an interest in portability of data analysis workflows. The goal is to create specifications like this one that enable data scientists to describe analysis tools and workflows that are powerful, easy to use, portable, and support reproducibility. cwltool-1.0.20170114120503/cwltool/schemas/v1.0/UserGuide.yml0000644000175000017500000006711713036412052023655 0ustar michaelmichael00000000000000- name: userguide type: documentation doc: - $include: userguide-intro.md - | # Wrapping Command Line Tools - | ## First example The simplest "hello world" program. This accepts one input parameter, writes a message to the terminal or job log, and produces no permanent output. CWL documents are written in [JSON](http://json.org) or [YAML](http://yaml.org), or a mix of the two. *1st-tool.cwl* ``` - $include: examples/1st-tool.cwl - | ``` Use a YAML object in a separate file to describe the input of a run: *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwl-runner 1st-tool.cwl echo-job.yml [job 140199012414352] $ echo 'Hello world!' Hello world! Final process status is success ``` What's going on here? Let's break it down: ``` cwlVersion: v1.0 class: CommandLineTool ``` The `cwlVersion` field indicates the version of the CWL spec used by the document. The `class` field indicates this document describes a command line tool. ``` baseCommand: echo ``` The `baseCommand` provides the name of program that will actually run (echo) ``` inputs: message: type: string inputBinding: position: 1 ``` The `inputs` section describes the inputs of the tool. This is a list of input parameters and each parameter includes an identifier, a data type, and optionally an `inputBinding` which describes how this input parameter should appear on the command line. In this example, the `position` field indicates where it should appear on the command line. ``` outputs: [] ``` This tool has no formal output, so the `outputs` section is an empty list. - | ## Essential input parameters The `inputs` of a tool is a list of input parameters that control how to run the tool. Each parameter has an `id` for the name of parameter, and `type` describing what types of values are valid for that parameter. Available primitive types are *string*, *int*, *long*, *float*, *double*, and *null*; complex types are *array* and *record*; in addition there are special types *File*, *Directory* and *Any*. The following example demonstrates some input parameters with different types and appearing on the command line in different ways: *inp.cwl* ``` - $include: examples/inp.cwl - | ``` *inp-job.yml* ``` - $include: examples/inp-job.yml - | ``` Notice that "example_file", as a `File` type, must be provided as an object with the fields `class: File` and `path`. Next, create a whale.txt and invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ touch whale.txt $ cwl-runner inp.cwl inp-job.yml [job 140020149614160] /home/example$ echo -f -i42 --example-string hello --file=/home/example/whale.txt -f -i42 --example-string hello --file=/home/example/whale.txt Final process status is success ``` The field `inputBinding` is optional and indicates whether and how the input parameter should be appear on the tool's command line. If `inputBinding` is missing, the parameter does not appear on the command line. Let's look at each example in detail. ``` example_flag: type: boolean inputBinding: position: 1 prefix: -f ``` Boolean types are treated as a flag. If the input parameter "example_flag" is "true", then `prefix` will be added to the command line. If false, no flag is added. ``` example_string: type: string inputBinding: position: 3 prefix: --example-string ``` String types appear on the command line as literal values. The `prefix` is optional, if provided, it appears as a separate argument on the command line before the parameter . In the example above, this is rendered as `--example-string hello`. ``` example_int: type: int inputBinding: position: 2 prefix: -i separate: false ``` Integer (and floating point) types appear on the command line with decimal text representation. When the option `separate` is false (the default value is true), the prefix and value are combined into a single argument. In the example above, this is rendered as `-i42`. ``` example_file: type: File? inputBinding: prefix: --file= separate: false position: 4 ``` File types appear on the command line as the path to the file. When the parameter type ends with a question mark `?` it indicates that the parameter is optional. In the example above, this is rendered as `--file=/home/example/whale.txt`. However, if the "example_file" parameter were not provided in the input, nothing would appear on the command line. Input files are read-only. If you wish to update an input file, you must first copy it to the output directory. The value of `position` is used to determine where parameter should appear on the command line. Positions are relative to one another, not abosolute. As a result, positions do not have to be sequential, three parameters with positions `[1, 3, 5]` will result in the same command line as `[1, 2, 3]`. More than one parameter can have the same position (ties are broken using the parameter name), and the position field itself is optional. the default position is 0. The `baseCommand` field always comes before parameters. - | ## Returning output files The `outputs` of a tool is a list of output parameters that should be returned after running the tool. Each parameter has an `id` for the name of parameter, and `type` describing what types of values are valid for that parameter. When a tool runs under CWL, the starting working directory is the designated output directory. The underlying tool or script must record its results in the form of files created in the output directory. The output parameters returned by the CWL tool are either the output files themselves, or come from examining the content of those files. *tar.cwl* ``` - $include: examples/tar.cwl - | ``` *tar-job.yml* ``` - $include: examples/tar-job.yml - | ``` Next, create a tar file for the example and invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ touch hello.txt && tar -cvf hello.tar hello.txt $ cwl-runner tar.cwl tar-job.yml [job 139868145165200] $ tar xf /home/example/hello.tar Final process status is success { "example_out": { "location": "hello.txt", "size": 13, "class": "File", "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b" } } ``` The field `outputBinding` describes how to to set the value of each output parameter. ``` outputs: - id: example_out type: File outputBinding: glob: hello.txt ``` The `glob` field consists of the name of a file in the output directory. If you don't know name of the file in advance, you can use a wildcard pattern. - | ## Capturing a tool's standard output stream To capture a tool's standard output stream, add the `stdout` field with the name of the file where the output stream should go. Then add `type: stdout` on the corresponding output parameter. *stdout.cwl* ``` - $include: examples/stdout.cwl - | ``` *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ cwl-runner stdout.cwl echo-job.yml [job 140199012414352] $ echo 'Hello world!' > output.txt Final process status is success { "output": { "location": "output.txt", "size": 13, "class": "File", "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b" } } $ cat output.txt Hello world! ``` - | ## Parameter references In a previous example, we extracted a file using the "tar" program. However, that example was very limited because it assumed that the file we were interested in was called "hello.txt". In this example, you will see how to reference the value of input parameters dynamically from other fields. *tar-param.cwl* ``` - $include: examples/tar-param.cwl - | ``` *tar-param-job.yml* ``` - $include: examples/tar-param-job.yml - | ``` Create your input files and invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ rm hello.tar || true && touch goodbye.txt && tar -cvf hello.tar goodbye.txt $ cwl-runner tar-param.cwl tar-param-job.yml [job 139868145165200] $ tar xf /home/example/hello.tar goodbye.txt Final process status is success { "example_out": { "location": "goodbye.txt", "size": 24, "class": "File", "checksum": "sha1$dd0a4c4c49ba43004d6611771972b6cf969c1c01" } } ``` Certain fields permit parameter references which are enclosed in `$(...)`. These are evaluated and replaced with value being referenced. ``` outputs: example_out: type: File outputBinding: glob: $(inputs.extractfile) ``` References are written using a subset of Javascript syntax. In this example, `$(inputs.extractfile)`, `$(inputs["extractfile"])`, and `$(inputs['extractfile'])` are equivalent. The value of the "inputs" variable is the input object provided when the CWL tool was invoked. Note that because File parameters are objects, to get the path to an input file you must reference the path field on a file object; to reference the path to the tar file in the above example you would write `$(inputs.tarfile.path)`. - | ## Running tools inside Docker [Docker](http://docker.io) containers simplify software installation by providing a complete known-good runtime for software and its dependencies. However, containers are also purposefully isolated from the host system, so in order to run a tool inside a Docker container there is additional work to ensure that input files are available inside the container and output files can be recovered from the contianer. CWL can perform this work automatically, allowing you to use Docker to simplify your software management while avoiding the complexity of invoking and managing Docker containers. This example runs a simple Node.js script inside a Docker container. *docker.cwl* ``` - $include: examples/docker.cwl - | ``` *docker-job.yml* ``` - $include: examples/docker-job.yml - | ``` Provide a hello.js and invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ echo "console.log(\"Hello World\");" > hello.js $ cwl-runner docker.cwl docker-job.yml [job 140259721854416] /home/example$ docker run -i --volume=/home/example/hello.js:/var/lib/cwl/job369354770_examples/hello.js:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpDLs5hm:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp node:slim node /var/lib/cwl/job369354770_examples/hello.js Hello world! Final process status is success ``` Notice the CWL runner has constructed a Docker command line to run the script. One of the responsibilies of the CWL runner is to the paths of input files to reflect the location where they appear inside the container. In this example, the path to the script `hello.js` is `/home/example/hello.js` outside the container but `/var/lib/cwl/job369354770_examples/hello.js` inside the container, as reflected in the invocation of the `node` command. - | ## Additional command line arguments and runtime parameters Sometimes tools require additional command line options that don't correspond exactly to input parameters. In this example, we will wrap the Java compiler to compile a java source file to a class file. By default, `javac` will create the class files in the same directory as the source file. However, CWL input files (and the directories in which they appear) may be read-only, so we need to instruct javac to write the class file to the designated output directory instead. *arguments.cwl* ``` - $include: examples/arguments.cwl - | ``` *arguments-job.yml* ``` - $include: examples/arguments-job.yml - | ``` Now create a sample Java file and invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ echo "public class Hello {}" > Hello.java $ cwl-runner arguments.cwl arguments-job.yml [job arguments.cwl] /tmp/tmpwYALo1$ docker \ run \ -i \ --volume=/home/peter/work/common-workflow-language/v1.0/examples/Hello.java:/var/lib/cwl/stg8939ac04-7443-4990-a518-1855b2322141/Hello.java:ro \ --volume=/tmp/tmpwYALo1:/var/spool/cwl:rw \ --volume=/tmp/tmpptIAJ8:/tmp:rw \ --workdir=/var/spool/cwl \ --read-only=true \ --user=1001 \ --rm \ --env=TMPDIR=/tmp \ --env=HOME=/var/spool/cwl \ java:7 \ javac \ -d \ /var/spool/cwl \ /var/lib/cwl/stg8939ac04-7443-4990-a518-1855b2322141/Hello.java Final process status is success { "classfile": { "size": 416, "location": "/home/example/Hello.class", "checksum": "sha1$2f7ac33c1f3aac3f1fec7b936b6562422c85b38a", "class": "File" } } ``` Here we use the `arguments` field to add an additional argument to the command line that isn't tied to a specific input parameter. ``` arguments: ["-d", $(runtime.outdir)] ``` This example references a runtime parameter. Runtime parameters provide information about the hardware or software environment when the tool is actually executed. The `$(runtime.outdir)` parameter is the path to the designated output directory. Other parameters include `$(runtime.tmpdir)`, `$(runtime.ram)`, `$(runtime.cores)`, `$(runtime.ram)`, `$(runtime.outdirSize)`, and `$(runtime.tmpdirSize)`. See the [Runtime Environment](CommandLineTool.html#Runtime_environment) section of the CWL specification for details. - | ## Array inputs It is easy to add arrays of input parameters represented to the command line. To specify an array parameter, the array definition is nested under the `type` field with `type: array` and `items` defining the valid data types that may appear in the array. *array-inputs.cwl* ``` - $include: examples/array-inputs.cwl - | ``` *array-inputs-job.yml* ``` - $include: examples/array-inputs-job.yml - | ``` Now invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ cwl-runner array-inputs.cwl array-inputs-job.yml [job 140334923640912] /home/example$ echo -A one two three -B=four -B=five -B=six -C=seven,eight,nine -A one two three -B=four -B=five -B=six -C=seven,eight,nine Final process status is success {} ``` The `inputBinding` can appear either on the outer array parameter definition or the inner array element definition, and these produce different behavior when constructing the command line, as shown above. In addition, the `itemSeperator` field, if provided, specifies that array values should be concatenated into a single argument separated by the item separator string. You can specify arrays of arrays, arrays of records, and other complex types. - | ## Array outputs You can also capture multiple output files into an array of files using `glob`. *array-outputs.cwl* ``` - $include: examples/array-outputs.cwl - | ``` *array-outpust-job.yml* ``` - $include: examples/array-outputs-job.yml - | ``` Now invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ cwl-runner array-outputs.cwl array-outputs-job.yml [job 140190876078160] /home/example$ touch foo.txt bar.dat baz.txt Final process status is success { "output": [ { "size": 0, "location": "/home/peter/work/common-workflow-language/draft-3/examples/foo.txt", "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", "class": "File" }, { "size": 0, "location": "/home/peter/work/common-workflow-language/draft-3/examples/baz.txt", "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", "class": "File" } ] } ``` - | ## Record inputs, dependent and mutually exclusive parameters Sometimes an underlying tool has several arguments that must be provided together (they are dependent) or several arguments that cannot be provided together (they are exclusive). You can use records and type unions to group parameters together to describe these two conditions. *record.cwl* ``` - $include: examples/record.cwl - | ``` *record-job1.yml* ``` - $include: examples/record-job1.yml - | ``` ``` $ cwl-runner record.cwl record-job1.yml Workflow error: Error validating input record, could not validate field `dependent_parameters` because missing required field `itemB` ``` In the first example, you can't provide `itemA` without also providing `itemB`. *record-job2.yml* ``` - $include: examples/record-job2.yml - | ``` ``` $ cwl-runner record.cwl record-job2.yml [job 140566927111376] /home/example$ echo -A one -B two -C three -A one -B two -C three Final process status is success {} ``` In the second example, `itemC` and `itemD` are exclusive, so only `itemC` is added to the command line and `itemD` is ignored. *record-job3.yml* ``` - $include: examples/record-job3.yml - | ``` ``` $ cwl-runner record.cwl record-job3.yml [job 140606932172880] /home/example$ echo -A one -B two -D four -A one -B two -D four Final process status is success {} ``` In the third example, only `itemD` is provided, so it appears on the command line. - | ## Environment variables Tools run in a restricted environment and do not inherit most environment variables from the parent process. You can set environment variables for the tool using `EnvVarRequirement`. *env.cwl* ``` - $include: examples/env.cwl - | ``` *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwl-runner env.cwl echo-job.yml [job 140710387785808] /home/example$ env PATH=/bin:/usr/bin:/usr/local/bin HELLO=Hello world! TMPDIR=/tmp/tmp63Obpk Final process status is success {} ``` - | ## Javascript expressions If you need to manipulate input parameters, include the requirement `InlineJavascriptRequirement` and then anywhere a parameter reference is legal you can provide a fragment of Javascript that will be evaluated by the CWL runner. *expression.cwl* ``` - $include: examples/expression.cwl - | ``` ``` $ cwl-runner expression.cwl empty.yml [job 140000594593168] /home/example$ echo -A 2 -B baz -C 10 9 8 7 6 5 4 3 2 1 -A 2 -B baz -C 10 9 8 7 6 5 4 3 2 1 Final process status is success {} ``` You can only use expressions in certain fields. These are: `filename`, `fileContent`, `envValue`, `valueFrom`, `glob`, `outputEval`, `stdin`, `stdout`, `coresMin`, `coresMax`, `ramMin`, `ramMax`, `tmpdirMin`, `tmpdirMax`, `outdirMin`, and `outdirMax`. - | ## Creating files at runtime Sometimes you need to create a file on the fly from input parameters, such as tools which expect to read their input configuration from a file rather than the command line parameters. To do this, use `InitialWorkDirRequirement`. *createfile.cwl* ``` - $include: examples/createfile.cwl - | ``` *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwltool createfile.cwl echo-job.yml [job 140528604979344] /home/example$ cat example.conf CONFIGVAR=Hello world! Final process status is success {} ``` - | ## Staging input files in the output directory Normally, input files are located in a read-only directory separate from the output directory. This causes problems if the underlying tool expects to write its output files alongside the input file in the same directory. You use `InitialWorkDirRequirement` to stage input files into the output directory. In this example, we use a Javascript expression to extract the base name of the input file from its leading directory path. *linkfile.cwl* ``` - $include: examples/linkfile.cwl - | ``` *arguments-job.yml* ``` - $include: examples/arguments-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwl-runner linkfile.cwl arguments-job.yml [job 139928309171664] /home/example$ docker run -i --volume=/home/example/Hello.java:/var/lib/cwl/job557617295_examples/Hello.java:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpmNbApw:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp java:7 javac Hello.java Final process status is success { "classfile": { "size": 416, "location": "/home/example/Hello.class", "checksum": "sha1$2f7ac33c1f3aac3f1fec7b936b6562422c85b38a", "class": "File" } } ``` - | # Writing Workflows ## First workflow This workflow extracts a java source file from a tar file and then compiles it. *1st-workflow.cwl* ``` - $include: examples/1st-workflow.cwl - | ``` Use a JSON object in a separate file to describe the input of a run: *1st-workflow-job.yml* ``` - $include: examples/1st-workflow-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ echo "public class Hello {}" > Hello.java && tar -cvf hello.tar Hello.java $ cwl-runner 1st-workflow.cwl 1st-workflow-job.yml [job untar] /tmp/tmp94qFiM$ tar xf /home/example/hello.tar Hello.java [step untar] completion status is success [job compile] /tmp/tmpu1iaKL$ docker run -i --volume=/tmp/tmp94qFiM/Hello.java:/var/lib/cwl/job301600808_tmp94qFiM/Hello.java:ro --volume=/tmp/tmpu1iaKL:/var/spool/cwl:rw --volume=/tmp/tmpfZnNdR:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp java:7 javac -d /var/spool/cwl /var/lib/cwl/job301600808_tmp94qFiM/Hello.java [step compile] completion status is success [workflow 1st-workflow.cwl] outdir is /home/example Final process status is success { "classout": { "location": "/home/example/Hello.class", "checksum": "sha1$e68df795c0686e9aa1a1195536bd900f5f417b18", "class": "File", "size": 416 } } ``` What's going on here? Let's break it down: ``` cwlVersion: v1.0 class: Workflow ``` The 'cwlVersion` field indicates the version of the CWL spec used by the document. The `class` field indicates this document describes a workflow. ``` inputs: inp: File ex: string ``` The `inputs` section describes the inputs of the workflow. This is a list of input parameters where each parameter consists of an identifier and a data type. These parameters can be used as sources for input to specific workflows steps. ``` outputs: classout: type: File outputSource: compile/classfile ``` The `outputs` section describes the outputs of the workflow. This is a list of output parameters where each parameter consists of an identifier and a data type. The `source` connects the output parameter `classfile` of the `compile` step to the workflow output parameter `classout`. ``` steps: untar: run: tar-param.cwl in: tarfile: inp extractfile: ex outputs: [example_out] ``` The `steps` section describes the actual steps of the workflow. In this example, the first step extracts a file from a tar file, and the second step compiles the file from the first step using the java compiler. Workflow steps are not necessarily run in the order they are listed, instead the order is determined by the dependencies between steps (using `source`). In addition, workflow steps which do not depend on one another may run in parallel. The first step, `untar` runs `tar-param.cwl` (described previously in [Parameter references](#Parameter_references)). This tool has two input parameters, `tarfile` and `extractfile` and one output parameter `example_out`. The `inputs` section of the workflow step connects these two input parameters to the inputs of the workflow, `inp` and `ex` using `source`. This means that when the workflow step is executed, the values assigned to `inp` and `ex` will be used for the parameters `tarfile` and `extractfile` in order to run the tool. The `outputs` section of the workflow step lists the output parameters that are expected from the tool. ``` compile: run: arguments.cwl in: src: untar/example_out outputs: [classfile] ``` The second step `compile` depends on the results from the first step by connecting the input parameter `src` to the output parameter of `untar` using `untar/example_out`. The output of this step `classfile` is connected to the `outputs` section for the Workflow, described above. cwltool-1.0.20170114120503/cwltool/schemas/v1.0/Workflow.yml0000644000175000017500000004713313036412052023567 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" $graph: - name: "WorkflowDoc" type: documentation doc: - | # Common Workflow Language (CWL) Workflow Description, v1.0 This version: * https://w3id.org/cwl/v1.0/ Current version: * https://w3id.org/cwl/ - "\n\n" - {$include: contrib.md} - "\n\n" - | # Abstract One way to define a workflow is: an analysis task represented by a directed graph describing a sequence of operations that transform an input data set to output. This specification defines the Common Workflow Language (CWL) Workflow description, a vendor-neutral standard for representing workflows intended to be portable across a variety of computing platforms. - {$include: intro.md} - | ## Introduction to v1.0 This specification represents the first full release from the CWL group. Since draft-3, this draft introduces the following changes and additions: * The `inputs` and `outputs` fields have been renamed `in` and `out`. * Syntax simplifcations: denoted by the `map<>` syntax. Example: `in` contains a list of items, each with an id. Now one can specify a mapping of that identifier to the corresponding `InputParameter`. ``` in: - id: one type: string doc: First input parameter - id: two type: int doc: Second input parameter ``` can be ``` in: one: type: string doc: First input parameter two: type: int doc: Second input parameter ``` * The common field `description` has been renamed to `doc`. ## Purpose The Common Workflow Language Command Line Tool Description express workflows for data-intensive science, such as Bioinformatics, Chemistry, Physics, and Astronomy. This specification is intended to define a data and execution model for Workflows that can be implemented on top of a variety of computing platforms, ranging from an individual workstation to cluster, grid, cloud, and high performance computing systems. - {$include: concepts.md} - name: ExpressionToolOutputParameter type: record extends: OutputParameter fields: - name: type type: - "null" - "#CWLType" - "#OutputRecordSchema" - "#OutputEnumSchema" - "#OutputArraySchema" - string - type: array items: - "#CWLType" - "#OutputRecordSchema" - "#OutputEnumSchema" - "#OutputArraySchema" - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" refScope: 2 typeDSL: True doc: | Specify valid types of data that may be assigned to this parameter. - type: record name: ExpressionTool extends: Process specialize: - specializeFrom: "#OutputParameter" specializeTo: "#ExpressionToolOutputParameter" documentRoot: true doc: | Execute an expression as a Workflow step. fields: - name: "class" jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: expression type: [string, Expression] doc: | The expression to execute. The expression must return a JSON object which matches the output parameters of the ExpressionTool. - name: LinkMergeMethod type: enum docParent: "#WorkflowStepInput" doc: The input link merge method, described in [WorkflowStepInput](#WorkflowStepInput). symbols: - merge_nested - merge_flattened - name: WorkflowOutputParameter type: record extends: OutputParameter docParent: "#Workflow" doc: | Describe an output parameter of a workflow. The parameter must be connected to one or more parameters defined in the workflow that will provide the value of the output parameter. fields: - name: outputSource doc: | Specifies one or more workflow parameters that supply the value of to the output parameter. jsonldPredicate: "_id": "cwl:outputSource" "_type": "@id" refScope: 0 type: - string? - string[]? - name: linkMerge type: ["null", "#LinkMergeMethod"] jsonldPredicate: "cwl:linkMerge" doc: | The method to use to merge multiple sources into a single array. If not specified, the default method is "merge_nested". - name: type type: - "null" - "#CWLType" - "#OutputRecordSchema" - "#OutputEnumSchema" - "#OutputArraySchema" - string - type: array items: - "#CWLType" - "#OutputRecordSchema" - "#OutputEnumSchema" - "#OutputArraySchema" - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" refScope: 2 typeDSL: True doc: | Specify valid types of data that may be assigned to this parameter. - name: Sink type: record abstract: true fields: - name: source doc: | Specifies one or more workflow parameters that will provide input to the underlying step parameter. jsonldPredicate: "_id": "cwl:source" "_type": "@id" refScope: 2 type: - string? - string[]? - name: linkMerge type: LinkMergeMethod? jsonldPredicate: "cwl:linkMerge" doc: | The method to use to merge multiple inbound links into a single array. If not specified, the default method is "merge_nested". - type: record name: WorkflowStepInput extends: Sink docParent: "#WorkflowStep" doc: | The input of a workflow step connects an upstream parameter (from the workflow inputs, or the outputs of other workflows steps) with the input parameters of the underlying step. ## Input object A WorkflowStepInput object must contain an `id` field in the form `#fieldname` or `#stepname.fieldname`. When the `id` field contains a period `.` the field name consists of the characters following the final period. This defines a field of the workflow step input object with the value of the `source` parameter(s). ## Merging To merge multiple inbound data links, [MultipleInputFeatureRequirement](#MultipleInputFeatureRequirement) must be specified in the workflow or workflow step requirements. If the sink parameter is an array, or named in a [workflow scatter](#WorkflowStep) operation, there may be multiple inbound data links listed in the `source` field. The values from the input links are merged depending on the method specified in the `linkMerge` field. If not specified, the default method is "merge_nested". * **merge_nested** The input must be an array consisting of exactly one entry for each input link. If "merge_nested" is specified with a single link, the value from the link must be wrapped in a single-item list. * **merge_flattened** 1. The source and sink parameters must be compatible types, or the source type must be compatible with single element from the "items" type of the destination array parameter. 2. Source parameters which are arrays are concatenated. Source parameters which are single element types are appended as single elements. fields: - name: id type: string jsonldPredicate: "@id" doc: "A unique identifier for this workflow input parameter." - name: default type: ["null", Any] doc: | The default value for this parameter if there is no `source` field. jsonldPredicate: "cwl:default" - name: valueFrom type: - "null" - "string" - "#Expression" jsonldPredicate: "cwl:valueFrom" doc: | To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must be specified in the workflow or workflow step requirements. If `valueFrom` is a constant string value, use this as the value for this input parameter. If `valueFrom` is a parameter reference or expression, it must be evaluated to yield the actual value to be assiged to the input field. The `self` value of in the parameter reference or expression must be the value of the parameter(s) specified in the `source` field, or null if there is no `source` field. The value of `inputs` in the parameter reference or expression must be the input object to the workflow step after assigning the `source` values and then scattering. The order of evaluating `valueFrom` among step input parameters is undefined and the result of evaluating `valueFrom` on a parameter must not be visible to evaluation of `valueFrom` on other parameters. - type: record name: WorkflowStepOutput docParent: "#WorkflowStep" doc: | Associate an output parameter of the underlying process with a workflow parameter. The workflow parameter (given in the `id` field) be may be used as a `source` to connect with input parameters of other workflow steps, or with an output parameter of the process. fields: - name: id type: string jsonldPredicate: "@id" doc: | A unique identifier for this workflow output parameter. This is the identifier to use in the `source` field of `WorkflowStepInput` to connect the output value to downstream parameters. - name: ScatterMethod type: enum docParent: "#WorkflowStep" doc: The scatter method, as described in [workflow step scatter](#WorkflowStep). symbols: - dotproduct - nested_crossproduct - flat_crossproduct - name: WorkflowStep type: record docParent: "#Workflow" doc: | A workflow step is an executable element of a workflow. It specifies the underlying process implementation (such as `CommandLineTool` or another `Workflow`) in the `run` field and connects the input and output parameters of the underlying process to workflow parameters. # Scatter/gather To use scatter/gather, [ScatterFeatureRequirement](#ScatterFeatureRequirement) must be specified in the workflow or workflow step requirements. A "scatter" operation specifies that the associated workflow step or subworkflow should execute separately over a list of input elements. Each job making up a scatter operation is independent and may be executed concurrently. The `scatter` field specifies one or more input parameters which will be scattered. An input parameter may be listed more than once. The declared type of each input parameter is implicitly wrapped in an array for each time it appears in the `scatter` field. As a result, upstream parameters which are connected to scattered parameters may be arrays. All output parameter types are also implicitly wrapped in arrays. Each job in the scatter results in an entry in the output array. If `scatter` declares more than one input parameter, `scatterMethod` describes how to decompose the input into a discrete set of jobs. * **dotproduct** specifies that each of the input arrays are aligned and one element taken from each array to construct each job. It is an error if all input arrays are not the same length. * **nested_crossproduct** specifies the Cartesian product of the inputs, producing a job for every combination of the scattered inputs. The output must be nested arrays for each level of scattering, in the order that the input arrays are listed in the `scatter` field. * **flat_crossproduct** specifies the Cartesian product of the inputs, producing a job for every combination of the scattered inputs. The output arrays must be flattened to a single level, but otherwise listed in the order that the input arrays are listed in the `scatter` field. # Subworkflows To specify a nested workflow as part of a workflow step, [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) must be specified in the workflow or workflow step requirements. fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this workflow step." - name: in type: WorkflowStepInput[] jsonldPredicate: _id: "cwl:in" mapSubject: id mapPredicate: source doc: | Defines the input parameters of the workflow step. The process is ready to run when all required input parameters are associated with concrete values. Input parameters include a schema for each parameter which is used to validate the input object. It may also be used build a user interface for constructing the input object. - name: out type: - type: array items: [string, WorkflowStepOutput] jsonldPredicate: _id: "cwl:out" _type: "@id" identity: true doc: | Defines the parameters representing the output of the process. May be used to generate and/or validate the output object. - name: requirements type: ProcessRequirement[]? jsonldPredicate: _id: "cwl:requirements" mapSubject: class doc: | Declares requirements that apply to either the runtime environment or the workflow engine that must be met in order to execute this workflow step. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. - name: hints type: Any[]? jsonldPredicate: _id: "cwl:hints" noLinkCheck: true mapSubject: class doc: | Declares hints applying to either the runtime environment or the workflow engine that may be helpful in executing this workflow step. It is not an error if an implementation cannot satisfy all hints, however the implementation may report a warning. - name: label type: string? jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: doc type: string? jsonldPredicate: "rdfs:comment" doc: "A long, human-readable description of this process object." - name: run type: [string, Process] jsonldPredicate: "_id": "cwl:run" "_type": "@id" doc: | Specifies the process to run. - name: scatter type: - string? - string[]? jsonldPredicate: "_id": "cwl:scatter" "_type": "@id" "_container": "@list" refScope: 0 - name: scatterMethod doc: | Required if `scatter` is an array of more than one element. type: ScatterMethod? jsonldPredicate: "_id": "cwl:scatterMethod" "_type": "@vocab" - name: Workflow type: record extends: "#Process" documentRoot: true specialize: - specializeFrom: "#OutputParameter" specializeTo: "#WorkflowOutputParameter" doc: | A workflow describes a set of **steps** and the **dependencies** between those steps. When a step produces output that will be consumed by a second step, the first step is a dependency of the second step. When there is a dependency, the workflow engine must execute the preceeding step and wait for it to successfully produce output before executing the dependent step. If two steps are defined in the workflow graph that are not directly or indirectly dependent, these steps are **independent**, and may execute in any order or execute concurrently. A workflow is complete when all steps have been executed. Dependencies between parameters are expressed using the `source` field on [workflow step input parameters](#WorkflowStepInput) and [workflow output parameters](#WorkflowOutputParameter). The `source` field expresses the dependency of one parameter on another such that when a value is associated with the parameter specified by `source`, that value is propagated to the destination parameter. When all data links inbound to a given step are fufilled, the step is ready to execute. ## Workflow success and failure A completed step must result in one of `success`, `temporaryFailure` or `permanentFailure` states. An implementation may choose to retry a step execution which resulted in `temporaryFailure`. An implementation may choose to either continue running other steps of a workflow, or terminate immediately upon `permanentFailure`. * If any step of a workflow execution results in `permanentFailure`, then the workflow status is `permanentFailure`. * If one or more steps result in `temporaryFailure` and all other steps complete `success` or are not executed, then the workflow status is `temporaryFailure`. * If all workflow steps are executed and complete with `success`, then the workflow status is `success`. # Extensions [ScatterFeatureRequirement](#ScatterFeatureRequirement) and [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) are available as standard [extensions](#Extensions_and_Metadata) to core workflow semantics. fields: - name: "class" jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: steps doc: | The individual steps that make up the workflow. Each step is executed when all of its input data links are fufilled. An implementation may choose to execute the steps in a different order than listed and/or execute steps concurrently, provided that dependencies between steps are met. type: - type: array items: "#WorkflowStep" jsonldPredicate: mapSubject: id - type: record name: SubworkflowFeatureRequirement extends: ProcessRequirement doc: | Indicates that the workflow platform must support nested workflows in the `run` field of [WorkflowStep](#WorkflowStep). fields: - name: "class" type: "string" doc: "Always 'SubworkflowFeatureRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: ScatterFeatureRequirement type: record extends: ProcessRequirement doc: | Indicates that the workflow platform must support the `scatter` and `scatterMethod` fields of [WorkflowStep](#WorkflowStep). fields: - name: "class" type: "string" doc: "Always 'ScatterFeatureRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: MultipleInputFeatureRequirement type: record extends: ProcessRequirement doc: | Indicates that the workflow platform must support multiple inbound data links listed in the `source` field of [WorkflowStepInput](#WorkflowStepInput). fields: - name: "class" type: "string" doc: "Always 'MultipleInputFeatureRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - type: record name: StepInputExpressionRequirement extends: ProcessRequirement doc: | Indicate that the workflow platform must support the `valueFrom` field of [WorkflowStepInput](#WorkflowStepInput). fields: - name: "class" type: "string" doc: "Always 'StepInputExpressionRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" cwltool-1.0.20170114120503/cwltool/schemas/v1.0/contrib.md0000644000175000017500000000200313036412052023177 0ustar michaelmichael00000000000000Authors: * Peter Amstutz , Arvados Project, Curoverse * Michael R. Crusoe , Common Workflow Language project * Nebojša Tijanić , Seven Bridges Genomics Contributors: * Brad Chapman , Harvard Chan School of Public Health * John Chilton , Galaxy Project, Pennsylvania State University * Michael Heuer ,UC Berkeley AMPLab * Andrey Kartashov , Cincinnati Children's Hospital * Dan Leehr , Duke University * Hervé Ménager , Institut Pasteur * Maya Nedeljkovich , Seven Bridges Genomics * Matt Scales , Institute of Cancer Research, London * Stian Soiland-Reyes [soiland-reyes@cs.manchester.ac.uk](mailto:soiland-reyes@cs.manchester.ac.uk), University of Manchester * Luka Stojanovic , Seven Bridges Genomics cwltool-1.0.20170114120503/cwltool/schemas/v1.0/README.md0000644000175000017500000000150113036412052022476 0ustar michaelmichael00000000000000# Common Workflow Language Specifications, v1.0 The CWL specifications are divided up into several documents. The [User Guide](UserGuide.html) provides a gentle introduction to writing CWL command line tools and workflows. The [Command Line Tool Description Specification](CommandLineTool.html) specifies the document schema and execution semantics for wrapping and executing command line tools. The [Workflow Description Specification](Workflow.html) specifies the document schema and execution semantics for composing workflows from components such as command line tools and other workflows. The [Semantic Annotations for Linked Avro Data (SALAD) Specification](SchemaSalad.html) specifies the preprocessing steps that must be applied when loading CWL documents and the schema language used to write the above specifications. cwltool-1.0.20170114120503/cwltool/schemas/v1.0/CommandLineTool.yml0000644000175000017500000007346413036412052025007 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" $graph: - name: CommandLineToolDoc type: documentation doc: - | # Common Workflow Language (CWL) Command Line Tool Description, v1.0 This version: * https://w3id.org/cwl/v1.0/ Current version: * https://w3id.org/cwl/ - "\n\n" - {$include: contrib.md} - "\n\n" - | # Abstract A Command Line Tool is a non-interactive executable program that reads some input, performs a computation, and terminates after producing some output. Command line programs are a flexible unit of code sharing and reuse, unfortunately the syntax and input/output semantics among command line programs is extremely heterogeneous. A common layer for describing the syntax and semantics of programs can reduce this incidental complexity by providing a consistent way to connect programs together. This specification defines the Common Workflow Language (CWL) Command Line Tool Description, a vendor-neutral standard for describing the syntax and input/output semantics of command line programs. - {$include: intro.md} - | ## Introduction to v1.0 This specification represents the first full release from the CWL group. Since draft-3, version 1.0 introduces the following changes and additions: * The [Directory](#Directory) type. * Syntax simplifcations: denoted by the `map<>` syntax. Example: inputs contains a list of items, each with an id. Now one can specify a mapping of that identifier to the corresponding `CommandInputParamater`. ``` inputs: - id: one type: string doc: First input parameter - id: two type: int doc: Second input parameter ``` can be ``` inputs: one: type: string doc: First input parameter two: type: int doc: Second input parameter ``` * [InitialWorkDirRequirement](#InitialWorkDirRequirement): list of files and subdirectories to be present in the output directory prior to execution. * Shortcuts for specifying the standard [output](#stdout) and/or [error](#stderr) streams as a (streamable) File output. * [SoftwareRequirement](#SoftwareRequirement) for describing software dependencies of a tool. * The common `description` field has been renamed to `doc`. ## Errata Post v1.0 release changes to the spec. * 13 July 2016: Mark `baseCommand` as optional and update descriptive text. ## Purpose Standalone programs are a flexible and interoperable form of code reuse. Unlike monolithic applications, applications and analysis workflows which are composed of multiple separate programs can be written in multiple languages and execute concurrently on multiple hosts. However, POSIX does not dictate computer-readable grammar or semantics for program input and output, resulting in extremely heterogeneous command line grammar and input/output semantics among program. This is a particular problem in distributed computing (multi-node compute clusters) and virtualized environments (such as Docker containers) where it is often necessary to provision resources such as input files before executing the program. Often this gap is filled by hard coding program invocation and implicitly assuming requirements will be met, or abstracting program invocation with wrapper scripts or descriptor documents. Unfortunately, where these approaches are application or platform specific it creates a significant barrier to reproducibility and portability, as methods developed for one platform must be manually ported to be used on new platforms. Similarly it creates redundant work, as wrappers for popular tools must be rewritten for each application or platform in use. The Common Workflow Language Command Line Tool Description is designed to provide a common standard description of grammar and semantics for invoking programs used in data-intensive fields such as Bioinformatics, Chemistry, Physics, Astronomy, and Statistics. This specification defines a precise data and execution model for Command Line Tools that can be implemented on a variety of computing platforms, ranging from a single workstation to cluster, grid, cloud, and high performance computing platforms. - {$include: concepts.md} - {$include: invocation.md} - type: record name: EnvironmentDef doc: | Define an environment variable that will be set in the runtime environment by the workflow platform when executing the command line tool. May be the result of executing an expression, such as getting a parameter from input. fields: - name: envName type: string doc: The environment variable name - name: envValue type: [string, Expression] doc: The environment variable value - type: record name: CommandLineBinding extends: InputBinding doc: | When listed under `inputBinding` in the input schema, the term "value" refers to the the corresponding value in the input object. For binding objects listed in `CommandLineTool.arguments`, the term "value" refers to the effective value after evaluating `valueFrom`. The binding behavior when building the command line depends on the data type of the value. If there is a mismatch between the type described by the input schema and the effective value, such as resulting from an expression evaluation, an implementation must use the data type of the effective value. - **string**: Add `prefix` and the string to the command line. - **number**: Add `prefix` and decimal representation to command line. - **boolean**: If true, add `prefix` to the command line. If false, add nothing. - **File**: Add `prefix` and the value of [`File.path`](#File) to the command line. - **array**: If `itemSeparator` is specified, add `prefix` and the join the array into a single string with `itemSeparator` separating the items. Otherwise first add `prefix`, then recursively process individual elements. - **object**: Add `prefix` only, and recursively add object fields for which `inputBinding` is specified. - **null**: Add nothing. fields: - name: position type: int? doc: "The sorting key. Default position is 0." - name: prefix type: string? doc: "Command line prefix to add before the value." - name: separate type: boolean? doc: | If true (default), then the prefix and value must be added as separate command line arguments; if false, prefix and value must be concatenated into a single command line argument. - name: itemSeparator type: string? doc: | Join the array elements into a single string with the elements separated by by `itemSeparator`. - name: valueFrom type: - "null" - string - Expression jsonldPredicate: "cwl:valueFrom" doc: | If `valueFrom` is a constant string value, use this as the value and apply the binding rules above. If `valueFrom` is an expression, evaluate the expression to yield the actual value to use to build the command line and apply the binding rules above. If the inputBinding is associated with an input parameter, the value of `self` in the expression will be the value of the input parameter. When a binding is part of the `CommandLineTool.arguments` field, the `valueFrom` field is required. - name: shellQuote type: boolean? doc: | If `ShellCommandRequirement` is in the requirements for the current command, this controls whether the value is quoted on the command line (default is true). Use `shellQuote: false` to inject metacharacters for operations such as pipes. - type: record name: CommandOutputBinding extends: OutputBinding doc: | Describes how to generate an output parameter based on the files produced by a CommandLineTool. The output parameter is generated by applying these operations in the following order: - glob - loadContents - outputEval fields: - name: glob type: - "null" - string - Expression - type: array items: string doc: | Find files relative to the output directory, using POSIX glob(3) pathname matching. If an array is provided, find files that match any pattern in the array. If an expression is provided, the expression must return a string or an array of strings, which will then be evaluated as one or more glob patterns. Must only match and return files which actually exist. - name: loadContents type: - "null" - boolean jsonldPredicate: "cwl:loadContents" doc: | For each file matched in `glob`, read up to the first 64 KiB of text from the file and place it in the `contents` field of the file object for manipulation by `outputEval`. - name: outputEval type: - "null" - string - Expression doc: | Evaluate an expression to generate the output value. If `glob` was specified, the value of `self` must be an array containing file objects that were matched. If no files were matched, `self` must be a zero length array; if a single file was matched, the value of `self` is an array of a single element. Additionally, if `loadContents` is `true`, the File objects must include up to the first 64 KiB of file contents in the `contents` field. - name: CommandInputRecordField type: record extends: InputRecordField specialize: - specializeFrom: InputRecordSchema specializeTo: CommandInputRecordSchema - specializeFrom: InputEnumSchema specializeTo: CommandInputEnumSchema - specializeFrom: InputArraySchema specializeTo: CommandInputArraySchema - specializeFrom: InputBinding specializeTo: CommandLineBinding - name: CommandInputRecordSchema type: record extends: InputRecordSchema specialize: - specializeFrom: InputRecordField specializeTo: CommandInputRecordField - name: CommandInputEnumSchema type: record extends: InputEnumSchema specialize: - specializeFrom: InputBinding specializeTo: CommandLineBinding - name: CommandInputArraySchema type: record extends: InputArraySchema specialize: - specializeFrom: InputRecordSchema specializeTo: CommandInputRecordSchema - specializeFrom: InputEnumSchema specializeTo: CommandInputEnumSchema - specializeFrom: InputArraySchema specializeTo: CommandInputArraySchema - specializeFrom: InputBinding specializeTo: CommandLineBinding - name: CommandOutputRecordField type: record extends: OutputRecordField specialize: - specializeFrom: OutputRecordSchema specializeTo: CommandOutputRecordSchema - specializeFrom: OutputEnumSchema specializeTo: CommandOutputEnumSchema - specializeFrom: OutputArraySchema specializeTo: CommandOutputArraySchema - specializeFrom: OutputBinding specializeTo: CommandOutputBinding - name: CommandOutputRecordSchema type: record extends: OutputRecordSchema specialize: - specializeFrom: OutputRecordField specializeTo: CommandOutputRecordField - name: CommandOutputEnumSchema type: record extends: OutputEnumSchema specialize: - specializeFrom: OutputRecordSchema specializeTo: CommandOutputRecordSchema - specializeFrom: OutputEnumSchema specializeTo: CommandOutputEnumSchema - specializeFrom: OutputArraySchema specializeTo: CommandOutputArraySchema - specializeFrom: OutputBinding specializeTo: CommandOutputBinding - name: CommandOutputArraySchema type: record extends: OutputArraySchema specialize: - specializeFrom: OutputRecordSchema specializeTo: CommandOutputRecordSchema - specializeFrom: OutputEnumSchema specializeTo: CommandOutputEnumSchema - specializeFrom: OutputArraySchema specializeTo: CommandOutputArraySchema - specializeFrom: OutputBinding specializeTo: CommandOutputBinding - type: record name: CommandInputParameter extends: InputParameter doc: An input parameter for a CommandLineTool. specialize: - specializeFrom: InputRecordSchema specializeTo: CommandInputRecordSchema - specializeFrom: InputEnumSchema specializeTo: CommandInputEnumSchema - specializeFrom: InputArraySchema specializeTo: CommandInputArraySchema - specializeFrom: InputBinding specializeTo: CommandLineBinding - type: record name: CommandOutputParameter extends: OutputParameter doc: An output parameter for a CommandLineTool. specialize: - specializeFrom: OutputBinding specializeTo: CommandOutputBinding fields: - name: type type: - "null" - CWLType - stdout - stderr - CommandOutputRecordSchema - CommandOutputEnumSchema - CommandOutputArraySchema - string - type: array items: - CWLType - CommandOutputRecordSchema - CommandOutputEnumSchema - CommandOutputArraySchema - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" refScope: 2 typeDSL: True doc: | Specify valid types of data that may be assigned to this parameter. - name: stdout type: enum symbols: [ "cwl:stdout" ] docParent: "#CommandOutputParameter" doc: | Only valid as a `type` for a `CommandLineTool` output with no `outputBinding` set. The following ``` outputs: an_output_name: type: stdout stdout: a_stdout_file ``` is equivalent to ``` outputs: an_output_name: type: File streamable: true outputBinding: glob: a_stdout_file stdout: a_stdout_file ``` If there is no `stdout` name provided, a random filename will be created. For example, the following ``` outputs: an_output_name: type: stdout ``` is equivalent to ``` outputs: an_output_name: type: File streamable: true outputBinding: glob: random_stdout_filenameABCDEFG stdout: random_stdout_filenameABCDEFG ``` - name: stderr type: enum symbols: [ "cwl:stderr" ] docParent: "#CommandOutputParameter" doc: | Only valid as a `type` for a `CommandLineTool` output with no `outputBinding` set. The following ``` outputs: an_output_name: type: stderr stderr: a_stderr_file ``` is equivalent to ``` outputs: an_output_name: type: File streamable: true outputBinding: glob: a_stderr_file stderr: a_stderr_file ``` If there is no `stderr` name provided, a random filename will be created. For example, the following ``` outputs: an_output_name: type: stderr ``` is equivalent to ``` outputs: an_output_name: type: File streamable: true outputBinding: glob: random_stderr_filenameABCDEFG stderr: random_stderr_filenameABCDEFG ``` - type: record name: CommandLineTool extends: Process documentRoot: true specialize: - specializeFrom: InputParameter specializeTo: CommandInputParameter - specializeFrom: OutputParameter specializeTo: CommandOutputParameter doc: | This defines the schema of the CWL Command Line Tool Description document. fields: - name: class jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: baseCommand doc: | Specifies the program to execute. If an array, the first element of the array is the command to execute, and subsequent elements are mandatory command line arguments. The elements in `baseCommand` must appear before any command line bindings from `inputBinding` or `arguments`. If `baseCommand` is not provided or is an empty array, the first element of the command line produced after processing `inputBinding` or `arguments` must be used as the program to execute. If the program includes a path separator character it must be an absolute path, otherwise it is an error. If the program does not include a path separator, search the `$PATH` variable in the runtime environment of the workflow runner find the absolute path of the executable. type: - string? - string[]? jsonldPredicate: "_id": "cwl:baseCommand" "_container": "@list" - name: arguments doc: | Command line bindings which are not directly associated with input parameters. type: - "null" - type: array items: [string, Expression, CommandLineBinding] jsonldPredicate: "_id": "cwl:arguments" "_container": "@list" - name: stdin type: ["null", string, Expression] doc: | A path to a file whose contents must be piped into the command's standard input stream. - name: stderr type: ["null", string, Expression] jsonldPredicate: "https://w3id.org/cwl/cwl#stderr" doc: | Capture the command's standard error stream to a file written to the designated output directory. If `stderr` is a string, it specifies the file name to use. If `stderr` is an expression, the expression is evaluated and must return a string with the file name to use to capture stderr. If the return value is not a string, or the resulting path contains illegal characters (such as the path separator `/`) it is an error. - name: stdout type: ["null", string, Expression] jsonldPredicate: "https://w3id.org/cwl/cwl#stdout" doc: | Capture the command's standard output stream to a file written to the designated output directory. If `stdout` is a string, it specifies the file name to use. If `stdout` is an expression, the expression is evaluated and must return a string with the file name to use to capture stdout. If the return value is not a string, or the resulting path contains illegal characters (such as the path separator `/`) it is an error. - name: successCodes type: int[]? doc: | Exit codes that indicate the process completed successfully. - name: temporaryFailCodes type: int[]? doc: | Exit codes that indicate the process failed due to a possibly temporary condition, where executing the process with the same runtime environment and inputs may produce different results. - name: permanentFailCodes type: int[]? doc: Exit codes that indicate the process failed due to a permanent logic error, where executing the process with the same runtime environment and same inputs is expected to always fail. - type: record name: DockerRequirement extends: ProcessRequirement doc: | Indicates that a workflow component should be run in a [Docker](http://docker.com) container, and specifies how to fetch or build the image. If a CommandLineTool lists `DockerRequirement` under `hints` (or `requirements`), it may (or must) be run in the specified Docker container. The platform must first acquire or install the correct Docker image as specified by `dockerPull`, `dockerImport`, `dockerLoad` or `dockerFile`. The platform must execute the tool in the container using `docker run` with the appropriate Docker image and tool command line. The workflow platform may provide input files and the designated output directory through the use of volume bind mounts. The platform may rewrite file paths in the input object to correspond to the Docker bind mounted locations. When running a tool contained in Docker, the workflow platform must not assume anything about the contents of the Docker container, such as the presence or absence of specific software, except to assume that the generated command line represents a valid command within the runtime environment of the container. ## Interaction with other requirements If [EnvVarRequirement](#EnvVarRequirement) is specified alongside a DockerRequirement, the environment variables must be provided to Docker using `--env` or `--env-file` and interact with the container's preexisting environment as defined by Docker. fields: - name: class type: string doc: "Always 'DockerRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: dockerPull type: string? doc: "Specify a Docker image to retrieve using `docker pull`." - name: dockerLoad type: string? doc: "Specify a HTTP URL from which to download a Docker image using `docker load`." - name: dockerFile type: string? doc: "Supply the contents of a Dockerfile which will be built using `docker build`." - name: dockerImport type: string? doc: "Provide HTTP URL to download and gunzip a Docker images using `docker import." - name: dockerImageId type: string? doc: | The image id that will be used for `docker run`. May be a human-readable image name or the image identifier hash. May be skipped if `dockerPull` is specified, in which case the `dockerPull` image id must be used. - name: dockerOutputDirectory type: string? doc: | Set the designated output directory to a specific location inside the Docker container. - type: record name: SoftwareRequirement extends: ProcessRequirement doc: | A list of software packages that should be configured in the environment of the defined process. fields: - name: class type: string doc: "Always 'SoftwareRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: packages type: SoftwarePackage[] doc: "The list of software to be configured." jsonldPredicate: mapSubject: package mapPredicate: specs - name: SoftwarePackage type: record fields: - name: package type: string doc: "The common name of the software to be configured." - name: version type: string[]? doc: "The (optional) version of the software to configured." - name: specs type: string[]? doc: | Must be one or more IRIs identifying resources for installing or enabling the software. Implementations may provide resolvers which map well-known software spec IRIs to some configuration action. For example, an IRI `https://packages.debian.org/jessie/bowtie` could be resolved with `apt-get install bowtie`. An IRI `https://anaconda.org/bioconda/bowtie` could be resolved with `conda install -c bioconda bowtie`. Tools may also provide IRIs to index entries such as [RRID](http://www.identifiers.org/rrid/), such as `http://identifiers.org/rrid/RRID:SCR_005476` - name: Dirent type: record doc: | Define a file or subdirectory that must be placed in the designated output directory prior to executing the command line tool. May be the result of executing an expression, such as building a configuration file from a template. fields: - name: entryname type: ["null", string, Expression] jsonldPredicate: _id: cwl:entryname doc: | The name of the file or subdirectory to create in the output directory. If `entry` is a File or Directory, this overrides `basename`. Optional. - name: entry type: [string, Expression] jsonldPredicate: _id: cwl:entry doc: | If the value is a string literal or an expression which evaluates to a string, a new file must be created with the string as the file contents. If the value is an expression that evaluates to a `File` object, this indicates the referenced file should be added to the designated output directory prior to executing the tool. If the value is an expression that evaluates to a `Dirent` object, this indicates that the File or Directory in `entry` should be added to the designated output directory with the name in `entryname`. If `writable` is false, the file may be made available using a bind mount or file system link to avoid unnecessary copying of the input file. - name: writable type: boolean? doc: | If true, the file or directory must be writable by the tool. Changes to the file or directory must be isolated and not visible by any other CommandLineTool process. This may be implemented by making a copy of the original file or directory. Default false (files and directories read-only by default). - name: InitialWorkDirRequirement type: record extends: ProcessRequirement doc: Define a list of files and subdirectories that must be created by the workflow platform in the designated output directory prior to executing the command line tool. fields: - name: class type: string doc: InitialWorkDirRequirement jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: listing type: - type: array items: [File, Directory, Dirent, string, Expression] - string - Expression jsonldPredicate: _id: "cwl:listing" doc: | The list of files or subdirectories that must be placed in the designated output directory prior to executing the command line tool. May be an expression. If so, the expression return value must validate as `{type: array, items: [File, Directory]}`. - name: EnvVarRequirement type: record extends: ProcessRequirement doc: | Define a list of environment variables which will be set in the execution environment of the tool. See `EnvironmentDef` for details. fields: - name: class type: string doc: "Always 'EnvVarRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: envDef type: EnvironmentDef[] doc: The list of environment variables. jsonldPredicate: mapSubject: envName mapPredicate: envValue - type: record name: ShellCommandRequirement extends: ProcessRequirement doc: | Modify the behavior of CommandLineTool to generate a single string containing a shell command line. Each item in the argument list must be joined into a string separated by single spaces and quoted to prevent intepretation by the shell, unless `CommandLineBinding` for that argument contains `shellQuote: false`. If `shellQuote: false` is specified, the argument is joined into the command string without quoting, which allows the use of shell metacharacters such as `|` for pipes. fields: - name: class type: string doc: "Always 'ShellCommandRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - type: record name: ResourceRequirement extends: ProcessRequirement doc: | Specify basic hardware resource requirements. "min" is the minimum amount of a resource that must be reserved to schedule a job. If "min" cannot be satisfied, the job should not be run. "max" is the maximum amount of a resource that the job shall be permitted to use. If a node has sufficient resources, multiple jobs may be scheduled on a single node provided each job's "max" resource requirements are met. If a job attempts to exceed its "max" resource allocation, an implementation may deny additional resources, which may result in job failure. If "min" is specified but "max" is not, then "max" == "min" If "max" is specified by "min" is not, then "min" == "max". It is an error if max < min. It is an error if the value of any of these fields is negative. If neither "min" nor "max" is specified for a resource, an implementation may provide a default. fields: - name: class type: string doc: "Always 'ResourceRequirement'" jsonldPredicate: "_id": "@type" "_type": "@vocab" - name: coresMin type: ["null", long, string, Expression] doc: Minimum reserved number of CPU cores - name: coresMax type: ["null", int, string, Expression] doc: Maximum reserved number of CPU cores - name: ramMin type: ["null", long, string, Expression] doc: Minimum reserved RAM in mebibytes (2**20) - name: ramMax type: ["null", long, string, Expression] doc: Maximum reserved RAM in mebibytes (2**20) - name: tmpdirMin type: ["null", long, string, Expression] doc: Minimum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20) - name: tmpdirMax type: ["null", long, string, Expression] doc: Maximum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20) - name: outdirMin type: ["null", long, string, Expression] doc: Minimum reserved filesystem based storage for the designated output directory, in mebibytes (2**20) - name: outdirMax type: ["null", long, string, Expression] doc: Maximum reserved filesystem based storage for the designated output directory, in mebibytes (2**20) cwltool-1.0.20170114120503/cwltool/schemas/v1.0/CommonWorkflowLanguage.yml0000644000175000017500000000032113036412052026370 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" sld: "https://w3id.org/cwl/salad#" $graph: - $import: Process.yml - $import: CommandLineTool.yml - $import: Workflow.yml cwltool-1.0.20170114120503/cwltool/schemas/v1.0/invocation.md0000644000175000017500000001732113036412052023721 0ustar michaelmichael00000000000000# Running a Command To accommodate the enormous variety in syntax and semantics for input, runtime environment, invocation, and output of arbitrary programs, a CommandLineTool defines an "input binding" that describes how to translate abstract input parameters to an concrete program invocation, and an "output binding" that describes how to generate output parameters from program output. ## Input binding The tool command line is built by applying command line bindings to the input object. Bindings are listed either as part of an [input parameter](#CommandInputParameter) using the `inputBinding` field, or separately using the `arguments` field of the CommandLineTool. The algorithm to build the command line is as follows. In this algorithm, the sort key is a list consisting of one or more numeric or string elements. Strings are sorted lexicographically based on UTF-8 encoding. 1. Collect `CommandLineBinding` objects from `arguments`. Assign a sorting key `[position, i]` where `position` is [`CommandLineBinding.position`](#CommandLineBinding) and `i` is the index in the `arguments` list. 2. Collect `CommandLineBinding` objects from the `inputs` schema and associate them with values from the input object. Where the input type is a record, array, or map, recursively walk the schema and input object, collecting nested `CommandLineBinding` objects and associating them with values from the input object. 3. Create a sorting key by taking the value of the `position` field at each level leading to each leaf binding object. If `position` is not specified, it is not added to the sorting key. For bindings on arrays and maps, the sorting key must include the array index or map key following the position. If and only if two bindings have the same sort key, the tie must be broken using the ordering of the field or parameter name immediately containing the leaf binding. 4. Sort elements using the assigned sorting keys. Numeric entries sort before strings. 5. In the sorted order, apply the rules defined in [`CommandLineBinding`](#CommandLineBinding) to convert bindings to actual command line elements. 6. Insert elements from `baseCommand` at the beginning of the command line. ## Runtime environment All files listed in the input object must be made available in the runtime environment. The implementation may use a shared or distributed file system or transfer files via explicit download to the host. Implementations may choose not to provide access to files not explicitly specified in the input object or process requirements. Output files produced by tool execution must be written to the **designated output directory**. The initial current working directory when executing the tool must be the designated output directory. Files may also be written to the **designated temporary directory**. This directory must be isolated and not shared with other processes. Any files written to the designated temporary directory may be automatically deleted by the workflow platform immediately after the tool terminates. For compatibility, files may be written to the **system temporary directory** which must be located at `/tmp`. Because the system temporary directory may be shared with other processes on the system, files placed in the system temporary directory are not guaranteed to be deleted automatically. A tool must not use the system temporary directory as a backchannel communication with other tools. It is valid for the system temporary directory to be the same as the designated temporary directory. When executing the tool, the tool must execute in a new, empty environment with only the environment variables described below; the child process must not inherit environment variables from the parent process except as specified or at user option. * `HOME` must be set to the designated output directory. * `TMPDIR` must be set to the designated temporary directory. * `PATH` may be inherited from the parent process, except when run in a container that provides its own `PATH`. * Variables defined by [EnvVarRequirement](#EnvVarRequirement) * The default environment of the container, such as when using [DockerRequirement](#DockerRequirement) An implementation may forbid the tool from writing to any location in the runtime environment file system other than the designated temporary directory, system temporary directory, and designated output directory. An implementation may provide read-only input files, and disallow in-place update of input files. The designated temporary directory, system temporary directory and designated output directory may each reside on different mount points on different file systems. An implementation may forbid the tool from directly accessing network resources. Correct tools must not assume any network access. Future versions of the specification may incorporate optional process requirements that describe the networking needs of a tool. The `runtime` section available in [parameter references](#Parameter_references) and [expressions](#Expressions) contains the following fields. As noted earlier, an implementation may perform deferred resolution of runtime fields by providing opaque strings for any or all of the following fields; parameter references and expressions may only use the literal string value of the field and must not perform computation on the contents. * `runtime.outdir`: an absolute path to the designated output directory * `runtime.tmpdir`: an absolute path to the designated temporary directory * `runtime.cores`: number of CPU cores reserved for the tool process * `runtime.ram`: amount of RAM in mebibytes (2\*\*20) reserved for the tool process * `runtime.outdirSize`: reserved storage space available in the designated output directory * `runtime.tmpdirSize`: reserved storage space available in the designated temporary directory For `cores`, `ram`, `outdirSize` and `tmpdirSize`, if an implementation can't provide the actual number of reserved cores during the expression evaluation time, it should report back the minimal requested amount. See [ResourceRequirement](#ResourceRequirement) for details on how to describe the hardware resources required by a tool. The standard input stream and standard output stream may be redirected as described in the `stdin` and `stdout` fields. ## Execution Once the command line is built and the runtime environment is created, the actual tool is executed. The standard error stream and standard output stream (unless redirected by setting `stdout` or `stderr`) may be captured by platform logging facilities for storage and reporting. Tools may be multithreaded or spawn child processes; however, when the parent process exits, the tool is considered finished regardless of whether any detached child processes are still running. Tools must not require any kind of console, GUI, or web based user interaction in order to start and run to completion. The exit code of the process indicates if the process completed successfully. By convention, an exit code of zero is treated as success and non-zero exit codes are treated as failure. This may be customized by providing the fields `successCodes`, `temporaryFailCodes`, and `permanentFailCodes`. An implementation may choose to default unspecified non-zero exit codes to either `temporaryFailure` or `permanentFailure`. ## Output binding If the output directory contains a file named "cwl.output.json", that file must be loaded and used as the output object. Otherwise, the output object must be generated by walking the parameters listed in `outputs` and applying output bindings to the tool output. Output bindings are associated with output parameters using the `outputBinding` field. See [`CommandOutputBinding`](#CommandOutputBinding) for details. cwltool-1.0.20170114120503/cwltool/schemas/v1.0/userguide-intro.md0000644000175000017500000000236413036412052024676 0ustar michaelmichael00000000000000# A Gentle Introduction to the Common Workflow Language Hello! This guide will introduce you to writing tool wrappers and workflows using the Common Workflow Language (CWL). This guide describes the current stable specification, version 1.0. Note: This document is a work in progress. Not all features are covered, yet. # Introduction CWL is a way to describe command line tools and connect them together to create workflows. Because CWL is a specification and not a specific piece of software, tools and workflows described using CWL are portable across a variety of platforms that support the CWL standard. CWL has roots in "make" and many similar tools that determine order of execution based on dependencies between tasks. However unlike "make", CWL tasks are isolated and you must be explicit about your inputs and outputs. The benefit of explicitness and isolation are flexibility, portability, and scalability: tools and workflows described with CWL can transparently leverage technologies such as Docker, be used with CWL implementations from different vendors, and is well suited for describing large-scale workflows in cluster, cloud and high performance computing environments where tasks are scheduled in parallel across many nodes. cwltool-1.0.20170114120503/cwltool/schemas/v1.0/CommandLineTool-standalone.yml0000644000175000017500000000006513036412052027120 0ustar michaelmichael00000000000000- $import: Process.yml - $import: CommandLineTool.ymlcwltool-1.0.20170114120503/cwltool/schemas/draft-3/0000755000175000017500000000000013036412107021777 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/0000755000175000017500000000000013036412107023063 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/0000755000175000017500000000000013036412107025467 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/0000755000175000017500000000000013036412107027576 5ustar michaelmichael00000000000000././@LongLink0000000000000000000000000000014700000000000011217 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/field_name_schema.ymlcwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/field_name_schema.y0000644000175000017500000000040113036412052033365 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "ExampleType", "type": "record", "fields": [{ "name": "base", "type": "string", "jsonldPredicate": "http://example.com/base" }] }] } cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/field_name_src.yml0000644000175000017500000000025313036412052033252 0ustar michaelmichael00000000000000 { "base": "one", "form": { "http://example.com/base": "two", "http://example.com/three": "three", }, "acid:four": "four" } cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/link_res_src.yml0000644000175000017500000000047113036412052032777 0ustar michaelmichael00000000000000{ "$base": "http://example.com/base", "link": "http://example.com/base/zero", "form": { "link": "one", "things": [ { "link": "two" }, { "link": "#three", }, { "link": "four#five", }, { "link": "acid:six", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/ident_res.yml0000644000175000017500000000323413036412052032276 0ustar michaelmichael00000000000000- | ## Identifier resolution The schema may designate one or more fields as identifier fields to identify specific objects. Processing must resolve relative identifiers to absolute identifiers using the following rules: * If an identifier URI is prefixed with `#` it is a URI relative fragment identifier. It is resolved relative to the base URI by setting or replacing the fragment portion of the base URI. * If an identifier URI does not contain a scheme and is not prefixed `#` it is a parent relative fragment identifier. It is resolved relative to the base URI by the following rule: if the base URI does not contain a document fragment, set the fragment portion of the base URI. If the base URI does contain a document fragment, append a slash `/` followed by the identifier field to the fragment portion of the base URI. * If an identifier URI begins with a namespace prefix declared in `$namespaces` followed by a colon `:`, the prefix and colon must be replaced by the namespace declared in `$namespaces`. * If an identifier URI is an absolute URI consisting of a scheme and path, no processing occurs. When preprocessing visits a node containing an identifier, that identifier must be used as the base URI to process child nodes. It is an error for more than one object in a document to have the same absolute URI. ### Identifier resolution example Given the following schema: ``` - $include: ident_res_schema.yml - | ``` Process the following example: ``` - $include: ident_res_src.yml - | ``` This becomes: ``` - $include: ident_res_proc.yml - | ``` cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/vocab_res_proc.yml0000644000175000017500000000036313036412052033310 0ustar michaelmichael00000000000000 { "form": { "things": [ { "voc": "red", }, { "voc": "red", }, { "voc": "http://example.com/acid#blue", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/link_res_proc.yml0000644000175000017500000000063313036412052033153 0ustar michaelmichael00000000000000{ "$base": "http://example.com/base", "link": "http://example.com/base/zero", "form": { "link": "http://example.com/one", "things": [ { "link": "http://example.com/two" }, { "link": "http://example.com/base#three" }, { "link": "http://example.com/four#five", }, { "link": "http://example.com/acid#six", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/ident_res_src.yml0000644000175000017500000000052213036412052033142 0ustar michaelmichael00000000000000 { "id": "http://example.com/base", "form": { "id": "one", "things": [ { "id": "two" }, { "id": "#three", }, { "id": "four#five", }, { "id": "acid:six", } ] } } ././@LongLink0000000000000000000000000000014600000000000011216 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/vocab_res_schema.ymlcwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/vocab_res_schema.ym0000644000175000017500000000053113036412052033426 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "Colors", "type": "enum", "symbols": ["acid:red"] }, { "name": "ExampleType", "type": "record", "fields": [{ "name": "voc", "type": "string", "jsonldPredicate": { "_type": "@vocab" } }] }] } cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/field_name.yml0000644000175000017500000000246013036412052032405 0ustar michaelmichael00000000000000- | ## Field name resolution The document schema declares the vocabulary of known field names. During preprocessing traversal, field name in the document which are not part of the schema vocabulary must be resolved to absolute URIs. Under "strict" validation, it is an error for a document to include fields which are not part of the vocabulary and not resolvable to absolute URIs. Fields names which are not part of the vocabulary are resolved using the following rules: * If an field name URI begins with a namespace prefix declared in the document context (`@context`) followed by a colon `:`, the prefix and colon must be replaced by the namespace declared in `@context`. * If there is a vocabulary term which maps to the URI of a resolved field, the field name must be replace with the vocabulary term. * If a field name URI is an absolute URI consisting of a scheme and path and is not part of the vocabulary, no processing occurs. Field name resolution is not relative. It must not be affected by the base URI. ### Field name resolution example Given the following schema: ``` - $include: field_name_schema.yml - | ``` Process the following example: ``` - $include: field_name_src.yml - | ``` This becomes: ``` - $include: field_name_proc.yml - | ``` cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/vocab_res.yml0000644000175000017500000000142313036412052032263 0ustar michaelmichael00000000000000- | ## Vocabulary resolution The schema may designate one or more vocabulary fields which use terms defined in the vocabulary. Processing must resolve vocabulary fields to either vocabulary terms or absolute URIs by first applying the link resolution rules defined above, then applying the following additional rule: * If a reference URI is a vocabulary field, and there is a vocabulary term which maps to the resolved URI, the reference must be replace with the vocabulary term. ### Vocabulary resolution example Given the following schema: ``` - $include: vocab_res_schema.yml - | ``` Process the following example: ``` - $include: vocab_res_src.yml - | ``` This becomes: ``` - $include: vocab_res_proc.yml - | ``` cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/field_name_proc.yml0000644000175000017500000000025313036412052033426 0ustar michaelmichael00000000000000 { "base": "one", "form": { "base": "two", "http://example.com/three": "three", }, "http://example.com/acid#four": "four" } ././@LongLink0000000000000000000000000000014600000000000011216 Lustar 00000000000000cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/ident_res_schema.ymlcwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/ident_res_schema.ym0000644000175000017500000000035313036412052033441 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "ExampleType", "type": "record", "fields": [{ "name": "id", "type": "string", "jsonldPredicate": "@id" }] }] } cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/import_include.md0000644000175000017500000000554313036412052033143 0ustar michaelmichael00000000000000## Import During preprocessing traversal, an implementation must resolve `$import` directives. An `$import` directive is an object consisting of exactly one field `$import` specifying resource by URI string. It is an error if there are additional fields in the `$import` object, such additional fields must be ignored. The URI string must be resolved to an absolute URI using the link resolution rules described previously. Implementations must support loading from `file`, `http` and `https` resources. The URI referenced by `$import` must be loaded and recursively preprocessed as a Salad document. The external imported document does not inherit the context of the importing document, and the default base URI for processing the imported document must be the URI used to retrieve the imported document. If the `$import` URI includes a document fragment, the fragment must be excluded from the base URI used to preprocess the imported document. Once loaded and processed, the `$import` node is replaced in the document structure by the object or array yielded from the import operation. URIs may reference document fragments which refer to specific an object in the target document. This indicates that the `$import` node must be replaced by only the object with the appropriate fragment identifier. It is a fatal error if an import directive refers to an external resource or resource fragment which does not exist or is not accessible. ### Import example import.yml: ``` { "hello": "world" } ``` parent.yml: ``` { "form": { "bar": { "$import": "import.yml" } } } ``` This becomes: ``` { "form": { "bar": { "hello": "world" } } } ``` ## Include During preprocessing traversal, an implementation must resolve `$include` directives. An `$include` directive is an object consisting of exactly one field `$include` specifying a URI string. It is an error if there are additional fields in the `$include` object, such additional fields must be ignored. The URI string must be resolved to an absolute URI using the link resolution rules described previously. The URI referenced by `$include` must be loaded as a text data. Implementations must support loading from `file`, `http` and `https` resources. Implementations may transcode the character encoding of the text data to match that of the parent document, but must not interpret or parse the text document in any other way. Once loaded, the `$include` node is replaced in the document structure by a string containing the text data loaded from the resource. It is a fatal error if an import directive refers to an external resource which does not exist or is not accessible. ### Include example parent.yml: ``` { "form": { "bar": { "$include": "include.txt" } } } ``` include.txt: ``` hello world ``` This becomes: ``` { "form": { "bar": "hello world" } } ``` cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/vocab_res_src.yml0000644000175000017500000000041313036412052033130 0ustar michaelmichael00000000000000 { "form": { "things": [ { "voc": "red", }, { "voc": "http://example.com/acid#red", }, { "voc": "http://example.com/acid#blue", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/link_res_schema.yml0000644000175000017500000000041013036412052033441 0ustar michaelmichael00000000000000{ "$namespaces": { "acid": "http://example.com/acid#" }, "$graph": [{ "name": "ExampleType", "type": "record", "fields": [{ "name": "link", "type": "string", "jsonldPredicate": { "_type": "@id" } }] }] } cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/salad.md0000644000175000017500000002435313036412052031212 0ustar michaelmichael00000000000000# Semantic Annotations for Linked Avro Data (SALAD) Author: * Peter Amstutz , Curoverse Contributors: * The developers of Apache Avro * The developers of JSON-LD * Nebojša Tijanić , Seven Bridges Genomics # Abstract Salad is a schema language for describing structured linked data documents in JSON or YAML documents. A Salad schema provides rules for preprocessing, structural validation, and link checking for documents described by a Salad schema. Salad builds on JSON-LD and the Apache Avro data serialization system, and extends Avro with features for rich data modeling such as inheritance, template specialization, object identifiers, and object references. Salad was developed to provide a bridge between the record oriented data modeling supported by Apache Avro and the Semantic Web. # Status of This Document This document is the product of the [Common Workflow Language working group](https://groups.google.com/forum/#!forum/common-workflow-language). The latest version of this document is available in the "schema_salad" directory at https://github.com/common-workflow-language/schema_salad The products of the CWL working group (including this document) are made available under the terms of the Apache License, version 2.0. # Introduction The JSON data model is an extremely popular way to represent structured data. It is attractive because of it's relative simplicity and is a natural fit with the standard types of many programming languages. However, this simplicity means that basic JSON lacks expressive features useful for working with complex data structures and document formats, such as schemas, object references, and namespaces. JSON-LD is a W3C standard providing a way to describe how to interpret a JSON document as Linked Data by means of a "context". JSON-LD provides a powerful solution for representing object references and namespaces in JSON based on standard web URIs, but is not itself a schema language. Without a schema providing a well defined structure, it is difficult to process an arbitrary JSON-LD document as idiomatic JSON because there are many ways to express the same data that are logically equivalent but structurally distinct. Several schema languages exist for describing and validating JSON data, such as the Apache Avro data serialization system, however none understand linked data. As a result, to fully take advantage of JSON-LD to build the next generation of linked data applications, one must maintain separate JSON schema, JSON-LD context, RDF schema, and human documentation, despite significant overlap of content and obvious need for these documents to stay synchronized. Schema Salad is designed to address this gap. It provides a schema language and processing rules for describing structured JSON content permitting URI resolution and strict document validation. The schema language supports linked data through annotations that describe the linked data interpretation of the content, enables generation of JSON-LD context and RDF schema, and production of RDF triples by applying the JSON-LD context. The schema language also provides for robust support of inline documentation. ## Introduction to draft 1 This is the first version of Schema Salad. It is developed concurrently with draft 3 of the Common Workflow Language for use in specifying the Common Workflow Language, however Schema Salad is intended to be useful to a broader audience. ## References to Other Specifications **Javascript Object Notation (JSON)**: http://json.org **JSON Linked Data (JSON-LD)**: http://json-ld.org **YAML**: http://yaml.org **Avro**: https://avro.apache.org/docs/current/spec.html **Uniform Resource Identifier (URI) Generic Syntax**: https://tools.ietf.org/html/rfc3986) **Resource Description Framework (RDF)**: http://www.w3.org/RDF/ **UTF-8**: https://www.ietf.org/rfc/rfc2279.txt) ## Scope This document describes the syntax, data model, algorithms, and schema language for working with Salad documents. It is not intended to document a specific implementation of Salad, however it may serve as a reference for the behavior of conforming implementations. ## Terminology The terminology used to describe Salad documents is defined in the Concepts section of the specification. The terms defined in the following list are used in building those definitions and in describing the actions of an Salad implementation: **may**: Conforming Salad documents and Salad implementations are permitted but not required to be interpreted as described. **must**: Conforming Salad documents and Salad implementations are required to be interpreted as described; otherwise they are in error. **error**: A violation of the rules of this specification; results are undefined. Conforming implementations may detect and report an error and may recover from it. **fatal error**: A violation of the rules of this specification; results are undefined. Conforming implementations must not continue to process the document and may report an error. **at user option**: Conforming software may or must (depending on the modal verb in the sentence) behave as described; if it does, it must provide users a means to enable or disable the behavior described. # Document model ## Data concepts An **object** is a data structure equivalent to the "object" type in JSON, consisting of a unordered set of name/value pairs (referred to here as **fields**) and where the name is a string and the value is a string, number, boolean, array, or object. A **document** is a file containing a serialized object, or an array of objects. A **document type** is a class of files that share a common structure and semantics. A **document schema** is a formal description of the grammar of a document type. A **base URI** is a context-dependent URI used to resolve relative references. An **identifier** is a URI that designates a single document or single object within a document. A **vocabulary** is the set of symbolic field names and enumerated symbols defined by a document schema, where each term maps to absolute URI. ## Syntax Conforming Salad documents are serialized and loaded using YAML syntax and UTF-8 text encoding. Salad documents are written using the JSON-compatible subset of YAML. Features of YAML such as headers and type tags that are not found in the standard JSON data model must not be used in conforming Salad documents. It is a fatal error if the document is not valid YAML. A Salad document must consist only of either a single root object or an array of objects. ## Document context ### Implied context The implicit context consists of the vocabulary defined by the schema and the base URI. By default, the base URI must be the URI that was used to load the document. It may be overridden by an explicit context. ### Explicit context If a document consists of a root object, this object may contain the fields `$base`, `$namespaces`, `$schemas`, and `$graph`: * `$base`: Must be a string. Set the base URI for the document used to resolve relative references. * `$namespaces`: Must be an object with strings as values. The keys of the object are namespace prefixes used in the document; the values of the object are the prefix expansions. * `$schemas`: Must be an array of strings. This field may list URI references to documents in RDF-XML format which will be queried for RDF schema data. The subjects and predicates described by the RDF schema may provide additional semantic context for the document, and may be used for validation of prefixed extension fields found in the document. Other directives beginning with `$` must be ignored. ## Document graph If a document consists of a single root object, this object may contain the field `$graph`. This field must be an array of objects. If present, this field holds the primary content of the document. A document that consists of array of objects at the root is an implicit graph. ## Document metadata If a document consists of a single root object, metadata about the document, such as authorship, may be declared in the root object. ## Document schema Document preprocessing, link validation and schema validation require a document schema. A schema may consist of: * At least one record definition object which defines valid fields that make up a record type. Record field definitions include the valid types that may be assigned to each field and annotations to indicate fields that represent identifiers and links, described below in "Semantic Annotations". * Any number of enumerated type objects which define a set of finite set of symbols that are valid value of the type. * Any number of documentation objects which allow in-line documentation of the schema. The schema for defining a salad schema (the metaschema) is described in detail in "Schema validation". ### Record field annotations In a document schema, record field definitions may include the field `jsonldPredicate`, which may be either a string or object. Implementations must use the following document preprocessing of fields by the following rules: * If the value of `jsonldPredicate` is `@id`, the field is an identifier field. * If the value of `jsonldPredicate` is an object, and contains that object contains the field `_type` with the value `@id`, the field is a link field. * If the value of `jsonldPredicate` is an object, and contains that object contains the field `_type` with the value `@vocab`, the field is a vocabulary field, which is a subtype of link field. ## Document traversal To perform document document preprocessing, link validation and schema validation, the document must be traversed starting from the fields or array items of the root object or array and recursively visiting each child item which contains an object or arrays. # Document preprocessing After processing the explicit context (if any), document preprocessing begins. Starting from the document root, object fields values or array items which contain objects or arrays are recursively traversed depth-first. For each visited object, field names, identifier fields, link fields, vocabulary fields, and `$import` and `$include` directives must be processed as described in this section. The order of traversal of child nodes within a parent node is undefined. cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/metaschema.yml0000644000175000017500000002565213036412052032441 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/salad#" $namespaces: sld: "https://w3id.org/cwl/salad#" dct: "http://purl.org/dc/terms/" rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" rdfs: "http://www.w3.org/2000/01/rdf-schema#" xsd: "http://www.w3.org/2001/XMLSchema#" $graph: - name: "Semantic_Annotations_for_Linked_Avro_Data" type: documentation doc: - $include: salad.md - $import: field_name.yml - $import: ident_res.yml - $import: link_res.yml - $import: vocab_res.yml - $include: import_include.md - name: "Link_Validation" type: documentation doc: | # Link validation Once a document has been preprocessed, an implementation may validate links. The link validation traversal may visit fields which the schema designates as link fields and check that each URI references an existing object in the current document, an imported document, file system, or network resource. Failure to validate links may be a fatal error. Link validation behavior for individual fields may be modified by `identity` and `noLinkCheck` in the `jsonldPredicate` section of the field schema. - name: "Schema_validation" type: documentation doc: "" # - name: "JSON_LD_Context" # type: documentation # doc: | # # Generating JSON-LD Context # How to generate the json-ld context... - name: PrimitiveType type: enum symbols: - "sld:null" - "xsd:boolean" - "xsd:int" - "xsd:long" - "xsd:float" - "xsd:double" - "xsd:string" doc: - | Salad data types are based on Avro schema declarations. Refer to the [Avro schema declaration documentation](https://avro.apache.org/docs/current/spec.html#schemas) for detailed information. - "null: no value" - "boolean: a binary value" - "int: 32-bit signed integer" - "long: 64-bit signed integer" - "float: single precision (32-bit) IEEE 754 floating-point number" - "double: double precision (64-bit) IEEE 754 floating-point number" - "string: Unicode character sequence" - name: "Any" type: enum symbols: ["#Any"] doc: | The **Any** type validates for any non-null value. - name: JsonldPredicate type: record doc: | Attached to a record field to define how the parent record field is handled for URI resolution and JSON-LD context generation. fields: - name: _id type: ["null", string] jsonldPredicate: _id: sld:_id _type: "@id" identity: true doc: | The predicate URI that this field corresponds to. Corresponds to JSON-LD `@id` directive. - name: _type type: ["null", string] doc: | The context type hint, corresponds to JSON-LD `@type` directive. * If the value of this field is `@id` and `identity` is false or unspecified, the parent field must be resolved using the link resolution rules. If `identity` is true, the parent field must be resolved using the identifier expansion rules. * If the value of this field is `@vocab`, the parent field must be resolved using the vocabulary resolution rules. - name: _container type: ["null", string] doc: | Structure hint, corresponds to JSON-LD `@container` directive. - name: identity type: ["null", boolean] doc: | If true and `_type` is `@id` this indicates that the parent field must be resolved according to identity resolution rules instead of link resolution rules. In addition, the field value is considered an assertion that the linked value exists; absence of an object in the loaded document with the URI is not an error. - name: noLinkCheck type: ["null", boolean] doc: | If true, this indicates that link validation traversal must stop at this field. This field (it is is a URI) or any fields under it (if it is an object or array) are not subject to link checking. - name: SpecializeDef type: record fields: - name: specializeFrom type: string doc: "The data type to be replaced" jsonldPredicate: _id: "sld:specializeFrom" _type: "@id" - name: specializeTo type: string doc: "The new data type to replace with" jsonldPredicate: _id: "sld:specializeTo" _type: "@id" - name: NamedType type: record abstract: true fields: - name: name type: string jsonldPredicate: "@id" doc: "The identifier for this type" - name: DocType type: record abstract: true fields: - name: doc type: - "null" - string - type: array items: string doc: "A documentation string for this type, or an array of strings which should be concatenated." jsonldPredicate: "sld:doc" - name: docParent type: ["null", string] doc: | Hint to indicate that during documentation generation, documentation for this type should appear in a subsection under `docParent`. jsonldPredicate: _id: "sld:docParent" _type: "@id" - name: docChild type: - "null" - string - type: array items: string doc: | Hint to indicate that during documentation generation, documentation for `docChild` should appear in a subsection under this type. jsonldPredicate: _id: "sld:docChild" _type: "@id" - name: docAfter type: ["null", string] doc: | Hint to indicate that during documentation generation, documentation for this type should appear after the `docAfter` section at the same level. jsonldPredicate: _id: "sld:docAfter" _type: "@id" - name: SchemaDefinedType type: record extends: "#DocType" doc: | Abstract base for schema-defined types. abstract: true fields: - name: jsonldPredicate type: - "null" - string - "#JsonldPredicate" doc: | Annotate this type with linked data context. jsonldPredicate: "sld:jsonldPredicate" - name: documentRoot type: ["null", boolean] doc: | If true, indicates that the type is a valid at the document root. At least one type in a schema must be tagged with `documentRoot: true`. - name: RecordField type: record doc: "A field of a record." fields: - name: name type: string jsonldPredicate: "@id" doc: | The name of the field - name: doc type: ["null", string] doc: | A documentation string for this field jsonldPredicate: "sld:doc" - name: type type: - "#PrimitiveType" - "#RecordSchema" - "#EnumSchema" - "#ArraySchema" - string - type: array items: - "#PrimitiveType" - "#RecordSchema" - "#EnumSchema" - "#ArraySchema" - string jsonldPredicate: _id: "sld:type" _type: "@vocab" doc: | The field type - name: SaladRecordField type: record extends: "#RecordField" doc: "A field of a record." fields: - name: jsonldPredicate type: - "null" - string - "#JsonldPredicate" doc: | Annotate this type with linked data context. jsonldPredicate: "sld:jsonldPredicate" - name: RecordSchema type: record fields: - name: type doc: "Must be `record`" type: name: Record_symbol type: enum symbols: - "sld:record" jsonldPredicate: _id: "sld:type" _type: "@vocab" - name: "fields" type: - "null" - type: "array" items: "#RecordField" jsonldPredicate: "sld:fields" doc: "Defines the fields of the record." - name: SaladRecordSchema type: record extends: ["#NamedType", "#RecordSchema", "#SchemaDefinedType"] documentRoot: true specialize: - specializeFrom: "#RecordField" specializeTo: "#SaladRecordField" fields: - name: abstract type: ["null", boolean] doc: | If true, this record is abstract and may be used as a base for other records, but is not valid on its own. - name: extends type: - "null" - string - type: array items: string jsonldPredicate: _id: "sld:extends" _type: "@id" doc: | Indicates that this record inherits fields from one or more base records. - name: specialize type: - "null" - "#SpecializeDef" - type: array items: "#SpecializeDef" doc: | Only applies if `extends` is declared. Apply type specialization using the base record as a template. For each field inherited from the base record, replace any instance of the type `specializeFrom` with `specializeTo`. - name: EnumSchema type: record doc: | Define an enumerated type. fields: - name: type doc: "Must be `enum`" type: name: Enum_symbol type: enum symbols: - "sld:enum" jsonldPredicate: _id: "sld:type" _type: "@vocab" - name: "symbols" type: - type: "array" items: "string" jsonldPredicate: _id: "sld:symbols" _type: "@id" identity: true doc: "Defines the set of valid symbols." - name: SaladEnumSchema type: record extends: ["#EnumSchema", "#SchemaDefinedType"] documentRoot: true doc: | Define an enumerated type. fields: - name: extends type: - "null" - string - type: array items: string jsonldPredicate: _id: "sld:extends" _type: "@id" doc: | Indicates that this enum inherits symbols from a base enum. - name: ArraySchema type: record fields: - name: type doc: "Must be `array`" type: name: Array_symbol type: enum symbols: - "sld:array" jsonldPredicate: _id: "sld:type" _type: "@vocab" - name: items type: - "#PrimitiveType" - "#RecordSchema" - "#EnumSchema" - "#ArraySchema" - string - type: array items: - "#PrimitiveType" - "#RecordSchema" - "#EnumSchema" - "#ArraySchema" - string jsonldPredicate: _id: "sld:items" _type: "@vocab" doc: "Defines the type of the array elements." - name: Documentation type: record extends: ["#NamedType", "#DocType"] documentRoot: true doc: | A documentation section. This type exists to facilitate self-documenting schemas but has no role in formal validation. fields: - name: type doc: "Must be `documentation`" type: name: Documentation_symbol type: enum symbols: - "sld:documentation" jsonldPredicate: _id: "sld:type" _type: "@vocab" cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/link_res.yml0000644000175000017500000000341613036412052032132 0ustar michaelmichael00000000000000- | ## Link resolution The schema may designate one or more fields as link fields reference other objects. Processing must resolve links to either absolute URIs using the following rules: * If a reference URI is prefixed with `#` it is a relative fragment identifier. It is resolved relative to the base URI by setting or replacing the fragment portion of the base URI. * If a reference URI does not contain a scheme and is not prefixed with `#` it is a path relative reference. If the reference URI contains `#` in any position other than the first character, the reference URI must be divided into a path portion and a fragment portion split on the first instance of `#`. The path portion is resolved relative to the base URI by the following rule: if the path portion of the base URI ends in a slash `/`, append the path portion of the reference URI to the path portion of the base URI. If the path portion of the base URI does not end in a slash, replace the final path segment with the path portion of the reference URI. Replace the fragment portion of the base URI with the fragment portion of the reference URI. * If a reference URI begins with a namespace prefix declared in `$namespaces` followed by a colon `:`, the prefix and colon must be replaced by the namespace declared in `$namespaces`. * If a reference URI is an absolute URI consisting of a scheme and path, no processing occurs. Link resolution must not affect the base URI used to resolve identifiers and other links. ### Link resolution example Given the following schema: ``` - $include: link_res_schema.yml - | ``` Process the following example: ``` - $include: link_res_src.yml - | ``` This becomes: ``` - $include: link_res_proc.yml - | ``` cwltool-1.0.20170114120503/cwltool/schemas/draft-3/salad/schema_salad/metaschema/ident_res_proc.yml0000644000175000017500000000056213036412052033322 0ustar michaelmichael00000000000000{ "id": "http://example.com/base", "form": { "id": "http://example.com/base#one", "things": [ { "id": "http://example.com/base#one/two" }, { "id": "http://example.com/base#three" }, { "id": "http://example.com/four#five", }, { "id": "http://example.com/acid#six", } ] } } cwltool-1.0.20170114120503/cwltool/schemas/draft-3/Process.yml0000644000175000017500000004010713036412052024141 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" sld: "https://w3id.org/cwl/salad#" rdfs: "http://www.w3.org/2000/01/rdf-schema#" $graph: - name: "Common Workflow Language, Draft 3" type: documentation doc: {$include: concepts.md} - $import: "salad/schema_salad/metaschema/metaschema.yml" - name: BaseTypesDoc type: documentation doc: | ## Base types docChild: - "#CWLType" - "#Process" - type: enum name: CWLVersions doc: "Version symbols for published CWL document versions." symbols: - cwl:draft-2 - cwl:draft-3.dev1 - cwl:draft-3.dev2 - cwl:draft-3.dev3 - cwl:draft-3.dev4 - cwl:draft-3.dev5 - cwl:draft-3 - name: CWLType type: enum extends: "sld:PrimitiveType" symbols: - cwl:File doc: - "Extends primitive types with the concept of a file as a first class type." - "File: A File object" - name: File type: record docParent: "#CWLType" doc: | Represents a file (or group of files if `secondaryFiles` is specified) that must be accessible by tools using standard POSIX file system call API such as open(2) and read(2). fields: - name: class type: type: enum name: File_class symbols: - cwl:File jsonldPredicate: "_id": "@type" "_type": "@vocab" doc: Must be `File` to indicate this object describes a file. - name: path type: string doc: The path to the file. jsonldPredicate: "_id": "cwl:path" "_type": "@id" - name: checksum type: ["null", string] doc: | Optional hash code for validating file integrity. Currently must be in the form "sha1$ + hexidecimal string" using the SHA-1 algorithm. - name: size type: ["null", long] doc: Optional file size. - name: "secondaryFiles" type: - "null" - type: array items: "#File" jsonldPredicate: "cwl:secondaryFiles" doc: | A list of additional files that are associated with the primary file and must be transferred alongside the primary file. Examples include indexes of the primary file, or external references which must be included when loading primary document. A file object listed in `secondaryFiles` may itself include `secondaryFiles` for which the same rules apply. - name: format type: ["null", string] jsonldPredicate: _id: cwl:format _type: "@id" identity: true doc: | The format of the file. This must be a URI of a concept node that represents the file format, preferrably defined within an ontology. If no ontology is available, file formats may be tested by exact match. Reasoning about format compatability must be done by checking that an input file format is the same, `owl:equivalentClass` or `rdfs:subClassOf` the format required by the input parameter. `owl:equivalentClass` is transitive with `rdfs:subClassOf`, e.g. if ` owl:equivalentClass ` and ` owl:subclassOf ` then infer ` owl:subclassOf `. File format ontologies may be provided in the "$schema" metadata at the root of the document. If no ontologies are specified in `$schema`, the runtime may perform exact file format matches. - name: SchemaBase type: record abstract: true fields: - name: secondaryFiles type: - "null" - "string" - "#Expression" - type: "array" items: ["string", "#Expression"] jsonldPredicate: "cwl:secondaryFiles" doc: | Only valid when `type: File` or is an array of `items: File`. Describes files that must be included alongside the primary file(s). If the value is an expression, the value of `self` in the expression must be the primary input or output File to which this binding applies. If the value is a string, it specifies that the following pattern should be applied to the primary file: 1. If string begins with one or more caret `^` characters, for each caret, remove the last file extension from the path (the last period `.` and all following characters). If there are no file extensions, the path is unchanged. 2. Append the remainder of the string to the end of the file path. - name: format type: - "null" - string - type: array items: string - "#Expression" jsonldPredicate: _id: cwl:format _type: "@id" identity: true doc: | Only valid when `type: File` or is an array of `items: File`. For input parameters, this must be one or more URIs of a concept nodes that represents file formats which are allowed as input to this parameter, preferrably defined within an ontology. If no ontology is available, file formats may be tested by exact match. For output parameters, this is the file format that will be assigned to the output parameter. - name: streamable type: ["null", "boolean"] doc: | Only valid when `type: File` or is an array of `items: File`. A value of `true` indicates that the file is read or written sequentially without seeking. An implementation may use this flag to indicate whether it is valid to stream file contents using a named pipe. Default: `false`. - name: Parameter type: record extends: "#SchemaBase" abstract: true doc: | Define an input or output parameter to a process. fields: - name: type type: - "null" - "#CWLType" - "sld:RecordSchema" - "sld:EnumSchema" - "sld:ArraySchema" - string - type: array items: - "#CWLType" - "sld:RecordSchema" - "sld:EnumSchema" - "sld:ArraySchema" - string jsonldPredicate: "_id": "sld:type" "_type": "@vocab" doc: | Specify valid types of data that may be assigned to this parameter. - name: label type: - "null" - string jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this parameter object." - name: description type: - "null" - string jsonldPredicate: "rdfs:comment" doc: "A long, human-readable description of this parameter object." - type: enum name: Expression doc: | Not a real type. Indicates that a field must allow runtime parameter references. If [InlineJavascriptRequirement](#InlineJavascriptRequirement) is declared and supported by the platform, the field must also allow Javascript expressions. symbols: - cwl:ExpressionPlaceholder - name: InputBinding type: record abstract: true fields: - name: loadContents type: - "null" - boolean jsonldPredicate: "cwl:loadContents" doc: | Only valid when `type: File` or is an array of `items: File`. Read up to the first 64 KiB of text from the file and place it in the "contents" field of the file object for use by expressions. - name: OutputBinding type: record abstract: true - name: InputSchema extends: "#SchemaBase" type: record abstract: true - name: OutputSchema extends: "#SchemaBase" type: record abstract: true - name: InputRecordField type: record extends: "sld:RecordField" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: "#InputRecordSchema" - specializeFrom: "sld:EnumSchema" specializeTo: "#InputEnumSchema" - specializeFrom: "sld:ArraySchema" specializeTo: "#InputArraySchema" fields: - name: inputBinding type: [ "null", "#InputBinding" ] jsonldPredicate: "cwl:inputBinding" - name: InputRecordSchema type: record extends: ["sld:RecordSchema", "#InputSchema"] specialize: - specializeFrom: "sld:RecordField" specializeTo: "#InputRecordField" - name: InputEnumSchema type: record extends: ["sld:EnumSchema", "#InputSchema"] fields: - name: inputBinding type: [ "null", "#InputBinding" ] jsonldPredicate: "cwl:inputBinding" - name: InputArraySchema type: record extends: ["sld:ArraySchema", "#InputSchema"] specialize: - specializeFrom: "sld:RecordSchema" specializeTo: "#InputRecordSchema" - specializeFrom: "sld:EnumSchema" specializeTo: "#InputEnumSchema" - specializeFrom: "sld:ArraySchema" specializeTo: "#InputArraySchema" fields: - name: inputBinding type: [ "null", "#InputBinding" ] jsonldPredicate: "cwl:inputBinding" - name: OutputRecordField type: record extends: "sld:RecordField" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: "#OutputRecordSchema" - specializeFrom: "sld:EnumSchema" specializeTo: "#OutputEnumSchema" - specializeFrom: "sld:ArraySchema" specializeTo: "#OutputArraySchema" fields: - name: outputBinding type: [ "null", "#OutputBinding" ] jsonldPredicate: "cwl:outputBinding" - name: OutputRecordSchema type: record extends: ["sld:RecordSchema", "#OutputSchema"] docParent: "#OutputParameter" specialize: - specializeFrom: "sld:RecordField" specializeTo: "#OutputRecordField" - name: OutputEnumSchema type: record extends: ["sld:EnumSchema", "#OutputSchema"] docParent: "#OutputParameter" fields: - name: outputBinding type: [ "null", "#OutputBinding" ] jsonldPredicate: "cwl:outputBinding" - name: OutputArraySchema type: record extends: ["sld:ArraySchema", "#OutputSchema"] docParent: "#OutputParameter" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: "#OutputRecordSchema" - specializeFrom: "sld:EnumSchema" specializeTo: "#OutputEnumSchema" - specializeFrom: "sld:ArraySchema" specializeTo: "#OutputArraySchema" fields: - name: outputBinding type: [ "null", "#OutputBinding" ] jsonldPredicate: "cwl:outputBinding" - name: InputParameter type: record extends: "#Parameter" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: "#InputRecordSchema" - specializeFrom: "sld:EnumSchema" specializeTo: "#InputEnumSchema" - specializeFrom: "sld:ArraySchema" specializeTo: "#InputArraySchema" fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this parameter object." - name: "inputBinding" type: [ "null", "#InputBinding" ] jsonldPredicate: "cwl:inputBinding" doc: | Describes how to handle the inputs of a process and convert them into a concrete form for execution, such as command line parameters. - name: default type: ["null", "Any"] jsonldPredicate: "cwl:default" doc: | The default value for this parameter if not provided in the input object. - name: OutputParameter type: record extends: "#Parameter" specialize: - specializeFrom: "sld:RecordSchema" specializeTo: "#OutputRecordSchema" - specializeFrom: "sld:EnumSchema" specializeTo: "#OutputEnumSchema" - specializeFrom: "sld:ArraySchema" specializeTo: "#OutputArraySchema" fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this parameter object." - name: "outputBinding" type: [ "null", "#OutputBinding" ] jsonldPredicate: "cwl:outputBinding" doc: | Describes how to handle the outputs of a process. - type: record name: ProcessRequirement abstract: true doc: | A process requirement declares a prerequisite that may or must be fulfilled before executing a process. See [`Process.hints`](#process) and [`Process.requirements`](#process). Process requirements are the primary mechanism for specifying extensions to the CWL core specification. fields: - name: "class" type: "string" doc: "The specific requirement type." jsonldPredicate: "_id": "@type" "_type": "@vocab" - type: record name: Process abstract: true doc: | The base executable type in CWL is the `Process` object defined by the document. Note that the `Process` object is abstract and cannot be directly executed. fields: - name: id type: ["null", string] jsonldPredicate: "@id" doc: "The unique identifier for this process object." - name: inputs type: type: array items: "#InputParameter" jsonldPredicate: "cwl:inputs" doc: | Defines the input parameters of the process. The process is ready to run when all required input parameters are associated with concrete values. Input parameters include a schema for each parameter which is used to validate the input object. It may also be used to build a user interface for constructing the input object. - name: outputs type: type: array items: "#OutputParameter" jsonldPredicate: "cwl:outputs" doc: | Defines the parameters representing the output of the process. May be used to generate and/or validate the output object. - name: requirements type: - "null" - type: array items: "#ProcessRequirement" jsonldPredicate: "cwl:requirements" doc: | Declares requirements that apply to either the runtime environment or the workflow engine that must be met in order to execute this process. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. - name: hints type: - "null" - type: array items: Any jsonldPredicate: "cwl:hints" doc: | Declares hints applying to either the runtime environment or the workflow engine that may be helpful in executing this process. It is not an error if an implementation cannot satisfy all hints, however the implementation may report a warning. jsonldPredicate: _id: cwl:hints noLinkCheck: true - name: label type: - "null" - string jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: description type: - "null" - string jsonldPredicate: "rdfs:comment" doc: "A long, human-readable description of this process object." - name: cwlVersion type: - "null" - "#CWLVersions" doc: "CWL document version" jsonldPredicate: "_id": "cwl:cwlVersion" "_type": "@vocab" - name: InlineJavascriptRequirement type: record extends: "#ProcessRequirement" doc: | Indicates that the workflow platform must support inline Javascript expressions. If this requirement is not present, the workflow platform must not perform expression interpolatation. fields: - name: expressionLib type: - "null" - type: array items: string doc: | Additional code fragments that will also be inserted before executing the expression code. Allows for function definitions that may be called from CWL expressions. - name: SchemaDefRequirement type: record extends: "#ProcessRequirement" doc: | This field consists of an array of type definitions which must be used when interpreting the `inputs` and `outputs` fields. When a `type` field contain a URI, the implementation must check if the type is defined in `schemaDefs` and use that definition. If the type is not found in `schemaDefs`, it is an error. The entries in `schemaDefs` must be processed in the order listed such that later schema definitions may refer to earlier schema definitions. fields: - name: types type: type: array items: "#InputSchema" doc: The list of type definitions. cwltool-1.0.20170114120503/cwltool/schemas/draft-3/concepts.md0000644000175000017500000004240113036412052024137 0ustar michaelmichael00000000000000## References to Other Specifications **Javascript Object Notation (JSON)**: http://json.org **JSON Linked Data (JSON-LD)**: http://json-ld.org **YAML**: http://yaml.org **Avro**: https://avro.apache.org/docs/current/spec.html **Uniform Resource Identifier (URI) Generic Syntax**: https://tools.ietf.org/html/rfc3986) **Portable Operating System Interface (POSIX.1-2008)**: http://pubs.opengroup.org/onlinepubs/9699919799/ **Resource Description Framework (RDF)**: http://www.w3.org/RDF/ ## Scope This document describes CWL syntax, execution, and object model. It is not intended to document a CWL specific implementation, however it may serve as a reference for the behavior of conforming implementations. ## Terminology The terminology used to describe CWL documents is defined in the Concepts section of the specification. The terms defined in the following list are used in building those definitions and in describing the actions of an CWL implementation: **may**: Conforming CWL documents and CWL implementations are permitted but not required to behave as described. **must**: Conforming CWL documents and CWL implementations are required to behave as described; otherwise they are in error. **error**: A violation of the rules of this specification; results are undefined. Conforming implementations may detect and report an error and may recover from it. **fatal error**: A violation of the rules of this specification; results are undefined. Conforming implementations must not continue to execute the current process and may report an error. **at user option**: Conforming software may or must (depending on the modal verb in the sentence) behave as described; if it does, it must provide users a means to enable or disable the behavior described. **deprecated**: Conforming software may implement a behavior for backwards compatibility. Portable CWL documents should not rely on deprecated behavior. Behavior marked as deprecated may be removed entirely from future revisions of the CWL specification. # Data model ## Data concepts An **object** is a data structure equivalent to the "object" type in JSON, consisting of a unordered set of name/value pairs (referred to here as **fields**) and where the name is a string and the value is a string, number, boolean, array, or object. A **document** is a file containing a serialized object, or an array of objects. A **process** is a basic unit of computation which accepts input data, performs some computation, and produces output data. An **input object** is an object describing the inputs to a invocation of process. An **output object** is an object describing the output of an invocation of a process. An **input schema** describes the valid format (required fields, data types) for an input object. An **output schema** describes the valid format for a output object. **Metadata** is information about workflows, tools, or input items that is not used directly in the computation. ## Syntax CWL documents must consist of an object or array of objects represented using JSON or YAML syntax. Upon loading, a CWL implementation must apply the preprocessing steps described in the [Semantic Annotations for Linked Avro Data (SALAD) Specification](SchemaSalad.html). A implementation may formally validate the structure of a CWL document using SALAD schemas located at https://github.com/common-workflow-language/common-workflow-language/tree/master/draft-3 ## Identifiers If an object contains an `id` field, that is used to uniquely identify the object in that document. The value of the `id` field must be unique over the entire document. Identifiers may be resolved relative to other the document base and/or other identifiers following the rules are described in the [Schema Salad specification](SchemaSalad.html#Identifier_resolution). An implementation may choose to only honor references to object types for which the `id` field is explicitly listed in this specification. ## Document preprocessing An implementation must resolve [$import](SchemaSalad.html#Import) and [$include](SchemaSalad.html#Import) directives as described in the [Schema Salad specification](SchemaSalad.html). ## Extensions and Metadata Input metadata (for example, a lab sample identifier) may be represented within a tool or workflow using input parameters which are explicitly propagated to output. Future versions of this specification may define additional facilities for working with input/output metadata. Implementation extensions not required for correct execution (for example, fields related to GUI presentation) and metadata about the tool or workflow itself (for example, authorship for use in citations) may be provided as additional fields on any object. Such extensions fields must use a namespace prefix listed in the `$namespaces` section of the document as described in the [Schema Salad specification](SchemaSalad.html#Explicit_context). Implementation extensions which modify execution semantics must be [listed in the `requirements` field](#Requirements_and_hints). # Execution model ## Execution concepts A **parameter** is a named symbolic input or output of process, with an associated datatype or schema. During execution, values are assigned to parameters to make the input object or output object used for concrete process invocation. A **command line tool** is a process characterized by the execution of a standalone, non-interactive program which is invoked on some input, produces output, and then terminates. A **workflow** is a process characterized by multiple subprocess steps, where step outputs are connected to the inputs of other downstream steps to form a directed graph, and independent steps may run concurrently. A **runtime environment** is the actual hardware and software environment when executing a command line tool. It includes, but is not limited to, the hardware architecture, hardware resources, operating system, software runtime (if applicable, such as the Python interpreter or the JVM), libraries, modules, packages, utilities, and data files required to run the tool. A **workflow platform** is a specific hardware and software implementation capable of interpreting CWL documents and executing the processes specified by the document. The responsibilities of the workflow platform may include scheduling process invocation, setting up the necessary runtime environment, making input data available, invoking the tool process, and collecting output. A workflow platform may choose to only implement the Command Line Tool Description part of the CWL specification. It is intended that the workflow platform has broad leeway outside of this specification to optimize use of computing resources and enforce policies not covered by this specification. Some areas that are currently out of scope for CWL specification but may be handled by a specific workflow platform include: * Data security and permissions. * Scheduling tool invocations on remote cluster or cloud compute nodes. * Using virtual machines or operating system containers to manage the runtime (except as described in [DockerRequirement](CommandLineTool.html#DockerRequirement)). * Using remote or distributed file systems to manage input and output files. * Transforming file paths. * Determining if a process has previously been executed, skipping it and reusing previous results. * Pausing, resuming or checkpointing processes or workflows. Conforming CWL processes must not assume anything about the runtime environment or workflow platform unless explicitly declared though the use of [process requirements](#Requirements_and_hints). ## Generic execution process The generic execution sequence of a CWL process (including workflows and command line line tools) is as follows. 1. Load, process and validate a CWL document, yielding a process object. 2. Load input object. 3. Validate the input object against the `inputs` schema for the process. 4. Validate that process requirements are met. 5. Perform any further setup required by the specific process type. 6. Execute the process. 7. Capture results of process execution into the output object. 8. Validate the output object against the `outputs` schema for the process. 9. Report the output object to the process caller. ## Requirements and hints A **process requirement** modifies the semantics or runtime environment of a process. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. A **hint** is similar to a requirement, however it is not an error if an implementation cannot satisfy all hints. The implementation may report a warning if a hint cannot be satisfied. Requirements are inherited. A requirement specified in a Workflow applies to all workflow steps; a requirement specified on a workflow step will apply to the process implementation. If the same process requirement appears at different levels of the workflow, the most specific instance of the requirement is used, that is, an entry in `requirements` on a process implementation such as CommandLineTool will take precedence over an entry in `requirements` specified in a workflow step, and an entry in `requirements` on a workflow step takes precedence over the workflow. Entries in `hints` are resolved the same way. Requirements override hints. If a process implementation provides a process requirement in `hints` which is also provided in `requirements` by an enclosing workflow or workflow step, the enclosing `requirements` takes precedence. ## Parameter references Parameter references are denoted by the syntax `$(...)` and may be used in any field permitting the pseudo-type `Expression`, as specified by this document. Conforming implementations must support parameter references. Parameter references use the following subset of [Javascript/ECMAScript 5.1](http://www.ecma-international.org/ecma-262/5.1/) syntax. In the following BNF grammar, character classes and grammar rules are denoted in '{}', '-' denotes exclusion from a character class, '(())' denotes grouping, '|' denotes alternates, trailing '*' denotes zero or more repeats, '+' denote one or more repeats, all other characters are literal values.

symbol:: {Unicode alphanumeric}+
singleq:: [' (( {character - '} | \' ))* ']
doubleq:: [" (( {character - "} | \" ))* "]
index:: [ {decimal digit}+ ]
segment:: . {symbol} | {singleq} | {doubleq} | {index}
parameter::$( {symbol} {segment}*)

Use the following algorithm to resolve a parameter reference: 1. Match the leading symbol as key 2. Look up the key in the parameter context (described below) to get the current value. It is an error if the key is not found in the parameter context. 3. If there are no subsequent segments, terminate and return current value 4. Else, match the next segment 5. Extract the symbol, string, or index from the segment as key 6. Look up the key in current value and assign as new current value. If the key is a symbol or string, the current value must be an object. If the key is an index, the current value must be an array or string. It is an error if the key does not match the required type, or the key is not found or out of range. 7. Repeat steps 3-6 The root namespace is the parameter context. The following parameters must be provided: * `inputs`: The input object to the current Process. * `self`: A context-specific value. The contextual values for 'self' are documented for specific fields elsewhere in this specification. If a contextual value of 'self' is not documented for a field, it must be 'null'. * `runtime`: An object containing configuration details. Specific to the process type. An implementation may provide may provide opaque strings for any or all fields of `runtime`. These must be filled in by the platform after processing the Tool but before actual execution. Parameter references and expressions may only use the literal string value of the field and must not perform computation on the contents. If the value of a field has no leading or trailing non-whitespace characters around a parameter reference, the effective value of the field becomes the value of the referenced parameter, preserving the return type. If the value of a field has non-whitespace leading or trailing characters around an parameter reference, it is subject to string interpolation. The effective value of the field is a string containing the leading characters; followed by the string value of the parameter reference; followed by the trailing characters. The string value of the parameter reference is its textual JSON representation with the following rules: * Leading and trailing quotes are stripped from strings * Objects entries are sorted by key Multiple parameter references may appear in a single field. This case is must be treated as a string interpolation. After interpolating the first parameter reference, interpolation must be recursively applied to the trailing characters to yield the final string value. ## Expressions An expression is a fragment of [Javascript/ECMAScript 5.1](http://www.ecma-international.org/ecma-262/5.1/) code which is evaluated by the workflow platform to affect the inputs, outputs, or behavior of a process. In the generic execution sequence, expressions may be evaluated during step 5 (process setup), step 6 (execute process), and/or step 7 (capture output). Expressions are distinct from regular processes in that they are intended to modify the behavior of the workflow itself rather than perform the primary work of the workflow. To declare the use of expressions, the document must include the process requirement `InlineJavascriptRequirement`. Expressions may be used in any field permitting the pseudo-type `Expression`, as specified by this document. Expressions are denoted by the syntax `$(...)` or `${...}`. A code fragment wrapped in the `$(...)` syntax must be evaluated as a [ECMAScript expression](http://www.ecma-international.org/ecma-262/5.1/#sec-11). A code fragment wrapped in the `${...}` syntax must be evaluated as a [EMACScript function body](http://www.ecma-international.org/ecma-262/5.1/#sec-13) for an anonymous, zero-argument function. Expressions must return a valid JSON data type: one of null, string, number, boolean, array, object. Implementations must permit any syntactically valid Javascript and account for nesting of parenthesis or braces and that strings that may contain parenthesis or braces when scanning for expressions. The runtime must include any code defined in the ["expressionLib" field of InlineJavascriptRequirement](#InlineJavascriptRequirement) prior to executing the actual expression. Before executing the expression, the runtime must initialize as global variables the fields of the parameter context described above. The effective value of the field after expression evaluation follows the same rules as parameter references discussed above. Multiple expressions may appear in a single field. Expressions must be evaluated in an isolated context (a "sandbox") which permits no side effects to leak outside the context. Expressions also must be evaluated in [Javascript strict mode](http://www.ecma-international.org/ecma-262/5.1/#sec-4.2.2). The order in which expressions are evaluated is undefined except where otherwise noted in this document. An implementation may choose to implement parameter references by evaluating as a Javascript expression. The results of evaluating parameter references must be identical whether implemented by Javascript evaluation or some other means. Implementations may apply other limits, such as process isolation, timeouts, and operating system containers/jails to minimize the security risks associated with running untrusted code embedded in a CWL document. ## Success and failure A completed process must result in one of `success`, `temporaryFailure` or `permanentFailure` states. An implementation may choose to retry a process execution which resulted in `temporaryFailure`. An implementation may choose to either continue running other steps of a workflow, or terminate immediately upon `permanentFailure`. * If any step of a workflow execution results in `permanentFailure`, then the workflow status is `permanentFailure`. * If one or more steps result in `temporaryFailure` and all other steps complete `success` or are not executed, then the workflow status is `temporaryFailure`. * If all workflow steps are executed and complete with `success`, then the workflow status is `success`. ## Executing CWL documents as scripts By convention, a CWL document may begin with `#!/usr/bin/env cwl-runner` and be marked as executable (the POSIX "+x" permission bits) to enable it to be executed directly. A workflow platform may support this mode of operation; if so, it must provide `cwl-runner` as an alias for the platform's CWL implementation. A CWL input object document may similarly begin with `#!/usr/bin/env cwl-runner` and be marked as executable. In this case, the input object must include the field `cwl:tool` supplying a URI to the default CWL document that should be executed using the fields of the input object as input parameters. cwltool-1.0.20170114120503/cwltool/schemas/draft-3/intro.md0000644000175000017500000000156713036412052023464 0ustar michaelmichael00000000000000# Status of This Document This document is the product of the [Common Workflow Language working group](https://groups.google.com/forum/#!forum/common-workflow-language). The latest version of this document is available in the "draft-3" directory at https://github.com/common-workflow-language/common-workflow-language The products of the CWL working group (including this document) are made available under the terms of the Apache License, version 2.0. # Introduction The Common Workflow Language (CWL) working group is an informal, multi-vendor working group consisting of various organizations and individuals that have an interest in portability of data analysis workflows. The goal is to create specifications like this one that enable data scientists to describe analysis tools and workflows that are powerful, easy to use, portable, and support reproducibility. cwltool-1.0.20170114120503/cwltool/schemas/draft-3/UserGuide.yml0000644000175000017500000006715213036412052024430 0ustar michaelmichael00000000000000- name: userguide type: documentation doc: - $include: userguide-intro.md - | # Wrapping Command Line Tools - | ## First example The simplest "hello world" program. This accepts one input parameter, writes a message to the terminal or job log, and produces no permanent output. CWL documents are written in [JSON](http://json.org) or [YAML](http://yaml.org), or a mix of the two. *1st-tool.cwl* ``` - $include: examples/1st-tool.cwl - | ``` Use a YAML object in a separate file to describe the input of a run: *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwl-runner 1st-tool.cwl echo-job.yml [job 140199012414352] $ echo 'Hello world!' Hello world! Final process status is success ``` What's going on here? Let's break it down: ``` cwlVersion: cwl:draft-3 class: CommandLineTool ``` The 'cwlVersion` field indicates the version of the CWL spec used by the document. The `class` field indicates this document describes a command line tool. ``` baseCommand: echo ``` The `baseCommand` provides the name of program that will actually run (echo) ``` inputs: - id: message type: string inputBinding: position: 1 ``` The `inputs` section describes the inputs of the tool. This is a list of input parameters and each parameter includes an identifier, a data type, and optionally an `inputBinding` which describes how this input parameter should appear on the command line. In this example, the `position` field indicates where it should appear on the command line. ``` outputs: [] ``` This tool has no formal output, so the `outputs` section is an empty list. - | ## Essential input parameters The `inputs` of a tool is a list of input parameters that control how to run the tool. Each parameter has an `id` for the name of parameter, and `type` describing what types of values are valid for that parameter. Available primitive types are *string*, *int*, *long*, *float*, *double*, and *null*; complex types are *array* and *record*; in addition there are special types *File* and *Any*. The following example demonstrates some input parameters with different types and appearing on the command line in different ways: *inp.cwl* ``` - $include: examples/inp.cwl - | ``` *inp-job.yml* ``` - $include: examples/inp-job.yml - | ``` Notice that "example_file", as a `File` type, must be provided as an object with the fields `class: File` and `path`. Next, create a whale.txt and invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ touch whale.txt $ cwl-runner inp.cwl inp-job.yml [job 140020149614160] /home/example$ echo -f -i42 --example-string hello --file=/home/example/whale.txt -f -i42 --example-string hello --file=/home/example/whale.txt Final process status is success ``` The field `inputBinding` is optional and indicates whether and how the input parameter should be appear on the tool's command line. If `inputBinding` is missing, the parameter does not appear on the command line. Let's look at each example in detail. ``` - id: example_flag type: boolean inputBinding: position: 1 prefix: -f ``` Boolean types are treated as a flag. If the input parameter "example_flag" is "true", then `prefix` will be added to the command line. If false, no flag is added. ``` - id: example_string type: string inputBinding: position: 3 prefix: --example-string ``` String types appear on the command line as literal values. The `prefix` is optional, if provided, it appears as a separate argument on the command line before the parameter . In the example above, this is rendered as `--example-string hello`. ``` - id: example_int type: int inputBinding: position: 2 prefix: -i separate: false ``` Integer (and floating point) types appear on the command line with decimal text representation. When the option `separate` is false (the default value is true), the prefix and value are combined into a single argument. In the example above, this is rendered as `-i42`. ``` - id: example_file type: ["null", File] inputBinding: prefix: --file= separate: false position: 4 ``` File types appear on the command line as the path to the file. When the parameter type is a list, this indicates several alternate types are valid for this parameter. The most common use is to provide "null" as an alternate parameter type, which indicates that the parameter is optional. In the example above, this is rendered as `--file=/home/example/whale.txt`. However, if the "example_file" parameter were not provided in the input, nothing would appear on the command line. Input files are read-only. If you wish to update an input file, you must first copy it to the output directory. The value of `position` is used to determine where parameter should appear on the command line. Positions are relative to one another, not abosolute. As a result, positions do not have to be sequential, three parameters with positions `[1, 3, 5]` will result in the same command line as `[1, 2, 3]`. More than one parameter can have the same position (ties are broken using the parameter name), and the position field itself is optional. the default position is 0. The `baseCommand` field always comes before parameters. - | ## Returning output files The `outputs` of a tool is a list of output parameters that should be returned after running the tool. Each parameter has an `id` for the name of parameter, and `type` describing what types of values are valid for that parameter. When a tool runs under CWL, the starting working directory is the designated output directory. The underlying tool or script must record its results in the form of files created in the output directory. The output parameters returned by the CWL tool are either the output files themselves, or come from examining the content of those files. *tar.cwl* ``` - $include: examples/tar.cwl - | ``` *tar-job.yml* ``` - $include: examples/tar-job.yml - | ``` Next, create a tar file for the example and invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ touch hello.txt && tar -cvf hello.tar hello.txt $ cwl-runner tar.cwl tar-job.yml [job 139868145165200] $ tar xf /home/example/hello.tar Final process status is success { "example_out": { "path": "hello.txt", "size": 13, "class": "File", "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b" } } ``` The field `outputBinding` describes how to to set the value of each output parameter. ``` outputs: - id: example_out type: File outputBinding: glob: hello.txt ``` The `glob` field consists of the name of a file in the output directory. If you don't know name of the file in advance, you can use a wildcard pattern. - | ## Capturing a tool's standard output stream To capture a tool's standard output stream, add the `stdout` field with the name of the file where the output stream should go. Then use `glob` on `outputBinding` to return the file. *stdout.cwl* ``` - $include: examples/stdout.cwl - | ``` *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ cwl-runner stdout.cwl echo-job.yml [job 140199012414352] $ echo 'Hello world!' > output.txt Final process status is success { "output": { "path": "output.txt", "size": 13, "class": "File", "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b" } } $ cat output.txt Hello world! ``` - | ## Parameter references In a previous example, we used extracted a file using the "tar" program. However, that example was very limited becuase it assumed that the file we were interested in was called "hello.txt". In this example, you will see how to reference the value of input parameters dynamically from other fields. *tar-param.cwl* ``` - $include: examples/tar-param.cwl - | ``` *tar-param-job.yml* ``` - $include: examples/tar-param-job.yml - | ``` Create your input files and invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ rm hello.tar || true && touch goodbye.txt && tar -cvf hello.tar $ cwl-runner tar-param.cwl tar-param-job.yml [job 139868145165200] $ tar xf /home/example/hello.tar goodbye.txt Final process status is success { "example_out": { "path": "goodbye.txt", "size": 24, "class": "File", "checksum": "sha1$dd0a4c4c49ba43004d6611771972b6cf969c1c01" } } ``` Certain fields permit parameter references which are enclosed in `$(...)`. These are evaluated and replaced with value being referenced. ``` outputs: - id: example_out type: File outputBinding: glob: $(inputs.extractfile) ``` References are written using a subset of Javascript syntax. In this example, `$(inputs.extractfile)`, `$(inputs["extractfile"])`, and `$(inputs['extractfile'])` are equivalent. The value of the "inputs" variable is the input object provided when the CWL tool was invoked. Note that because File parameters are objects, to get the path to an input file you must reference the path field on a file object; to reference the path to the tar file in the above example you would write `$(inputs.tarfile.path)`. - | ## Running tools inside Docker [Docker](http://docker.io) containers simplify software installation by providing a complete known-good runtime for software and its dependencies. However, containers are also purposefully isolated from the host system, so in order to run a tool inside a Docker container there is additional work to ensure that input files are available inside the container and output files can be recovered from the contianer. CWL can perform this work automatically, allowing you to use Docker to simplify your software management while avoiding the complexity of invoking and managing Docker containers. This example runs a simple Node.js script inside a Docker container. *docker.cwl* ``` - $include: examples/docker.cwl - | ``` *docker-job.yml* ``` - $include: examples/docker-job.yml - | ``` Provide a hello.js and invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ echo "console.log(\"Hello World\");" > hello.js $ cwl-runner docker.cwl docker-job.yml [job 140259721854416] /home/example$ docker run -i --volume=/home/example/hello.js:/var/lib/cwl/job369354770_examples/hello.js:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpDLs5hm:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp node:slim node /var/lib/cwl/job369354770_examples/hello.js Hello world! Final process status is success ``` Notice the CWL runner has constructed a Docker command line to run the script. One of the responsibilies of the CWL runner is to the paths of input files to reflect the location where they appear inside the container. In this example, the path to the script `hello.js` is `/home/example/hello.js` outside the container but `/var/lib/cwl/job369354770_examples/hello.js` inside the container, as reflected in the invocation of the `node` command. - | ## Additional command line arguments and runtime parameters Sometimes tools require additional command line options that don't correspond exactly to input parameters. In this example, we will wrap the Java compiler to compile a java source file to a class file. By default, `javac` will create the class files in the same directory as the source file. However, CWL input files (and the directories in which they appear) may be read-only, so we need to instruct javac to write the class file to the designated output directory instead. *arguments.cwl* ``` - $include: examples/arguments.cwl - | ``` *arguments-job.yml* ``` - $include: examples/arguments-job.yml - | ``` Now create a sample Java file and invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ echo "public class Hello {}" > Hello.java $ cwl-runner arguments.cwl arguments-job.yml [job 140051188854928] /home/example$ docker run -i --volume=/home/example/Hello.java:/var/lib/cwl/job710906416_example/Hello.java:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpdlQDWi:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp java:7 javac -d /var/spool/cwl /var/lib/cwl/job710906416_examples/Hello.java Final process status is success { "classfile": { "size": 416, "path": "/home/example/Hello.class", "checksum": "sha1$2f7ac33c1f3aac3f1fec7b936b6562422c85b38a", "class": "File" } } ``` Here we use the `arguments` field to add an additional argument to the command line that isn't tied to a specific input parameter. ``` arguments: - prefix: "-d" valueFrom: $(runtime.outdir) ``` This example references a runtime parameter. Runtime parameters provide information about the hardware or software environment when the tool is actually executed. The `$(runtime.outdir)` parameter is the path to the designated output directory. Other parameters include `$(runtime.tmpdir)`, `$(runtime.ram)`, `$(runtime.cores)`, `$(runtime.ram)`, `$(runtime.outdirSize)`, and `$(runtime.tmpdirSize)`. See the [Runtime Environment](CommandLineTool.html#Runtime_environment) section of the CWL specification for details. - | ## Array inputs It is easy to add arrays of input parameters represented to the command line. To specify an array parameter, the array definition is nested under the `type` field with `type: array` and `items` defining the valid data types that may appear in the array. *array-inputs.cwl* ``` - $include: examples/array-inputs.cwl - | ``` *array-inputs-job.yml* ``` - $include: examples/array-inputs-job.yml - | ``` Now invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ cwl-runner array-inputs.cwl array-inputs-job.yml [job 140334923640912] /home/example$ echo -A one two three -B=four -B=five -B=six -C=seven,eight,nine -A one two three -B=four -B=five -B=six -C=seven,eight,nine Final process status is success {} ``` The `inputBinding` can appear either on the outer array parameter definition or the inner array element definition, and these produce different behavior when constructing the command line, as shown above. In addition, the `itemSeperator` field, if provided, specifies that array values should be concatenated into a single argument separated by the item separator string. You can specify arrays of arrays, arrays of records, and other complex types. - | ## Array outputs You can also capture multiple output files into an array of files using `glob`. *array-outputs.cwl* ``` - $include: examples/array-outputs.cwl - | ``` *array-outpust-job.yml* ``` - $include: examples/array-outputs-job.yml - | ``` Now invoke `cwl-runner` providing the tool wrapper and the input object on the command line: ``` $ cwl-runner array-outputs.cwl array-outputs-job.yml [job 140190876078160] /home/example$ touch foo.txt bar.dat baz.txt Final process status is success { "output": [ { "size": 0, "path": "/home/peter/work/common-workflow-language/draft-3/examples/foo.txt", "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", "class": "File" }, { "size": 0, "path": "/home/peter/work/common-workflow-language/draft-3/examples/baz.txt", "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", "class": "File" } ] } ``` - | ## Record inputs, dependent and mutually exclusive parameters Sometimes an underlying tool has several arguments that must be provided together (they are dependent) or several arguments that cannot be provided together (they are exclusive). You can use records and type unions to group parameters together to describe these two conditions. *record.cwl* ``` - $include: examples/record.cwl - | ``` *record-job1.yml* ``` - $include: examples/record-job1.yml - | ``` ``` $ cwl-runner record.cwl record-job1.yml Workflow error: Error validating input record, could not validate field `dependent_parameters` because missing required field `itemB` ``` In the first example, you can't provide `itemA` without also providing `itemB`. *record-job2.yml* ``` - $include: examples/record-job2.yml - | ``` ``` $ cwl-runner record.cwl record-job2.yml [job 140566927111376] /home/example$ echo -A one -B two -C three -A one -B two -C three Final process status is success {} ``` In the second example, `itemC` and `itemD` are exclusive, so only `itemC` is added to the command line and `itemD` is ignored. *record-job3.yml* ``` - $include: examples/record-job2.yml - | ``` ``` $ cwl-runner record.cwl record-job3.yml [job 140606932172880] /home/example$ echo -A one -B two -D four -A one -B two -D four Final process status is success {} ``` In the third example, only `itemD` is provided, so it appears on the command line. - | ## Environment variables Tools run in a restricted environment and do not inherit most environment variables from the parent process. You can set environment variables for the tool using `EnvVarRequirement`. *env.cwl* ``` - $include: examples/env.cwl - | ``` *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwl-runner env.cwl echo-job.yml [job 140710387785808] /home/example$ env PATH=/bin:/usr/bin:/usr/local/bin HELLO=Hello world! TMPDIR=/tmp/tmp63Obpk Final process status is success {} ``` - | ## Javascript expressions If you need to manipulate input parameters, include the requirement `InlineJavascriptRequirement` and then anywhere a parameter reference is legal you can provide a fragment of Javascript that will be evaluated by the CWL runner. *expression.cwl* ``` - $include: examples/expression.cwl - | ``` ``` $ cwl-runner expression.cwl empty.yml [job 140000594593168] /home/example$ echo -A 2 -B baz -C 10 9 8 7 6 5 4 3 2 1 -A 2 -B baz -C 10 9 8 7 6 5 4 3 2 1 Final process status is success {} ``` You can only use expressions in certain fields. These are: `filename`, `fileContent`, `envValue`, `valueFrom`, `glob`, `outputEval`, `stdin`, `stdout`, `coresMin`, `coresMax`, `ramMin`, `ramMax`, `tmpdirMin`, `tmpdirMax`, `outdirMin`, and `outdirMax`. - | ## Creating files at runtime Sometimes you need to create a file on the fly from input parameters, such as tools which expect to read their input configuration from a file rather than the command line parameters. To do this, use `CreateFileRequirement`. *createfile.cwl* ``` - $include: examples/createfile.cwl - | ``` *echo-job.yml* ``` - $include: examples/echo-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwltool createfile.cwl echo-job.yml [job 140528604979344] /home/example$ cat example.conf CONFIGVAR=Hello world! Final process status is success {} ``` - | ## Staging input files in the output directory Normally, input files are located in a read-only directory separate from the output directory. This causes problems if the underlying tool expects to write its output files alongside the input file in the same directory. You use `CreateFileRequirement` to stage input files into the output directory. In this example, we use a Javascript expression to extract the base name of the input file from its leading directory path. *linkfile.cwl* ``` - $include: examples/linkfile.cwl - | ``` *arguments-job.yml* ``` - $include: examples/arguments-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwl-runner linkfile.cwl arguments-job.yml [job 139928309171664] /home/example$ docker run -i --volume=/home/example/Hello.java:/var/lib/cwl/job557617295_examples/Hello.java:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpmNbApw:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp java:7 javac Hello.java Final process status is success { "classfile": { "size": 416, "path": "/home/example/Hello.class", "checksum": "sha1$2f7ac33c1f3aac3f1fec7b936b6562422c85b38a", "class": "File" } } ``` - | # Writing Workflows ## First workflow This workflow extracts a java source file from a tar file and then compiles it. *1st-workflow.cwl* ``` - $include: examples/1st-workflow.cwl - | ``` Use a JSON object in a separate file to describe the input of a run: *1st-workflow-job.yml* ``` - $include: examples/1st-workflow-job.yml - | ``` Now invoke `cwl-runner` with the tool wrapper and the input object on the command line: ``` $ cwl-runner 1st-workflow.cwl 1st-workflow-job.yml [job untar] /tmp/tmp94qFiM$ tar xf /home/example/hello.tar Hello.java [step untar] completion status is success [job compile] /tmp/tmpu1iaKL$ docker run -i --volume=/tmp/tmp94qFiM/Hello.java:/var/lib/cwl/job301600808_tmp94qFiM/Hello.java:ro --volume=/tmp/tmpu1iaKL:/var/spool/cwl:rw --volume=/tmp/tmpfZnNdR:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp java:7 javac -d /var/spool/cwl /var/lib/cwl/job301600808_tmp94qFiM/Hello.java [step compile] completion status is success [workflow 1st-workflow.cwl] outdir is /home/example Final process status is success { "classout": { "path": "/home/example/Hello.class", "checksum": "sha1$2f7ac33c1f3aac3f1fec7b936b6562422c85b38a", "class": "File", "size": 416 } } ``` What's going on here? Let's break it down: ``` cwlVersion: cwl:draft-3 class: Workflow ``` The 'cwlVersion` field indicates the version of the CWL spec used by the document. The `class` field indicates this document describes a workflow. ``` inputs: - id: inp type: File - id: ex type: string ``` The `inputs` section describes the inputs of the workflow. This is a list of input parameters where each parameter consists of an identifier and a data type. These parameters can be used as sources for input to specific workflows steps. ``` outputs: - id: classout type: File source: "#compile/classfile" ``` The `outputs` section describes the outputs of the workflow. This is a list of output parameters where each parameter consists of an identifier and a data type. The `source` connects the output parameter `classfile` of the `compile` step to the workflow output parameter `classout`. ``` steps: - id: untar run: tar-param.cwl inputs: - id: tarfile source: "#inp" - id: extractfile source: "#ex" outputs: - id: example_out ``` The `steps` section describes the actual steps of the workflow. In this example, the first step extracts a file from a tar file, and the second step compiles the file from the first step using the java compiler. Workflow steps are not necessarily run in the order they are listed, instead the order is determined by the dependencies between steps (using `source`). In addition, workflow steps which do not depend on one another may run in parallel. The first step, `untar` runs `tar-param.cwl` (described previously in [Parameter references](#Parameter_references)). This tool has two input parameters, `tarfile` and `extractfile` and one output parameter `example_out`. The `inputs` section of the workflow step connects these two input parameters to the inputs of the workflow, `#inp` and `#ex` using `source`. This means that when the workflow step is executed, the values assigned to `#inp` and `#ex` will be used for the parameters `tarfile` and `extractfile` in order to run the tool. The `outputs` section of the workflow step lists the output parameters that are expected from the tool. ``` - id: compile run: arguments.cwl inputs: - id: src source: "#untar/example_out" outputs: - id: classfile ``` The second step `compile` depends on the results from the first step by connecting the input parameter `src` to the output parameter of `untar` using `#untar/example_out`. The output of this step `classfile` is connected to the `outputs` section for the Workflow, described above. cwltool-1.0.20170114120503/cwltool/schemas/draft-3/Workflow.yml0000644000175000017500000004171013036412052024336 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" $graph: - name: "WorkflowDoc" type: documentation doc: - | # Common Workflow Language (CWL) Workflow Description, draft 3 This version: * https://w3id.org/cwl/draft-3/ Current version: * https://w3id.org/cwl/ - "\n\n" - {$include: contrib.md} - "\n\n" - | # Abstract A Workflow is an analysis task represented by a directed graph describing a sequence of operations that transform an input data set to output. This specification defines the Common Workflow Language (CWL) Workflow description, a vendor-neutral standard for representing workflows intended to be portable across a variety of computing platforms. - {$include: intro.md} - | ## Introduction to draft 3 This specification represents the third milestone of the CWL group. Since draft-2, this draft introduces the following changes and additions: * Greatly simplified naming within a document with scoped identifiers, as described in the [Schema Salad specification](SchemaSalad.html). * The draft-2 concept of pluggable expression engines has been replaced by a [streamlined expression syntax)[#Parameter_references] and standardization on [Javascript](#Expressions). * [File](#File) objects can now include a `format` field to indicate the file type. * The addition of [MultipleInputFeatureRequirement](#MultipleInputFeatureRequirement). * The addition of [StepInputExpressionRequirement](#StepInputExpressionRequirement). * The separation of Workflow and CommandLineTool components into separate specifications. ## Purpose The Common Workflow Language Command Line Tool Description express workflows for data-intensive science, such as Bioinformatics, Chemistry, Physics, and Astronomy. This specification is intended to define a data and execution model for Workflows that can be implemented on top of a variety of computing platforms, ranging from an individual workstation to cluster, grid, cloud, and high performance computing systems. - {$include: concepts.md} - type: record name: ExpressionTool extends: "#Process" documentRoot: true doc: | Execute an expression as a process step. fields: - name: "class" jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: expression type: [string, "#Expression"] doc: | The expression to execute. The expression must return a JSON object which matches the output parameters of the ExpressionTool. - name: LinkMergeMethod type: enum docParent: "#WorkflowStepInput" doc: The input link merge method, described in [WorkflowStepInput](#WorkflowStepInput). symbols: - merge_nested - merge_flattened - name: WorkflowOutputParameter type: record extends: ["#OutputParameter", "#Sink"] docParent: "#Workflow" doc: | Describe an output parameter of a workflow. The parameter must be connected to one or more parameters defined in the workflow that will provide the value of the output parameter. - name: Sink type: record abstract: true fields: - name: source doc: | Specifies one or more workflow parameters that will provide input to the underlying process parameter. jsonldPredicate: "_id": "cwl:source" "_type": "@id" type: - "null" - string - type: array items: string - name: linkMerge type: ["null", "#LinkMergeMethod"] doc: | The method to use to merge multiple inbound links into a single array. If not specified, the default method is "merge_nested". - type: record name: WorkflowStepInput extends: "#Sink" docParent: "#WorkflowStep" doc: | The input of a workflow step connects an upstream parameter (from the workflow inputs, or the outputs of other workflows steps) with the input parameters of the underlying process. ## Input object A WorkflowStepInput object must contain an `id` field in the form `#fieldname` or `#stepname.fieldname`. When the `id` field contains a period `.` the field name consists of the characters following the final period. This defines a field of the workflow step input object with the value of the `source` parameter(s). ## Merging To merge multiple inbound data links, [MultipleInputFeatureRequirement](#MultipleInputFeatureRequirement) must be specified in the workflow or workflow step requirements. If the sink parameter is an array, or named in a [workflow scatter](#WorkflowStep) operation, there may be multiple inbound data links listed in the `source` field. The values from the input links are merged depending on the method specified in the `linkMerge` field. If not specified, the default method is "merge_nested". * **merge_nested** The input must be an array consisting of exactly one entry for each input link. If "merge_nested" is specified with a single link, the value from the link must be wrapped in a single-item list. * **merge_flattened** 1. The source and sink parameters must be compatible types, or the source type must be compatible with single element from the "items" type of the destination array parameter. 2. Source parameters which are arrays are concatenated. Source parameters which are single element types are appended as single elements. fields: - name: id type: string jsonldPredicate: "@id" doc: "A unique identifier for this workflow input parameter." - name: default type: ["null", Any] doc: | The default value for this parameter if there is no `source` field. jsonldPredicate: "cwl:default" - name: valueFrom type: - "null" - "string" - "#Expression" jsonldPredicate: "cwl:valueFrom" doc: | To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must be specified in the workflow or workflow step requirements. If `valueFrom` is a constant string value, use this as the value for this input parameter. If `valueFrom` is a parameter reference or expression, it must be evaluated to yield the actual value to be assiged to the input field. The `self` value of in the parameter reference or expression must be the value of the parameter(s) specified in the `source` field, or null if there is no `source` field. The value of `inputs` in the parameter reference or expression is the input object to the workflow step after assigning the `source` values, but before evaluating any step with `valueFrom`. The order of evaluating `valueFrom` among step input parameters is undefined. - type: record name: WorkflowStepOutput docParent: "#WorkflowStep" doc: | Associate an output parameter of the underlying process with a workflow parameter. The workflow parameter (given in the `id` field) be may be used as a `source` to connect with input parameters of other workflow steps, or with an output parameter of the process. fields: - name: id type: string jsonldPredicate: "@id" doc: | A unique identifier for this workflow output parameter. This is the identifier to use in the `source` field of `WorkflowStepInput` to connect the output value to downstream parameters. - name: ScatterMethod type: enum docParent: "#WorkflowStep" doc: The scatter method, as described in [workflow step scatter](#WorkflowStep). symbols: - dotproduct - nested_crossproduct - flat_crossproduct - name: WorkflowStep type: record docParent: "#Workflow" doc: | A workflow step is an executable element of a workflow. It specifies the underlying process implementation (such as `CommandLineTool`) in the `run` field and connects the input and output parameters of the underlying process to workflow parameters. # Scatter/gather To use scatter/gather, [ScatterFeatureRequirement](#ScatterFeatureRequirement) must be specified in the workflow or workflow step requirements. A "scatter" operation specifies that the associated workflow step or subworkflow should execute separately over a list of input elements. Each job making up a scatter operation is independent and may be executed concurrently. The `scatter` field specifies one or more input parameters which will be scattered. An input parameter may be listed more than once. The declared type of each input parameter is implicitly wrapped in an array for each time it appears in the `scatter` field. As a result, upstream parameters which are connected to scattered parameters may be arrays. All output parameter types are also implicitly wrapped in arrays. Each job in the scatter results in an entry in the output array. If `scatter` declares more than one input parameter, `scatterMethod` describes how to decompose the input into a discrete set of jobs. * **dotproduct** specifies that each of the input arrays are aligned and one element taken from each array to construct each job. It is an error if all input arrays are not the same length. * **nested_crossproduct** specifies the Cartesian product of the inputs, producing a job for every combination of the scattered inputs. The output must be nested arrays for each level of scattering, in the order that the input arrays are listed in the `scatter` field. * **flat_crossproduct** specifies the Cartesian product of the inputs, producing a job for every combination of the scattered inputs. The output arrays must be flattened to a single level, but otherwise listed in the order that the input arrays are listed in the `scatter` field. # Subworkflows To specify a nested workflow as part of a workflow step, [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) must be specified in the workflow or workflow step requirements. fields: - name: id type: string jsonldPredicate: "@id" doc: "The unique identifier for this workflow step." - name: inputs type: type: array items: "#WorkflowStepInput" jsonldPredicate: "cwl:inputs" doc: | Defines the input parameters of the workflow step. The process is ready to run when all required input parameters are associated with concrete values. Input parameters include a schema for each parameter which is used to validate the input object. It may also be used build a user interface for constructing the input object. - name: outputs type: type: array items: "#WorkflowStepOutput" jsonldPredicate: "cwl:outputs" doc: | Defines the parameters representing the output of the process. May be used to generate and/or validate the output object. - name: requirements type: - "null" - type: array items: "#ProcessRequirement" jsonldPredicate: "cwl:requirements" doc: | Declares requirements that apply to either the runtime environment or the workflow engine that must be met in order to execute this workflow step. If an implementation cannot satisfy all requirements, or a requirement is listed which is not recognized by the implementation, it is a fatal error and the implementation must not attempt to run the process, unless overridden at user option. - name: hints type: - "null" - type: array items: "Any" jsonldPredicate: "cwl:hints" doc: | Declares hints applying to either the runtime environment or the workflow engine that may be helpful in executing this workflow step. It is not an error if an implementation cannot satisfy all hints, however the implementation may report a warning. jsonldPredicate: _id: cwl:hints noLinkCheck: true - name: label type: - "null" - string jsonldPredicate: "rdfs:label" doc: "A short, human-readable label of this process object." - name: description type: - "null" - string jsonldPredicate: "rdfs:comment" doc: "A long, human-readable description of this process object." - name: run type: [string, "#Process"] jsonldPredicate: "_id": "cwl:run" "_type": "@id" doc: | Specifies the process to run. - name: scatter type: - "null" - string - type: array items: string jsonldPredicate: "_id": "cwl:scatter" "_type": "@id" "_container": "@list" - name: scatterMethod doc: | Required if `scatter` is an array of more than one element. type: - "null" - "#ScatterMethod" jsonldPredicate: "_id": "cwl:scatterMethod" "_type": "@vocab" - name: Workflow type: record extends: "#Process" documentRoot: true specialize: - specializeFrom: "#OutputParameter" specializeTo: "#WorkflowOutputParameter" doc: | A workflow describes a set of **steps** and the **dependencies** between those processes. When a process produces output that will be consumed by a second process, the first process is a dependency of the second process. When there is a dependency, the workflow engine must execute the preceeding process and wait for it to successfully produce output before executing the dependent process. If two processes are defined in the workflow graph that are not directly or indirectly dependent, these processes are **independent**, and may execute in any order or execute concurrently. A workflow is complete when all steps have been executed. Dependencies between parameters are expressed using the `source` field on [workflow step input parameters](#WorkflowStepInput) and [workflow output parameters](#WorkflowOutputParameter). The `source` field expresses the dependency of one parameter on another such that when a value is associated with the parameter specified by `source`, that value is propagated to the destination parameter. When all data links inbound to a given step are fufilled, the step is ready to execute. ## Workflow success and failure A completed process must result in one of `success`, `temporaryFailure` or `permanentFailure` states. An implementation may choose to retry a process execution which resulted in `temporaryFailure`. An implementation may choose to either continue running other steps of a workflow, or terminate immediately upon `permanentFailure`. * If any step of a workflow execution results in `permanentFailure`, then the workflow status is `permanentFailure`. * If one or more steps result in `temporaryFailure` and all other steps complete `success` or are not executed, then the workflow status is `temporaryFailure`. * If all workflow steps are executed and complete with `success`, then the workflow status is `success`. # Extensions [ScatterFeatureRequirement](#ScatterFeatureRequirement) and [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) are available as standard extensions to core workflow semantics. fields: - name: "class" jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: steps doc: | The individual steps that make up the workflow. Each step is executed when all of its input data links are fufilled. An implementation may choose to execute the steps in a different order than listed and/or execute steps concurrently, provided that dependencies between steps are met. type: - type: array items: "#WorkflowStep" - type: record name: SubworkflowFeatureRequirement extends: "#ProcessRequirement" doc: | Indicates that the workflow platform must support nested workflows in the `run` field of (WorkflowStep)(#WorkflowStep). - name: ScatterFeatureRequirement type: record extends: "#ProcessRequirement" doc: | Indicates that the workflow platform must support the `scatter` and `scatterMethod` fields of [WorkflowStep](#WorkflowStep). - name: MultipleInputFeatureRequirement type: record extends: "#ProcessRequirement" doc: | Indicates that the workflow platform must support multiple inbound data links listed in the `source` field of [WorkflowStepInput](#WorkflowStepInput). - type: record name: StepInputExpressionRequirement extends: "#ProcessRequirement" doc: | Indicate that the workflow platform must support the `valueFrom` field of [WorkflowStepInput](#WorkflowStepInput).cwltool-1.0.20170114120503/cwltool/schemas/draft-3/contrib.md0000644000175000017500000000145113036412052023761 0ustar michaelmichael00000000000000Authors: * Peter Amstutz , Arvados Project, Curoverse * Nebojša Tijanić , Seven Bridges Genomics Contributers: * Brad Chapman , Harvard Chan School of Public Health * John Chilton , Galaxy Project, Pennsylvania State University * Michael R. Crusoe , University of California, Davis * Andrey Kartashov , Cincinnati Children's Hospital * Dan Leehr , Duke University * Hervé Ménager , Institut Pasteur * Stian Soiland-Reyes [soiland-reyes@cs.manchester.ac.uk](mailto:soiland-reyes@cs.manchester.ac.uk), University of Manchester * Luka Stojanovic , Seven Bridges Genomics cwltool-1.0.20170114120503/cwltool/schemas/draft-3/README.md0000644000175000017500000000175313036412052023263 0ustar michaelmichael00000000000000# Common Workflow Language Specifications, draft-3 The CWL specifications are divided up into several documents. The [User Guide](UserGuide.html) provides a gentle introduction to writing CWL command line tools and workflows. The [Command Line Tool Description Specification](CommandLineTool.html) specifies the document schema and execution semantics for wrapping and executing command line tools. The [Workflow Description Specification](Workflow.html) specifies the document schema and execution semantics for composing workflows from components such as command line tools and other workflows. The [Semantic Annotations for Linked Avro Data (SALAD) Specification](SchemaSalad.html) specifies the preprocessing steps that must be applied when loading CWL documents and the schema language used to write the above specifications. If you use the CWL specifications or distribute CWL descriptions with a publication you should [cite the standard](https://dx.doi.org/10.6084/m9.figshare.3115156.v1) cwltool-1.0.20170114120503/cwltool/schemas/draft-3/CommandLineTool.yml0000644000175000017500000005635213036412052025560 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" $graph: - name: CommandLineToolDoc type: documentation doc: - | # Common Workflow Language (CWL) Command Line Tool Description, draft 3 This version: * https://w3id.org/cwl/draft-3/ Current version: * https://w3id.org/cwl/ - "\n\n" - {$include: contrib.md} - "\n\n" - | # Abstract A Command Line Tool is a non-interactive executable program that reads some input, performs a computation, and terminates after producing some output. Command line programs are a flexible unit of code sharing and reuse, unfortunately the syntax and input/output semantics among command line programs is extremely heterogeneous. A common layer for describing the syntax and semantics of programs can reduce this incidental complexity by providing a consistent way to connect programs together. This specification defines the Common Workflow Language (CWL) Command Line Tool Description, a vendor-neutral standard for describing the syntax and input/output semantics of command line programs. - {$include: intro.md} - | ## Introduction to draft 3 This specification represents the third milestone of the CWL group. Since draft-2, this draft introduces the following major changes and additions: * Greatly simplified naming within a document with scoped identifiers, as described in the [Schema Salad specification](SchemaSalad.html). * The draft-2 concept of pluggable expression engines has been replaced by a [streamlined expression syntax](#Parameter_references) and standardization on [Javascript](#Expressions). * [File](#File) objects can now include a `format` field to indicate the file type. * The addition of [ShellCommandRequirement](#ShellCommandRequirement). * The addition of [ResourceRequirement](#ResourceRequirement). * The separation of CommandLineTool and Workflow components into separate specifications. ## Purpose Standalone programs are a flexible and interoperable form of code reuse. Unlike monolithic applications, applications and analysis workflows which are composed of multiple separate programs can be written in multiple languages and execute concurrently on multiple hosts. However, POSIX does not dictate computer-readable grammar or semantics for program input and output, resulting in extremely heterogeneous command line grammar and input/output semantics among program. This a particular problem in distributed computing (multi-node compute clusters) and virtualized environments (such as Docker containers) where it is often necessary to provision resources such as input files before executing the program. Often this is gap is filled by hard coding program invocation and implicitly assuming requirements will be met, or abstracting program invocation with wrapper scripts or descriptor documents. Unfortunately, where these approaches are application or platform specific it creates a significant barrier to reproducibility and portability, as methods developed for one platform must be manually ported to be used on new platforms. Similarly it creates redundant work, as wrappers for popular tools must be rewritten for each application or platform in use. The Common Workflow Language Command Line Tool Description is designed to provide a common standard description of grammar and semantics for invoking programs used in data-intensive fields such as Bioinformatics, Chemistry, Physics, Astronomy, and Statistics. This specification defines a precise data and execution model for Command Line Tools that can be implemented on a variety of computing platforms, ranging from a single workstation to cluster, grid, cloud, and high performance computing platforms. - {$include: concepts.md} - {$include: invocation.md} - type: record name: FileDef doc: | Define a file that must be placed in the designated output directory prior to executing the command line tool. May be the result of executing an expression, such as building a configuration file from a template. fields: - name: "filename" type: ["string", "#Expression"] doc: "The name of the file to create in the output directory." - name: "fileContent" type: ["string", "#Expression"] doc: | If the value is a string literal or an expression which evaluates to a string, a new file must be created with the string as the file contents. If the value is an expression that evaluates to a File object, this indicates the referenced file should be added to the designated output directory prior to executing the tool. Files added in this way may be read-only, and may be provided by bind mounts or file system links to avoid unnecessary copying of the input file. - type: record name: EnvironmentDef doc: | Define an environment variable that will be set in the runtime environment by the workflow platform when executing the command line tool. May be the result of executing an expression, such as getting a parameter from input. fields: - name: "envName" type: "string" doc: The environment variable name - name: "envValue" type: ["string", "#Expression"] doc: The environment variable value - type: record name: CommandLineBinding extends: "#InputBinding" doc: | When listed under `inputBinding` in the input schema, the term "value" refers to the the corresponding value in the input object. For binding objects listed in `CommandLineTool.arguments`, the term "value" refers to the effective value after evaluating `valueFrom`. The binding behavior when building the command line depends on the data type of the value. If there is a mismatch between the type described by the input schema and the effective value, such as resulting from an expression evaluation, an implementation must use the data type of the effective value. - **string**: Add `prefix` and the string to the command line. - **number**: Add `prefix` and decimal representation to command line. - **boolean**: If true, add `prefix` to the command line. If false, add nothing. - **File**: Add `prefix` and the value of [`File.path`](#File) to the command line. - **array**: If `itemSeparator` is specified, add `prefix` and the join the array into a single string with `itemSeparator` separating the items. Otherwise first add `prefix`, then recursively process individual elements. - **object**: Add `prefix` only, and recursively add object fields for which `inputBinding` is specified. - **null**: Add nothing. fields: - name: "position" type: ["null", "int"] doc: "The sorting key. Default position is 0." - name: "prefix" type: [ "null", "string"] doc: "Command line prefix to add before the value." - name: "separate" type: ["null", boolean] doc: | If true (default), then the prefix and value must be added as separate command line arguments; if false, prefix and value must be concatenated into a single command line argument. - name: "itemSeparator" type: ["null", "string"] doc: | Join the array elements into a single string with the elements separated by by `itemSeparator`. - name: "valueFrom" type: - "null" - "string" - "#Expression" jsonldPredicate: "cwl:valueFrom" doc: | If `valueFrom` is a constant string value, use this as the value and apply the binding rules above. If `valueFrom` is an expression, evaluate the expression to yield the actual value to use to build the command line and apply the binding rules above. If the inputBinding is associated with an input parameter, the value of `self` in the expression will be the value of the input parameter. When a binding is part of the `CommandLineTool.arguments` field, the `valueFrom` field is required. - name: shellQuote type: ["null", boolean] doc: | If `ShellCommandRequirement` is in the requirements for the current command, this controls whether the value is quoted on the command line (default is true). Use `shellQuote: false` to inject metacharacters for operations such as pipes. - type: record name: CommandOutputBinding extends: "#OutputBinding" doc: | Describes how to generate an output parameter based on the files produced by a CommandLineTool. The output parameter is generated by applying these operations in the following order: - glob - loadContents - outputEval fields: - name: glob type: - "null" - string - "#Expression" - type: array items: string doc: | Find files relative to the output directory, using POSIX glob(3) pathname matching. If provided an array, find files that match any pattern in the array. If provided an expression, the expression must return a string or an array of strings, which will then be evaluated as one or more glob patterns. Must only match and return files which actually exist. - name: loadContents type: - "null" - boolean jsonldPredicate: "cwl:loadContents" doc: | For each file matched in `glob`, read up to the first 64 KiB of text from the file and place it in the `contents` field of the file object for manipulation by `outputEval`. - name: outputEval type: - "null" - string - "#Expression" doc: | Evaluate an expression to generate the output value. If `glob` was specified, the value of `self` must be an array containing file objects that were matched. If no files were matched, `self' must be a zero length array; if a single file was matched, the value of `self` is an array of a single element. Additionally, if `loadContents` is `true`, the File objects must include up to the first 64 KiB of file contents in the `contents` field. - name: CommandInputRecordField type: record extends: "#InputRecordField" specialize: - specializeFrom: "#InputRecordSchema" specializeTo: "#CommandInputRecordSchema" - specializeFrom: "#InputEnumSchema" specializeTo: "#CommandInputEnumSchema" - specializeFrom: "#InputArraySchema" specializeTo: "#CommandInputArraySchema" - specializeFrom: "#InputBinding" specializeTo: "#CommandLineBinding" - name: CommandInputRecordSchema type: record extends: "#InputRecordSchema" specialize: - specializeFrom: "#InputRecordField" specializeTo: "#CommandInputRecordField" - name: CommandInputEnumSchema type: record extends: "#InputEnumSchema" specialize: - specializeFrom: "#InputBinding" specializeTo: "#CommandLineBinding" - name: CommandInputArraySchema type: record extends: "#InputArraySchema" specialize: - specializeFrom: "#InputRecordSchema" specializeTo: "#CommandInputRecordSchema" - specializeFrom: "#InputEnumSchema" specializeTo: "#CommandInputEnumSchema" - specializeFrom: "#InputArraySchema" specializeTo: "#CommandInputArraySchema" - specializeFrom: "#InputBinding" specializeTo: "#CommandLineBinding" - name: CommandOutputRecordField type: record extends: "#OutputRecordField" specialize: - specializeFrom: "#OutputRecordSchema" specializeTo: "#CommandOutputRecordSchema" - specializeFrom: "#OutputEnumSchema" specializeTo: "#CommandOutputEnumSchema" - specializeFrom: "#OutputArraySchema" specializeTo: "#CommandOutputArraySchema" - specializeFrom: "#OutputBinding" specializeTo: "#CommandOutputBinding" - name: CommandOutputRecordSchema type: record extends: "#OutputRecordSchema" specialize: - specializeFrom: "#OutputRecordField" specializeTo: "#CommandOutputRecordField" - name: CommandOutputEnumSchema type: record extends: "#OutputEnumSchema" specialize: - specializeFrom: "#OutputRecordSchema" specializeTo: "#CommandOutputRecordSchema" - specializeFrom: "#OutputEnumSchema" specializeTo: "#CommandOutputEnumSchema" - specializeFrom: "#OutputArraySchema" specializeTo: "#CommandOutputArraySchema" - specializeFrom: "#OutputBinding" specializeTo: "#CommandOutputBinding" - name: CommandOutputArraySchema type: record extends: "#OutputArraySchema" specialize: - specializeFrom: "#OutputRecordSchema" specializeTo: "#CommandOutputRecordSchema" - specializeFrom: "#OutputEnumSchema" specializeTo: "#CommandOutputEnumSchema" - specializeFrom: "#OutputArraySchema" specializeTo: "#CommandOutputArraySchema" - specializeFrom: "#OutputBinding" specializeTo: "#CommandOutputBinding" - type: record name: CommandInputParameter extends: "#InputParameter" doc: An input parameter for a CommandLineTool. specialize: - specializeFrom: "#InputRecordSchema" specializeTo: "#CommandInputRecordSchema" - specializeFrom: "#InputEnumSchema" specializeTo: "#CommandInputEnumSchema" - specializeFrom: "#InputArraySchema" specializeTo: "#CommandInputArraySchema" - specializeFrom: "#InputBinding" specializeTo: "#CommandLineBinding" - type: record name: CommandOutputParameter extends: "#OutputParameter" doc: An output parameter for a CommandLineTool. specialize: - specializeFrom: "#OutputRecordSchema" specializeTo: "#CommandOutputRecordSchema" - specializeFrom: "#OutputEnumSchema" specializeTo: "#CommandOutputEnumSchema" - specializeFrom: "#OutputArraySchema" specializeTo: "#CommandOutputArraySchema" - specializeFrom: "#OutputBinding" specializeTo: "#CommandOutputBinding" - type: record name: CommandLineTool extends: "#Process" documentRoot: true specialize: - specializeFrom: "#InputParameter" specializeTo: "#CommandInputParameter" - specializeFrom: "#OutputParameter" specializeTo: "#CommandOutputParameter" doc: | This defines the schema of the CWL Command Line Tool Description document. fields: - name: "class" jsonldPredicate: "_id": "@type" "_type": "@vocab" type: string - name: baseCommand doc: | Specifies the program to execute. If the value is an array, the first element is the program to execute, and subsequent elements are placed at the beginning of the command line in prior to any command line bindings. If the program includes a path separator character it must be an absolute path, otherwise it is an error. If the program does not include a path separator, search the `$PATH` variable in the runtime environment of the workflow runner find the absolute path of the executable. type: - string - type: array items: string jsonldPredicate: "_id": "cwl:baseCommand" "_container": "@list" - name: arguments doc: | Command line bindings which are not directly associated with input parameters. type: - "null" - type: array items: [string, "#CommandLineBinding"] jsonldPredicate: "_id": "cwl:arguments" "_container": "@list" - name: stdin type: ["null", string, "#Expression"] doc: | A path to a file whose contents must be piped into the command's standard input stream. - name: stdout type: ["null", string, "#Expression"] doc: | Capture the command's standard output stream to a file written to the designated output directory. If `stdout` is a string, it specifies the file name to use. If `stdout` is an expression, the expression is evaluated and must return a string with the file name to use to capture stdout. If the return value is not a string, or the resulting path contains illegal characters (such as the path separator `/`) it is an error. - name: successCodes type: - "null" - type: array items: int doc: | Exit codes that indicate the process completed successfully. - name: temporaryFailCodes type: - "null" - type: array items: int doc: | Exit codes that indicate the process failed due to a possibly temporary condition, where excuting the process with the same runtime environment and inputs may produce different results. - name: permanentFailCodes type: - "null" - type: array items: int doc: Exit codes that indicate the process failed due to a permanent logic error, where excuting the process with the same runtime environment and same inputs is expected to always fail. - type: record name: DockerRequirement extends: "#ProcessRequirement" doc: | Indicates that a workflow component should be run in a [Docker](http://docker.com) container, and specifies how to fetch or build the image. If a CommandLineTool lists `DockerRequirement` under `hints` or `requirements`, it may (or must) be run in the specified Docker container. The platform must first acquire or install the correct Docker image as specified by `dockerPull`, `dockerImport`, `dockerLoad` or `dockerFile`. The platform must execute the tool in the container using `docker run` with the appropriate Docker image and tool command line. The workflow platform may provide input files and the designated output directory through the use of volume bind mounts. The platform may rewrite file paths in the input object to correspond to the Docker bind mounted locations. When running a tool contained in Docker, the workflow platform must not assume anything about the contents of the Docker container, such as the presence or absence of specific software, except to assume that the generated command line represents a valid command within the runtime environment of the container. ## Interaction with other requirements If [EnvVarRequirement](#EnvVarRequirement) is specified alongside a DockerRequirement, the environment variables must be provided to Docker using `--env` or `--env-file` and interact with the container's preexisting environment as defined by Docker. fields: - name: dockerPull type: ["null", "string"] doc: "Specify a Docker image to retrieve using `docker pull`." - name: "dockerLoad" type: ["null", "string"] doc: "Specify a HTTP URL from which to download a Docker image using `docker load`." - name: dockerFile type: ["null", "string"] doc: "Supply the contents of a Dockerfile which will be built using `docker build`." - name: dockerImport type: ["null", "string"] doc: "Provide HTTP URL to download and gunzip a Docker images using `docker import." - name: dockerImageId type: ["null", "string"] doc: | The image id that will be used for `docker run`. May be a human-readable image name or the image identifier hash. May be skipped if `dockerPull` is specified, in which case the `dockerPull` image id must be used. - name: dockerOutputDirectory type: ["null", "string"] doc: | Set the designated output directory to a specific location inside the Docker container. - name: CreateFileRequirement type: record extends: "#ProcessRequirement" doc: | Define a list of files that must be created by the workflow platform in the designated output directory prior to executing the command line tool. See `FileDef` for details. fields: - name: fileDef type: type: "array" items: "#FileDef" doc: The list of files. - name: EnvVarRequirement type: record extends: "#ProcessRequirement" doc: | Define a list of environment variables which will be set in the execution environment of the tool. See `EnvironmentDef` for details. fields: - name: envDef type: type: "array" items: "#EnvironmentDef" doc: The list of environment variables. - type: record name: ShellCommandRequirement extends: "#ProcessRequirement" doc: | Modify the behavior of CommandLineTool to generate a single string containing a shell command line. Each item in the argument list must be joined into a string separated by single spaces and quoted to prevent intepretation by the shell, unless `CommandLineBinding` for that argument contains `shellQuote: false`. If `shellQuote: false` is specified, the argument is joined into the command string without quoting, which allows the use of shell metacharacters such as `|` for pipes. - type: record name: ResourceRequirement extends: "#ProcessRequirement" doc: | Specify basic hardware resource requirements. "min" is the minimum amount of a resource that must be reserved to schedule a job. If "min" cannot be satisfied, the job should not be run. "max" is the maximum amount of a resource that the job shall be permitted to use. If a node has sufficient resources, multiple jobs may be scheduled on a single node provided each job's "max" resource requirements are met. If a job attempts to exceed its "max" resource allocation, an implementation may deny additional resources, which may result in job failure. If "min" is specified but "max" is not, then "max" == "min" If "max" is specified by "min" is not, then "min" == "max". It is an error if max < min. It is an error if the value of any of these fields is negative. If neither "min" nor "max" is specified for a resource, an implementation may provide a default. fields: - name: coresMin type: ["null", long, string, "#Expression"] doc: Minimum reserved number of CPU cores - name: coresMax type: ["null", int, string, "#Expression"] doc: Maximum reserved number of CPU cores - name: ramMin type: ["null", long, string, "#Expression"] doc: Minimum reserved RAM in mebibytes (2**20) - name: ramMax type: ["null", long, string, "#Expression"] doc: Maximum reserved RAM in mebibytes (2**20) - name: tmpdirMin type: ["null", long, string, "#Expression"] doc: Minimum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20) - name: tmpdirMax type: ["null", long, string, "#Expression"] doc: Maximum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20) - name: outdirMin type: ["null", long, string, "#Expression"] doc: Minimum reserved filesystem based storage for the designated output directory, in mebibytes (2**20) - name: outdirMax type: ["null", long, string, "#Expression"] doc: Maximum reserved filesystem based storage for the designated output directory, in mebibytes (2**20) cwltool-1.0.20170114120503/cwltool/schemas/draft-3/CommonWorkflowLanguage.yml0000644000175000017500000000032113036412052027144 0ustar michaelmichael00000000000000$base: "https://w3id.org/cwl/cwl#" $namespaces: cwl: "https://w3id.org/cwl/cwl#" sld: "https://w3id.org/cwl/salad#" $graph: - $import: Process.yml - $import: CommandLineTool.yml - $import: Workflow.yml cwltool-1.0.20170114120503/cwltool/schemas/draft-3/invocation.md0000644000175000017500000001717113036412052024500 0ustar michaelmichael00000000000000# Running a Command To accommodate the enormous variety in syntax and semantics for input, runtime environment, invocation, and output of arbitrary programs, a CommandLineTool defines an "input binding" that describes how to translate abstract input parameters to an concrete program invocation, and an "output binding" that describes how to generate output parameters from program output. ## Input binding The tool command line is built by applying command line bindings to the input object. Bindings are listed either as part of an [input parameter](#CommandInputParameter) using the `inputBinding` field, or separately using the `arguments` field of the CommandLineTool. The algorithm to build the command line is as follows. In this algorithm, the sort key is a list consisting of one or more numeric or string elements. Strings are sorted lexicographically based on UTF-8 encoding. 1. Collect `CommandLineBinding` objects from `arguments`. Assign a sorting key `[position, i]` where `position` is [`CommandLineBinding.position`](#CommandLineBinding) and `i` is the index in the `arguments` list. 2. Collect `CommandLineBinding` objects from the `inputs` schema and associate them with values from the input object. Where the input type is a record, array, or map, recursively walk the schema and input object, collecting nested `CommandLineBinding` objects and associating them with values from the input object. 3. Create a sorting key by taking the value of the `position` field at each level leading to each leaf binding object. If `position` is not specified, it is not added to the sorting key. For bindings on arrays and maps, the sorting key must include the array index or map key following the position. If and only if two bindings have the same sort key, the tie must be broken using the ordering of the field or parameter name immediately containing the leaf binding. 4. Sort elements using the assigned sorting keys. Numeric entries sort before strings. 5. In the sorted order, apply the rules defined in [`CommandLineBinding`](#CommandLineBinding) to convert bindings to actual command line elements. 6. Insert elements from `baseCommand` at the beginning of the command line. ## Runtime environment All files listed in the input object must be made available in the runtime environment. The implementation may use a shared or distributed file system or transfer files via explicit download. Implementations may choose not to provide access to files not explicitly specified in the input object or process requirements. Output files produced by tool execution must be written to the **designated output directory**. The initial current working directory when executing the tool must be the designated output directory. Files may also be written to the **designated temporary directory**. This directory must be isolated and not shared with other processes. Any files written to the designated temporary directory may be automatically deleted by the workflow platform immediately after the tool terminates. For compatibility, files may be written to the **system temporary directory** which must be located at `/tmp`. Because the system temporary directory may be shared with other processes on the system, files placed in the system temporary directory are not guaranteed to be deleted automatically. Correct tools must clean up temporary files written to the system temporary directory. A tool must not use the system temporary directory as a backchannel communication with other tools. It is valid for the system temporary directory to be the same as the designated temporary directory. When executing the tool, the tool must execute in a new, empty environment with only the environment variables described below; the child process must not inherit environment variables from the parent process except as specified or at user option. * `HOME` must be set to the designated output directory. * `TMPDIR` must be set to the designated temporary directory. when the tool invocation and output collection is complete. * `PATH` may be inherited from the parent process, except when run in a container that provides its own `PATH`. * Variables defined by [EnvVarRequirement](#EnvVarRequirement) * The default environment of the container, such as when using [DockerRequirement](#DockerRequirement) An implementation may forbid the tool from writing to any location in the runtime environment file system other than the designated temporary directory, system temporary directory, and designated output directory. An implementation may provide read-only input files, and disallow in-place update of input files. The designated temporary directory, system temporary directory and designated output directory may each reside on different mount points on different file systems. An implementation may forbid the tool from directly accessing network resources. Correct tools must not assume any network access. Future versions of the specification may incorporate optional process requirements that describe the networking needs of a tool. The `runtime` section available in [parameter references](#Parameter_references) and [expressions](#Expressions) contains the following fields. As noted earlier, an implementation may perform deferred resolution of runtime fields by providing opaque strings for any or all of the following fields; parameter references and expressions may only use the literal string value of the field and must not perform computation on the contents. * `runtime.outdir`: an absolute path to the designated output directory * `runtime.tmpdir`: an absolute path to the designated temporary directory * `runtime.cores`: number of CPU cores reserved for the tool process * `runtime.ram`: amount of RAM in mebibytes (2**20) reserved for the tool process * `runtime.outdirSize`: reserved storage space available in the designated output directory * `runtime.tmpdirSize`: reserved storage space available in the designated temporary directory See [ResourceRequirement](#ResourceRequirement) for details on how to describe the hardware resources required by a tool. The standard input stream and standard output stream may be redirected as described in the `stdin` and `stdout` fields. ## Execution Once the command line is built and the runtime environment is created, the actual tool is executed. The standard error stream and standard output stream (unless redirected by setting `stdout`) may be captured by platform logging facilities for storage and reporting. Tools may be multithreaded or spawn child processes; however, when the parent process exits, the tool is considered finished regardless of whether any detached child processes are still running. Tools must not require any kind of console, GUI, or web based user interaction in order to start and run to completion. The exit code of the process indicates if the process completed successfully. By convention, an exit code of zero is treated as success and non-zero exit codes are treated as failure. This may be customized by providing the fields `successCodes`, `temporaryFailCodes`, and `permanentFailCodes`. An implementation may choose to default unspecified non-zero exit codes to either `temporaryFailure` or `permanentFailure`. ## Output binding If the output directory contains a file named "cwl.output.json", that file must be loaded and used as the output object. Otherwise, the output object must be generated by walking the parameters listed in `outputs` and applying output bindings to the tool output. Output bindings are associated with output parameters using the `outputBinding` field. See [`CommandOutputBinding`](#CommandOutputBinding) for details. cwltool-1.0.20170114120503/cwltool/schemas/draft-3/userguide-intro.md0000644000175000017500000000236013036412052025446 0ustar michaelmichael00000000000000# A Gentle Introduction to the Common Workflow Language Hello! This guide will introduce you to writing tool wrappers and workflows using the Common Workflow Language (CWL). This guide describes the current stable specification, draft 3. Note: This document is a work in progress. Not all features are covered, yet. # Introduction CWL is a way to describe command line tools and connect them together to create workflows. Because CWL is a specification and not a specific piece of software, tools and workflows described using CWL are portable across a variety of platforms that support the CWL standard. CWL has roots in "make" and many similar tools that determine order of execution based on dependencies between tasks. However unlike "make", CWL tasks are isolated and you must be explicit about your inputs and outputs. The benefit of explicitness and isolation are flexibility, portability, and scalability: tools and workflows described with CWL can transparently leverage technologies such as Docker, be used with CWL implementations from different vendors, and is well suited for describing large-scale workflows in cluster, cloud and high performance computing environments where tasks are scheduled in parallel across many nodes. cwltool-1.0.20170114120503/cwltool/schemas/draft-3/index.yml0000644000175000017500000000030613036412052023627 0ustar michaelmichael00000000000000# Common Workflow Language draft-3 specifications CWL draft-3 specification consists of the following documents: * Command Line Tool description specification * Workflow description specification cwltool-1.0.20170114120503/cwltool/schemas/draft-3/CommandLineTool-standalone.yml0000644000175000017500000000006513036412052027674 0ustar michaelmichael00000000000000- $import: Process.yml - $import: CommandLineTool.ymlcwltool-1.0.20170114120503/cwltool/flatten.py0000644000175000017500000000113113036412052021116 0ustar michaelmichael00000000000000from typing import Any, Callable, List, cast # http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html def flatten(l, ltypes=(list, tuple)): # type: (Any, Any) -> List[Any] if l is None: return [] if not isinstance(l, ltypes): return [l] ltype = type(l) l = list(l) i = 0 while i < len(l): while isinstance(l[i], ltypes): if not l[i]: l.pop(i) i -= 1 break else: l[i:i + 1] = l[i] i += 1 return cast(Callable[[Any], List], ltype)(l) cwltool-1.0.20170114120503/cwltool/cwlNodeEngine.js0000755000175000017500000000062213036412052022175 0ustar michaelmichael00000000000000"use strict"; process.stdin.setEncoding('utf8'); var incoming = ""; process.stdin.on('data', function(chunk) { incoming += chunk; var i = incoming.indexOf("\n"); if (i > -1) { var fn = JSON.parse(incoming.substr(0, i)); incoming = incoming.substr(i+1); process.stdout.write(JSON.stringify(require("vm").runInNewContext(fn, {})) + "\n"); } }); process.stdin.on('end', process.exit); cwltool-1.0.20170114120503/cwltool/resolver.py0000644000175000017500000000170713036412052021333 0ustar michaelmichael00000000000000import os import logging import urllib import urlparse from schema_salad.ref_resolver import file_uri _logger = logging.getLogger("cwltool") def resolve_local(document_loader, uri): if uri.startswith("/"): return None shares = [os.environ.get("XDG_DATA_HOME", os.path.join(os.environ["HOME"], ".local", "share"))] shares.extend(os.environ.get("XDG_DATA_DIRS", "/usr/local/share/:/usr/share/").split(":")) shares = [os.path.join(s, "commonwl", uri) for s in shares] shares.insert(0, os.path.join(os.getcwd(), uri)) _logger.debug("Search path is %s", shares) for s in shares: if os.path.exists(s): return file_uri(s) if os.path.exists("%s.cwl" % s): return file_uri(s) return None def tool_resolver(document_loader, uri): for r in [resolve_local]: ret = r(document_loader, uri) if ret is not None: return ret return file_uri(os.path.abspath(uri)) cwltool-1.0.20170114120503/cwltool/builder.py0000644000175000017500000002205413036412052021116 0ustar michaelmichael00000000000000import copy from .utils import aslist from . import expression import avro import schema_salad.validate as validate from schema_salad.sourceline import SourceLine from typing import Any, Callable, Text, Type, Union from .errors import WorkflowException from .stdfsaccess import StdFsAccess from .pathmapper import PathMapper, adjustFileObjs, adjustDirObjs, normalizeFilesDirs CONTENT_LIMIT = 64 * 1024 def substitute(value, replace): # type: (Text, Text) -> Text if replace[0] == "^": return substitute(value[0:value.rindex('.')], replace[1:]) else: return value + replace class Builder(object): def __init__(self): # type: () -> None self.names = None # type: avro.schema.Names self.schemaDefs = None # type: Dict[Text, Dict[Text, Any]] self.files = None # type: List[Dict[Text, Text]] self.fs_access = None # type: StdFsAccess self.job = None # type: Dict[Text, Union[Dict[Text, Any], List, Text]] self.requirements = None # type: List[Dict[Text, Any]] self.hints = None # type: List[Dict[Text, Any]] self.outdir = None # type: Text self.tmpdir = None # type: Text self.resources = None # type: Dict[Text, Union[int, Text]] self.bindings = [] # type: List[Dict[Text, Any]] self.timeout = None # type: int self.pathmapper = None # type: PathMapper self.stagedir = None # type: Text self.make_fs_access = None # type: Type[StdFsAccess] self.build_job_script = None # type: Callable[[List[str]], Text] self.debug = False # type: bool def bind_input(self, schema, datum, lead_pos=[], tail_pos=[]): # type: (Dict[Text, Any], Any, Union[int, List[int]], List[int]) -> List[Dict[Text, Any]] bindings = [] # type: List[Dict[Text,Text]] binding = None # type: Dict[Text,Any] if "inputBinding" in schema and isinstance(schema["inputBinding"], dict): binding = copy.copy(schema["inputBinding"]) if "position" in binding: binding["position"] = aslist(lead_pos) + aslist(binding["position"]) + aslist(tail_pos) else: binding["position"] = aslist(lead_pos) + [0] + aslist(tail_pos) binding["datum"] = datum # Handle union types if isinstance(schema["type"], list): for t in schema["type"]: if isinstance(t, (str, Text)) and self.names.has_name(t, ""): avsc = self.names.get_name(t, "") elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""): avsc = self.names.get_name(t["name"], "") else: avsc = avro.schema.make_avsc_object(t, self.names) if validate.validate(avsc, datum): schema = copy.deepcopy(schema) schema["type"] = t return self.bind_input(schema, datum, lead_pos=lead_pos, tail_pos=tail_pos) raise validate.ValidationException(u"'%s' is not a valid union %s" % (datum, schema["type"])) elif isinstance(schema["type"], dict): st = copy.deepcopy(schema["type"]) if binding and "inputBinding" not in st and st["type"] == "array" and "itemSeparator" not in binding: st["inputBinding"] = {} for k in ("secondaryFiles", "format", "streamable"): if k in schema: st[k] = schema[k] bindings.extend(self.bind_input(st, datum, lead_pos=lead_pos, tail_pos=tail_pos)) else: if schema["type"] in self.schemaDefs: schema = self.schemaDefs[schema["type"]] if schema["type"] == "record": for f in schema["fields"]: if f["name"] in datum: bindings.extend(self.bind_input(f, datum[f["name"]], lead_pos=lead_pos, tail_pos=f["name"])) else: datum[f["name"]] = f.get("default") if schema["type"] == "array": for n, item in enumerate(datum): b2 = None if binding: b2 = copy.deepcopy(binding) b2["datum"] = item itemschema = { u"type": schema["items"], u"inputBinding": b2 } for k in ("secondaryFiles", "format", "streamable"): if k in schema: itemschema[k] = schema[k] bindings.extend( self.bind_input(itemschema, item, lead_pos=n, tail_pos=tail_pos)) binding = None if schema["type"] == "File": self.files.append(datum) if binding and binding.get("loadContents"): with self.fs_access.open(datum["location"], "rb") as f: datum["contents"] = f.read(CONTENT_LIMIT) if "secondaryFiles" in schema: if "secondaryFiles" not in datum: datum["secondaryFiles"] = [] for sf in aslist(schema["secondaryFiles"]): if isinstance(sf, dict) or "$(" in sf or "${" in sf: secondary_eval = self.do_eval(sf, context=datum) if isinstance(secondary_eval, basestring): sfpath = {"location": secondary_eval, "class": "File"} else: sfpath = secondary_eval else: sfpath = {"location": substitute(datum["location"], sf), "class": "File"} if isinstance(sfpath, list): datum["secondaryFiles"].extend(sfpath) else: datum["secondaryFiles"].append(sfpath) normalizeFilesDirs(datum["secondaryFiles"]) def _capture_files(f): self.files.append(f) return f adjustFileObjs(datum.get("secondaryFiles", []), _capture_files) if schema["type"] == "Directory": self.files.append(datum) # Position to front of the sort key if binding: for bi in bindings: bi["position"] = binding["position"] + bi["position"] bindings.append(binding) return bindings def tostr(self, value): # type: (Any) -> Text if isinstance(value, dict) and value.get("class") in ("File", "Directory"): if "path" not in value: raise WorkflowException(u"%s object missing \"path\": %s" % (value["class"], value)) return value["path"] else: return Text(value) def generate_arg(self, binding): # type: (Dict[Text,Any]) -> List[Text] value = binding.get("datum") if "valueFrom" in binding: with SourceLine(binding, "valueFrom", WorkflowException): value = self.do_eval(binding["valueFrom"], context=value) prefix = binding.get("prefix") sep = binding.get("separate", True) l = [] # type: List[Dict[Text,Text]] if isinstance(value, list): if binding.get("itemSeparator"): l = [binding["itemSeparator"].join([self.tostr(v) for v in value])] elif binding.get("valueFrom"): value = [self.tostr(v) for v in value] return ([prefix] if prefix else []) + value elif prefix: return [prefix] else: return [] elif isinstance(value, dict) and value.get("class") in ("File", "Directory"): l = [value] elif isinstance(value, dict): return [prefix] if prefix else [] elif value is True and prefix: return [prefix] elif value is False or value is None: return [] else: l = [value] args = [] for j in l: if sep: args.extend([prefix, self.tostr(j)]) else: args.append(prefix + self.tostr(j)) return [a for a in args if a is not None] def do_eval(self, ex, context=None, pull_image=True, recursive=False): # type: (Union[Dict[Text, Text], Text], Any, bool, bool) -> Any if recursive: if isinstance(ex, dict): return {k: self.do_eval(v, context, pull_image, recursive) for k,v in ex.iteritems()} if isinstance(ex, list): return [self.do_eval(v, context, pull_image, recursive) for v in ex] return expression.do_eval(ex, self.job, self.requirements, self.outdir, self.tmpdir, self.resources, context=context, pull_image=pull_image, timeout=self.timeout, debug=self.debug) cwltool-1.0.20170114120503/cwltool/update.py0000644000175000017500000004641713036412053020764 0ustar michaelmichael00000000000000import sys import urlparse import json import re import traceback import copy from schema_salad.ref_resolver import Loader import schema_salad.validate from typing import Any, Callable, Dict, List, Text, Tuple, Union # pylint: disable=unused-import from ruamel.yaml.comments import CommentedSeq, CommentedMap from .utils import aslist def findId(doc, frg): # type: (Any, Any) -> Dict if isinstance(doc, dict): if "id" in doc and doc["id"] == frg: return doc else: for d in doc: f = findId(doc[d], frg) if f: return f if isinstance(doc, list): for d in doc: f = findId(d, frg) if f: return f return None def fixType(doc): # type: (Any) -> Any if isinstance(doc, list): for i, f in enumerate(doc): doc[i] = fixType(f) return doc if isinstance(doc, (str, Text)): if doc not in ( "null", "boolean", "int", "long", "float", "double", "string", "File", "record", "enum", "array", "Any") and "#" not in doc: return "#" + doc return doc def _draft2toDraft3dev1(doc, loader, baseuri, update_steps=True): # type: (Any, Loader, Text, bool) -> Any try: if isinstance(doc, dict): if "import" in doc: imp = urlparse.urljoin(baseuri, doc["import"]) impLoaded = loader.fetch(imp) r = None # type: Dict[Text, Any] if isinstance(impLoaded, list): r = {"@graph": impLoaded} elif isinstance(impLoaded, dict): r = impLoaded else: raise Exception("Unexpected code path.") r["id"] = imp _, frag = urlparse.urldefrag(imp) if frag: frag = "#" + frag r = findId(r, frag) return _draft2toDraft3dev1(r, loader, imp) if "include" in doc: return loader.fetch_text(urlparse.urljoin(baseuri, doc["include"])) for typename in ("type", "items"): if typename in doc: doc[typename] = fixType(doc[typename]) if "steps" in doc and update_steps: if not isinstance(doc["steps"], list): raise Exception("Value of 'steps' must be a list") for i, s in enumerate(doc["steps"]): if "id" not in s: s["id"] = "step%i" % i for inp in s.get("inputs", []): if isinstance(inp.get("source"), list): if "requirements" not in doc: doc["requirements"] = [] doc["requirements"].append({"class": "MultipleInputFeatureRequirement"}) for a in doc: doc[a] = _draft2toDraft3dev1(doc[a], loader, baseuri) if isinstance(doc, list): for i, a in enumerate(doc): doc[i] = _draft2toDraft3dev1(a, loader, baseuri) return doc except Exception as e: err = json.dumps(doc, indent=4) if "id" in doc: err = doc["id"] elif "name" in doc: err = doc["name"] raise Exception(u"Error updating '%s'\n %s\n%s" % (err, e, traceback.format_exc())) def draft2toDraft3dev1(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] return (_draft2toDraft3dev1(doc, loader, baseuri), "draft-3.dev1") digits = re.compile("\d+") def updateScript(sc): # type: (Text) -> Text sc = sc.replace("$job", "inputs") sc = sc.replace("$tmpdir", "runtime.tmpdir") sc = sc.replace("$outdir", "runtime.outdir") sc = sc.replace("$self", "self") return sc def _updateDev2Script(ent): # type: (Any) -> Any if isinstance(ent, dict) and "engine" in ent: if ent["engine"] == "https://w3id.org/cwl/cwl#JsonPointer": sp = ent["script"].split("/") if sp[0] in ("tmpdir", "outdir"): return u"$(runtime.%s)" % sp[0] else: if not sp[0]: sp.pop(0) front = sp.pop(0) sp = [Text(i) if digits.match(i) else "'"+i+"'" for i in sp] if front == "job": return u"$(inputs[%s])" % ']['.join(sp) elif front == "context": return u"$(self[%s])" % ']['.join(sp) else: sc = updateScript(ent["script"]) if sc[0] == "{": return "$" + sc else: return u"$(%s)" % sc else: return ent def _draftDraft3dev1toDev2(doc, loader, baseuri): # type: (Any, Loader, Text) -> Any doc = _updateDev2Script(doc) if isinstance(doc, basestring): return doc # Convert expressions if isinstance(doc, dict): if "@import" in doc: resolved_doc = loader.resolve_ref( doc["@import"], base_url=baseuri)[0] if isinstance(resolved_doc, dict): return _draftDraft3dev1toDev2( resolved_doc, loader, resolved_doc["id"]) else: raise Exception("Unexpected codepath") for a in doc: doc[a] = _draftDraft3dev1toDev2(doc[a], loader, baseuri) if "class" in doc and (doc["class"] in ("CommandLineTool", "Workflow", "ExpressionTool")): added = False if "requirements" in doc: for r in doc["requirements"]: if r["class"] == "ExpressionEngineRequirement": if "engineConfig" in r: doc["requirements"].append({ "class":"InlineJavascriptRequirement", "expressionLib": [updateScript(sc) for sc in aslist(r["engineConfig"])] }) added = True for i, rq in enumerate(doc["requirements"]): if rq["class"] == "ExpressionEngineRequirement": del doc["requirements"][i] break break else: doc["requirements"] = [] if not added: doc["requirements"].append({"class":"InlineJavascriptRequirement"}) elif isinstance(doc, list): for i, a in enumerate(doc): doc[i] = _draftDraft3dev1toDev2(a, loader, baseuri) return doc def draftDraft3dev1toDev2(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] return (_draftDraft3dev1toDev2(doc, loader, baseuri), "draft-3.dev2") def _draftDraft3dev2toDev3(doc, loader, baseuri): # type: (Any, Loader, Text) -> Any try: if isinstance(doc, dict): if "@import" in doc: if doc["@import"][0] == "#": return doc["@import"] else: imp = urlparse.urljoin(baseuri, doc["@import"]) impLoaded = loader.fetch(imp) r = {} # type: Dict[Text, Any] if isinstance(impLoaded, list): r = {"@graph": impLoaded} elif isinstance(impLoaded, dict): r = impLoaded else: raise Exception("Unexpected code path.") r["id"] = imp frag = urlparse.urldefrag(imp)[1] if frag: frag = "#" + frag r = findId(r, frag) return _draftDraft3dev2toDev3(r, loader, imp) if "@include" in doc: return loader.fetch_text(urlparse.urljoin(baseuri, doc["@include"])) for a in doc: doc[a] = _draftDraft3dev2toDev3(doc[a], loader, baseuri) if isinstance(doc, list): for i, a in enumerate(doc): doc[i] = _draftDraft3dev2toDev3(a, loader, baseuri) return doc except Exception as e: err = json.dumps(doc, indent=4) if "id" in doc: err = doc["id"] elif "name" in doc: err = doc["name"] import traceback raise Exception(u"Error updating '%s'\n %s\n%s" % (err, e, traceback.format_exc())) def draftDraft3dev2toDev3(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] return (_draftDraft3dev2toDev3(doc, loader, baseuri), "draft-3.dev3") def traverseImport(doc, loader, baseuri, func): # type: (Any, Loader, Text, Callable[[Any, Loader, Text], Any]) -> Any if "$import" in doc: if doc["$import"][0] == "#": return doc["$import"] else: imp = urlparse.urljoin(baseuri, doc["$import"]) impLoaded = loader.fetch(imp) r = {} # type: Dict[Text, Any] if isinstance(impLoaded, list): r = {"$graph": impLoaded} elif isinstance(impLoaded, dict): r = impLoaded else: raise Exception("Unexpected code path.") r["id"] = imp _, frag = urlparse.urldefrag(imp) if frag: frag = "#" + frag r = findId(r, frag) return func(r, loader, imp) def _draftDraft3dev3toDev4(doc, loader, baseuri): # type: (Any, Loader, Text) -> Any try: if isinstance(doc, dict): r = traverseImport(doc, loader, baseuri, _draftDraft3dev3toDev4) if r is not None: return r if "@graph" in doc: doc["$graph"] = doc["@graph"] del doc["@graph"] for a in doc: doc[a] = _draftDraft3dev3toDev4(doc[a], loader, baseuri) if isinstance(doc, list): for i, a in enumerate(doc): doc[i] = _draftDraft3dev3toDev4(a, loader, baseuri) return doc except Exception as e: err = json.dumps(doc, indent=4) if "id" in doc: err = doc["id"] elif "name" in doc: err = doc["name"] import traceback raise Exception(u"Error updating '%s'\n %s\n%s" % (err, e, traceback.format_exc())) def draftDraft3dev3toDev4(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] return (_draftDraft3dev3toDev4(doc, loader, baseuri), "draft-3.dev4") def _draftDraft3dev4toDev5(doc, loader, baseuri): # type: (Any, Loader, Text) -> Any try: if isinstance(doc, dict): r = traverseImport(doc, loader, baseuri, _draftDraft3dev4toDev5) if r is not None: return r for b in ("inputBinding", "outputBinding"): if b in doc and "secondaryFiles" in doc[b]: doc["secondaryFiles"] = doc[b]["secondaryFiles"] del doc[b]["secondaryFiles"] for a in doc: doc[a] = _draftDraft3dev4toDev5(doc[a], loader, baseuri) if isinstance(doc, list): for i, a in enumerate(doc): doc[i] = _draftDraft3dev4toDev5(a, loader, baseuri) return doc except Exception as e: err = json.dumps(doc, indent=4) if "id" in doc: err = doc["id"] elif "name" in doc: err = doc["name"] raise Exception(u"Error updating '%s'\n %s\n%s" % (err, e, traceback.format_exc())) def draftDraft3dev4toDev5(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] return (_draftDraft3dev4toDev5(doc, loader, baseuri), "draft-3.dev5") def draftDraft3dev5toFinal(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] return (doc, "draft-3") def _draft3toDraft4dev1(doc, loader, baseuri): # type: (Any, Loader, Text) -> Any if isinstance(doc, dict): if "class" in doc and doc["class"] == "Workflow": def fixup(f): # type: (Text) -> Text doc, frg = urlparse.urldefrag(f) frg = '/'.join(frg.rsplit('.', 1)) return doc + "#" + frg for step in doc["steps"]: step["in"] = step["inputs"] step["out"] = step["outputs"] del step["inputs"] del step["outputs"] for io in ("in", "out"): for i in step[io]: i["id"] = fixup(i["id"]) if "source" in i: i["source"] = [fixup(s) for s in aslist(i["source"])] if len(i["source"]) == 1: i["source"] = i["source"][0] if "scatter" in step: step["scatter"] = [fixup(s) for s in aslist(step["scatter"])] for out in doc["outputs"]: out["source"] = fixup(out["source"]) for key, value in doc.items(): doc[key] = _draft3toDraft4dev1(value, loader, baseuri) elif isinstance(doc, list): for i, a in enumerate(doc): doc[i] = _draft3toDraft4dev1(a, loader, baseuri) return doc def draft3toDraft4dev1(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] """Public updater for draft-3 to draft-4.dev1.""" return (_draft3toDraft4dev1(doc, loader, baseuri), "draft-4.dev1") def _draft4Dev1toDev2(doc, loader, baseuri): # type: (Any, Loader, Text) -> Any if isinstance(doc, dict): if "class" in doc and doc["class"] == "Workflow": for out in doc["outputs"]: out["outputSource"] = out["source"] del out["source"] for key, value in doc.items(): doc[key] = _draft4Dev1toDev2(value, loader, baseuri) elif isinstance(doc, list): for i, a in enumerate(doc): doc[i] = _draft4Dev1toDev2(a, loader, baseuri) return doc def draft4Dev1toDev2(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] """Public updater for draft-4.dev1 to draft-4.dev2.""" return (_draft4Dev1toDev2(doc, loader, baseuri), "draft-4.dev2") def _draft4Dev2toDev3(doc, loader, baseuri): # type: (Any, Loader, Text) -> Any if isinstance(doc, dict): if "class" in doc and doc["class"] == "File": doc["location"] = doc["path"] del doc["path"] if "secondaryFiles" in doc: for i, sf in enumerate(doc["secondaryFiles"]): if "$(" in sf or "${" in sf: doc["secondaryFiles"][i] = sf.replace('"path"', '"location"').replace(".path", ".location") if "class" in doc and doc["class"] == "CreateFileRequirement": doc["class"] = "InitialWorkDirRequirement" doc["listing"] = [] for f in doc["fileDef"]: doc["listing"].append({ "entryname": f["filename"], "entry": f["fileContent"] }) del doc["fileDef"] for key, value in doc.items(): doc[key] = _draft4Dev2toDev3(value, loader, baseuri) elif isinstance(doc, list): for i, a in enumerate(doc): doc[i] = _draft4Dev2toDev3(a, loader, baseuri) return doc def draft4Dev2toDev3(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] """Public updater for draft-4.dev2 to draft-4.dev3.""" return (_draft4Dev2toDev3(doc, loader, baseuri), "draft-4.dev3") def _draft4Dev3to1_0dev4(doc, loader, baseuri): # type: (Any, Loader, Text) -> Any if isinstance(doc, dict): if "description" in doc: doc["doc"] = doc["description"] del doc["description"] for key, value in doc.items(): doc[key] = _draft4Dev3to1_0dev4(value, loader, baseuri) elif isinstance(doc, list): for i, a in enumerate(doc): doc[i] = _draft4Dev3to1_0dev4(a, loader, baseuri) return doc def draft4Dev3to1_0dev4(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] """Public updater for draft-4.dev3 to v1.0.dev4.""" return (_draft4Dev3to1_0dev4(doc, loader, baseuri), "v1.0.dev4") def v1_0dev4to1_0(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] """Public updater for v1.0.dev4 to v1.0.""" return (doc, "v1.0") def v1_0to1_1_0dev1(doc, loader, baseuri): # type: (Any, Loader, Text) -> Tuple[Any, Text] """Public updater for v1.0 to v1.1.0-dev1.""" return (doc, "v1.1.0-dev1") UPDATES = { "draft-2": draft2toDraft3dev1, "draft-3": draft3toDraft4dev1, "v1.0": None } # type: Dict[Text, Callable[[Any, Loader, Text], Tuple[Any, Text]]] DEVUPDATES = { "draft-3.dev1": draftDraft3dev1toDev2, "draft-3.dev2": draftDraft3dev2toDev3, "draft-3.dev3": draftDraft3dev3toDev4, "draft-3.dev4": draftDraft3dev4toDev5, "draft-3.dev5": draftDraft3dev5toFinal, "draft-4.dev1": draft4Dev1toDev2, "draft-4.dev2": draft4Dev2toDev3, "draft-4.dev3": draft4Dev3to1_0dev4, "v1.0.dev4": v1_0dev4to1_0, "v1.0": v1_0to1_1_0dev1, "v1.1.0-dev1": None } # type: Dict[Text, Callable[[Any, Loader, Text], Tuple[Any, Text]]] ALLUPDATES = UPDATES.copy() ALLUPDATES.update(DEVUPDATES) LATEST = "v1.0" def identity(doc, loader, baseuri): # pylint: disable=unused-argument # type: (Any, Loader, Text) -> Tuple[Any, Union[Text, Text]] """The default, do-nothing, CWL document upgrade function.""" return (doc, doc["cwlVersion"]) def checkversion(doc, metadata, enable_dev): # type: (Union[CommentedSeq, CommentedMap], CommentedMap, bool) -> Tuple[Dict[Text, Any], Text] # pylint: disable=line-too-long """Checks the validity of the version of the give CWL document. Returns the document and the validated version string. """ cdoc = None # type: CommentedMap if isinstance(doc, CommentedSeq): lc = metadata.lc metadata = copy.copy(metadata) metadata.lc.data = copy.copy(lc.data) metadata.lc.filename = lc.filename metadata[u"$graph"] = doc cdoc = metadata elif isinstance(doc, CommentedMap): cdoc = doc else: raise Exception("Expected CommentedMap or CommentedSeq") version = cdoc[u"cwlVersion"] if version not in UPDATES: if version in DEVUPDATES: if enable_dev: pass else: raise schema_salad.validate.ValidationException( u"Version '%s' is a development or deprecated version.\n " "Update your document to a stable version (%s) or use " "--enable-dev to enable support for development and " "deprecated versions." % (version, ", ".join( UPDATES.keys()))) else: raise schema_salad.validate.ValidationException( u"Unrecognized version %s" % version) return (cdoc, version) def update(doc, loader, baseuri, enable_dev, metadata): # type: (Union[CommentedSeq, CommentedMap], Loader, Text, bool, Any) -> dict (cdoc, version) = checkversion(doc, metadata, enable_dev) nextupdate = identity # type: Callable[[Any, Loader, Text], Tuple[Any, Text]] while nextupdate: (cdoc, version) = nextupdate(cdoc, loader, baseuri) nextupdate = ALLUPDATES[version] cdoc[u"cwlVersion"] = version return cdoc cwltool-1.0.20170114120503/cwltool/factory.py0000644000175000017500000000312213036412052021132 0ustar michaelmichael00000000000000from . import main from . import load_tool from . import workflow import os from .process import Process from typing import Any, Text, Union, Tuple from typing import Callable as tCallable import argparse class WorkflowStatus(Exception): def __init__(self, out, status): # type: (Dict[Text,Any], Text) -> None super(WorkflowStatus, self).__init__("Completed %s" % status) self.out = out self.status = status class Callable(object): def __init__(self, t, factory): # type: (Process, Factory) -> None self.t = t self.factory = factory def __call__(self, **kwargs): # type: (**Any) -> Union[Text, Dict[Text, Text]] execkwargs = self.factory.execkwargs.copy() execkwargs["basedir"] = os.getcwd() out, status = self.factory.executor(self.t, kwargs, **execkwargs) if status != "success": raise WorkflowStatus(out, status) else: return out class Factory(object): def __init__(self, makeTool=workflow.defaultMakeTool, executor=main.single_job_executor, **execkwargs): # type: (tCallable[[Dict[Text, Any], Any], Process],tCallable[...,Tuple[Dict[Text,Any], Text]], **Any) -> None self.makeTool = makeTool self.executor = executor self.execkwargs = execkwargs def make(self, cwl): """Instantiate a CWL object from a CWl document.""" load = load_tool.load_tool(cwl, self.makeTool) if isinstance(load, int): raise Exception("Error loading tool") return Callable(load, self) cwltool-1.0.20170114120503/cwltool/errors.py0000644000175000017500000000015013036412052020775 0ustar michaelmichael00000000000000class WorkflowException(Exception): pass class UnsupportedRequirement(WorkflowException): pass cwltool-1.0.20170114120503/cwltool/expression.py0000644000175000017500000001577613036412052021704 0ustar michaelmichael00000000000000import subprocess import json import logging import os import re import copy from typing import Any, AnyStr, Union, Text, Dict, List import schema_salad.validate as validate import schema_salad.ref_resolver from .utils import aslist, get_feature from .errors import WorkflowException from . import sandboxjs from . import docker _logger = logging.getLogger("cwltool") def jshead(engineConfig, rootvars): # type: (List[Text], Dict[Text, Any]) -> Text return u"\n".join(engineConfig + [u"var %s = %s;" % (k, json.dumps(v, indent=4)) for k, v in rootvars.items()]) seg_symbol = r"""\w+""" seg_single = r"""\['([^']|\\')+'\]""" seg_double = r"""\["([^"]|\\")+"\]""" seg_index = r"""\[[0-9]+\]""" segments = r"(\.%s|%s|%s|%s)" % (seg_symbol, seg_single, seg_double, seg_index) segment_re = re.compile(segments, flags=re.UNICODE) param_re = re.compile(r"\((%s)%s*\)$" % (seg_symbol, segments), flags=re.UNICODE) JSON = Union[Dict[Any,Any], List[Any], Text, int, long, float, bool, None] class SubstitutionError(Exception): pass def scanner(scan): # type: (Text) -> List[int] DEFAULT = 0 DOLLAR = 1 PAREN = 2 BRACE = 3 SINGLE_QUOTE = 4 DOUBLE_QUOTE = 5 BACKSLASH = 6 i = 0 stack = [DEFAULT] start = 0 while i < len(scan): state = stack[-1] c = scan[i] if state == DEFAULT: if c == '$': stack.append(DOLLAR) elif c == '\\': stack.append(BACKSLASH) elif state == BACKSLASH: stack.pop() if stack[-1] == DEFAULT: return [i-1, i+1] elif state == DOLLAR: if c == '(': start = i-1 stack.append(PAREN) elif c == '{': start = i-1 stack.append(BRACE) else: stack.pop() elif state == PAREN: if c == '(': stack.append(PAREN) elif c == ')': stack.pop() if stack[-1] == DOLLAR: return [start, i+1] elif c == "'": stack.append(SINGLE_QUOTE) elif c == '"': stack.append(DOUBLE_QUOTE) elif state == BRACE: if c == '{': stack.append(BRACE) elif c == '}': stack.pop() if stack[-1] == DOLLAR: return [start, i+1] elif c == "'": stack.append(SINGLE_QUOTE) elif c == '"': stack.append(DOUBLE_QUOTE) elif state == SINGLE_QUOTE: if c == "'": stack.pop() elif c == '\\': stack.append(BACKSLASH) elif state == DOUBLE_QUOTE: if c == '"': stack.pop() elif c == '\\': stack.append(BACKSLASH) i += 1 if len(stack) > 1: raise SubstitutionError("Substitution error, unfinished block starting at position {}: {}".format(start, scan[start:])) else: return None def next_seg(remain, obj): # type: (Text, Any) -> Any if remain: m = segment_re.match(remain) key = None # type: Union[str, int] if m.group(0)[0] == '.': key = m.group(0)[1:] elif m.group(0)[1] in ("'", '"'): key = m.group(0)[2:-2].replace("\\'", "'").replace('\\"', '"') if key: if isinstance(obj, list) and key == "length" and not remain[m.end(0):]: return len(obj) if not isinstance(obj, dict): raise WorkflowException(" is a %s, cannot index on string '%s'" % (type(obj).__name__, key)) if key not in obj: raise WorkflowException(" does not contain key '%s'" % key) else: try: key = int(m.group(0)[1:-1]) except ValueError as v: raise WorkflowException(unicode(v)) if not isinstance(obj, list): raise WorkflowException(" is a %s, cannot index on int '%s'" % (type(obj).__name__, key)) if key >= len(obj): raise WorkflowException(" list index %i out of range" % key) try: return next_seg(remain[m.end(0):], obj[key]) except WorkflowException as w: raise WorkflowException("%s%s" % (m.group(0), w)) else: return obj def evaluator(ex, jslib, obj, fullJS=False, timeout=None, debug=False): # type: (Text, Text, Dict[Text, Any], bool, int, bool) -> JSON m = param_re.match(ex) if m: try: return next_seg(m.group(0)[m.end(1) - m.start(0):-1], obj[m.group(1)]) except Exception as w: raise WorkflowException("%s%s" % (m.group(1), w)) elif fullJS: return sandboxjs.execjs(ex, jslib, timeout=timeout, debug=debug) else: raise sandboxjs.JavascriptException("Syntax error in parameter reference '%s' or used Javascript code without specifying InlineJavascriptRequirement.", ex) def interpolate(scan, rootvars, timeout=None, fullJS=None, jslib="", debug=False): # type: (Text, Dict[Text, Any], int, bool, Union[str, Text], bool) -> JSON scan = scan.strip() parts = [] w = scanner(scan) while w: parts.append(scan[0:w[0]]) if scan[w[0]] == '$': e = evaluator(scan[w[0]+1:w[1]], jslib, rootvars, fullJS=fullJS, timeout=timeout, debug=debug) if w[0] == 0 and w[1] == len(scan): return e leaf = json.dumps(e, sort_keys=True) if leaf[0] == '"': leaf = leaf[1:-1] parts.append(leaf) elif scan[w[0]] == '\\': e = scan[w[1]-1] parts.append(e) scan = scan[w[1]:] w = scanner(scan) parts.append(scan) return ''.join(parts) def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources, context=None, pull_image=True, timeout=None, debug=False): # type: (Union[dict, AnyStr], Dict[Text, Union[Dict, List, Text]], List[Dict[Text, Any]], Text, Text, Dict[Text, Union[int, Text]], Any, bool, int, bool) -> Any runtime = copy.copy(resources) runtime["tmpdir"] = tmpdir runtime["outdir"] = outdir rootvars = { u"inputs": jobinput, u"self": context, u"runtime": runtime } if isinstance(ex, (str, Text)): fullJS = False jslib = u"" for r in reversed(requirements): if r["class"] == "InlineJavascriptRequirement": fullJS = True jslib = jshead(r.get("expressionLib", []), rootvars) break try: return interpolate(ex, rootvars, timeout=timeout, fullJS=fullJS, jslib=jslib, debug=debug) except Exception as e: raise WorkflowException("Expression evaluation error:\n%s" % e) else: return ex cwltool-1.0.20170114120503/cwltool/stdfsaccess.py0000644000175000017500000000226713036412053022002 0ustar michaelmichael00000000000000from typing import Any, BinaryIO, Text from .pathmapper import abspath import glob import os from schema_salad.ref_resolver import file_uri class StdFsAccess(object): def __init__(self, basedir): # type: (Text) -> None self.basedir = basedir def _abs(self, p): # type: (Text) -> Text return abspath(p, self.basedir) def glob(self, pattern): # type: (Text) -> List[Text] return [file_uri(str(self._abs(l))) for l in glob.glob(self._abs(pattern))] def open(self, fn, mode): # type: (Text, Text) -> BinaryIO return open(self._abs(fn), mode) def exists(self, fn): # type: (Text) -> bool return os.path.exists(self._abs(fn)) def isfile(self, fn): # type: (Text) -> bool return os.path.isfile(self._abs(fn)) def isdir(self, fn): # type: (Text) -> bool return os.path.isdir(self._abs(fn)) def listdir(self, fn): # type: (Text) -> List[Text] return [abspath(l, fn) for l in os.listdir(self._abs(fn))] def join(self, path, *paths): # type: (Text, *Text) -> Text return os.path.join(path, *paths) def realpath(self, path): # type: (Text) -> Text return os.path.realpath(path) cwltool-1.0.20170114120503/cwltool/job.py0000644000175000017500000003776513036412052020261 0ustar michaelmichael00000000000000import subprocess import io import os import tempfile import glob import json import logging import sys import requests from . import docker from .process import get_feature, empty_subtree, stageFiles from .errors import WorkflowException import shutil import stat import re import shellescape import string from .docker_uid import docker_vm_uid from .builder import Builder from typing import (Any, Callable, Union, Iterable, Mapping, MutableMapping, IO, cast, Text, Tuple) from .pathmapper import PathMapper import functools _logger = logging.getLogger("cwltool") needs_shell_quoting_re = re.compile(r"""(^$|[\s|&;()<>\'"$@])""") FORCE_SHELLED_POPEN = os.getenv("CWLTOOL_FORCE_SHELL_POPEN", "0") == "1" SHELL_COMMAND_TEMPLATE = """#!/bin/bash python "run_job.py" "job.json" """ PYTHON_RUN_SCRIPT = """ import json import sys import subprocess with open(sys.argv[1], "r") as f: popen_description = json.load(f) commands = popen_description["commands"] cwd = popen_description["cwd"] env = popen_description["env"] stdin_path = popen_description["stdin_path"] stdout_path = popen_description["stdout_path"] stderr_path = popen_description["stderr_path"] if stdin_path is not None: stdin = open(stdin_path, "rb") else: stdin = subprocess.PIPE if stdout_path is not None: stdout = open(stdout_path, "wb") else: stdout = sys.stderr if stderr_path is not None: stderr = open(stderr_path, "wb") else: stderr = sys.stderr sp = subprocess.Popen(commands, shell=False, close_fds=True, stdin=stdin, stdout=stdout, stderr=stderr, env=env, cwd=cwd) if sp.stdin: sp.stdin.close() rcode = sp.wait() if isinstance(stdin, file): stdin.close() if stdout is not sys.stderr: stdout.close() if stderr is not sys.stderr: stderr.close() sys.exit(rcode) """ def deref_links(outputs): # type: (Any) -> None if isinstance(outputs, dict): if outputs.get("class") == "File": st = os.lstat(outputs["path"]) if stat.S_ISLNK(st.st_mode): outputs["path"] = os.readlink(outputs["path"]) else: for v in outputs.values(): deref_links(v) if isinstance(outputs, list): for v in outputs: deref_links(v) class CommandLineJob(object): def __init__(self): # type: () -> None self.builder = None # type: Builder self.joborder = None # type: Dict[Text, Union[Dict[Text, Any], List, Text]] self.stdin = None # type: Text self.stderr = None # type: Text self.stdout = None # type: Text self.successCodes = None # type: Iterable[int] self.temporaryFailCodes = None # type: Iterable[int] self.permanentFailCodes = None # type: Iterable[int] self.requirements = None # type: List[Dict[Text, Text]] self.hints = None # type: Dict[Text,Text] self.name = None # type: Text self.command_line = None # type: List[Text] self.pathmapper = None # type: PathMapper self.collect_outputs = None # type: Union[Callable[[Any], Any], functools.partial[Any]] self.output_callback = None # type: Callable[[Any, Any], Any] self.outdir = None # type: Text self.tmpdir = None # type: Text self.environment = None # type: MutableMapping[Text, Text] self.generatefiles = None # type: Dict[Text, Union[List[Dict[Text, Text]], Dict[Text, Text], Text]] self.stagedir = None # type: Text def run(self, dry_run=False, pull_image=True, rm_container=True, rm_tmpdir=True, move_outputs="move", **kwargs): # type: (bool, bool, bool, bool, Text, **Any) -> Union[Tuple[Text, Dict[None, None]], None] if not os.path.exists(self.outdir): os.makedirs(self.outdir) #with open(os.path.join(outdir, "cwl.input.json"), "w") as fp: # json.dump(self.joborder, fp) runtime = [] # type: List[Text] (docker_req, docker_is_req) = get_feature(self, "DockerRequirement") for knownfile in self.pathmapper.files(): p = self.pathmapper.mapper(knownfile) if p.type == "File" and not os.path.isfile(p[0]): raise WorkflowException( u"Input file %s (at %s) not found or is not a regular " "file." % (knownfile, self.pathmapper.mapper(knownfile)[0])) img_id = None env = None # type: Union[MutableMapping[Text, Text], MutableMapping[str, str]] if docker_req and kwargs.get("use_container") is not False: env = os.environ img_id = docker.get_from_requirements(docker_req, docker_is_req, pull_image) elif kwargs.get("default_container", None) is not None: env = os.environ img_id = kwargs.get("default_container") if docker_is_req and img_id is None: raise WorkflowException("Docker is required for running this tool.") if img_id: runtime = ["docker", "run", "-i"] for src in self.pathmapper.files(): vol = self.pathmapper.mapper(src) if vol.type == "File": runtime.append(u"--volume=%s:%s:ro" % (vol.resolved, vol.target)) if vol.type == "CreateFile": createtmp = os.path.join(self.stagedir, os.path.basename(vol.target)) with open(createtmp, "w") as f: f.write(vol.resolved.encode("utf-8")) runtime.append(u"--volume=%s:%s:ro" % (createtmp, vol.target)) runtime.append(u"--volume=%s:%s:rw" % (os.path.realpath(self.outdir), self.builder.outdir)) runtime.append(u"--volume=%s:%s:rw" % (os.path.realpath(self.tmpdir), "/tmp")) runtime.append(u"--workdir=%s" % (self.builder.outdir)) runtime.append("--read-only=true") if kwargs.get("custom_net", None) is not None: runtime.append("--net={0}".format(kwargs.get("custom_net"))) elif kwargs.get("disable_net", None): runtime.append("--net=none") if self.stdout: runtime.append("--log-driver=none") euid = docker_vm_uid() or os.geteuid() if kwargs.get("no_match_user",None) is False: runtime.append(u"--user=%s" % (euid)) if rm_container: runtime.append("--rm") runtime.append("--env=TMPDIR=/tmp") # spec currently says "HOME must be set to the designated output # directory." but spec might change to designated temp directory. # runtime.append("--env=HOME=/tmp") runtime.append("--env=HOME=%s" % self.builder.outdir) for t,v in self.environment.items(): runtime.append(u"--env=%s=%s" % (t, v)) runtime.append(img_id) else: env = self.environment if not os.path.exists(self.tmpdir): os.makedirs(self.tmpdir) vars_to_preserve = kwargs.get("preserve_environment") if kwargs.get("preserve_entire_environment"): vars_to_preserve = os.environ if vars_to_preserve is not None: for key, value in os.environ.items(): if key in vars_to_preserve and key not in env: env[key] = value env["HOME"] = self.outdir env["TMPDIR"] = self.tmpdir stageFiles(self.pathmapper, os.symlink) scr, _ = get_feature(self, "ShellCommandRequirement") if scr: shouldquote = lambda x: False else: shouldquote = needs_shell_quoting_re.search _logger.info(u"[job %s] %s$ %s%s%s%s", self.name, self.outdir, " \\\n ".join([shellescape.quote(Text(arg)) if shouldquote(Text(arg)) else Text(arg) for arg in (runtime + self.command_line)]), u' < %s' % self.stdin if self.stdin else '', u' > %s' % os.path.join(self.outdir, self.stdout) if self.stdout else '', u' 2> %s' % os.path.join(self.outdir, self.stderr) if self.stderr else '') if dry_run: return (self.outdir, {}) outputs = {} # type: Dict[Text,Text] try: if self.generatefiles["listing"]: generatemapper = PathMapper([self.generatefiles], self.outdir, self.outdir, separateDirs=False) _logger.debug(u"[job %s] initial work dir %s", self.name, json.dumps({p: generatemapper.mapper(p) for p in generatemapper.files()}, indent=4)) def linkoutdir(src, tgt): # Need to make the link to the staged file (may be inside # the container) for _, item in self.pathmapper.items(): if src == item.resolved: os.symlink(item.target, tgt) break stageFiles(generatemapper, linkoutdir) stdin_path = None if self.stdin: stdin_path = self.pathmapper.reversemap(self.stdin)[1] stderr_path = None if self.stderr: abserr = os.path.join(self.outdir, self.stderr) dnerr = os.path.dirname(abserr) if dnerr and not os.path.exists(dnerr): os.makedirs(dnerr) stderr_path = abserr stdout_path = None if self.stdout: absout = os.path.join(self.outdir, self.stdout) dn = os.path.dirname(absout) if dn and not os.path.exists(dn): os.makedirs(dn) stdout_path = absout build_job_script = self.builder.build_job_script # type: Callable[[List[str]], Text] rcode = _job_popen( [Text(x).encode('utf-8') for x in runtime + self.command_line], stdin_path=stdin_path, stdout_path=stdout_path, stderr_path=stderr_path, env=env, cwd=self.outdir, build_job_script=build_job_script, ) if self.successCodes and rcode in self.successCodes: processStatus = "success" elif self.temporaryFailCodes and rcode in self.temporaryFailCodes: processStatus = "temporaryFail" elif self.permanentFailCodes and rcode in self.permanentFailCodes: processStatus = "permanentFail" elif rcode == 0: processStatus = "success" else: processStatus = "permanentFail" if self.generatefiles["listing"]: def linkoutdir(src, tgt): # Need to make the link to the staged file (may be inside # the container) if os.path.islink(tgt): os.remove(tgt) os.symlink(src, tgt) stageFiles(generatemapper, linkoutdir, ignoreWritable=True) outputs = self.collect_outputs(self.outdir) except OSError as e: if e.errno == 2: if runtime: _logger.error(u"'%s' not found", runtime[0]) else: _logger.error(u"'%s' not found", self.command_line[0]) else: _logger.exception("Exception while running job") processStatus = "permanentFail" except WorkflowException as e: _logger.error(u"[job %s] Job error:\n%s" % (self.name, e)) processStatus = "permanentFail" except Exception as e: _logger.exception("Exception while running job") processStatus = "permanentFail" if processStatus != "success": _logger.warn(u"[job %s] completed %s", self.name, processStatus) else: _logger.info(u"[job %s] completed %s", self.name, processStatus) if _logger.isEnabledFor(logging.DEBUG): _logger.debug(u"[job %s] %s", self.name, json.dumps(outputs, indent=4)) self.output_callback(outputs, processStatus) if self.stagedir and os.path.exists(self.stagedir): _logger.debug(u"[job %s] Removing input staging directory %s", self.name, self.stagedir) shutil.rmtree(self.stagedir, True) if rm_tmpdir: _logger.debug(u"[job %s] Removing temporary directory %s", self.name, self.tmpdir) shutil.rmtree(self.tmpdir, True) if move_outputs == "move" and empty_subtree(self.outdir): _logger.debug(u"[job %s] Removing empty output directory %s", self.name, self.outdir) shutil.rmtree(self.outdir, True) def _job_popen( commands, # type: List[str] stdin_path, # type: Text stdout_path, # type: Text stderr_path, # type: Text env, # type: Union[MutableMapping[Text, Text], MutableMapping[str, str]] cwd, # type: Text job_dir=None, # type: Text build_job_script=None, # type: Callable[[List[str]], Text] ): # type: (...) -> int job_script_contents = None # type: Text if build_job_script: job_script_contents = build_job_script(commands) if not job_script_contents and not FORCE_SHELLED_POPEN: stdin = None # type: Union[IO[Any], int] stderr = None # type: IO[Any] stdout = None # type: IO[Any] if stdin_path is not None: stdin = open(stdin_path, "rb") else: stdin = subprocess.PIPE if stdout_path is not None: stdout = open(stdout_path, "wb") else: stdout = sys.stderr if stderr_path is not None: stderr = open(stderr_path, "wb") else: stderr = sys.stderr sp = subprocess.Popen(commands, shell=False, close_fds=True, stdin=stdin, stdout=stdout, stderr=stderr, env=env, cwd=cwd) if sp.stdin: sp.stdin.close() rcode = sp.wait() if isinstance(stdin, file): stdin.close() if stdout is not sys.stderr: stdout.close() if stderr is not sys.stderr: stderr.close() return rcode else: if job_dir is None: job_dir = tempfile.mkdtemp(prefix="cwltooljob") if not job_script_contents: job_script_contents = SHELL_COMMAND_TEMPLATE env_copy = {} for key in env: key = key.encode("utf-8") env_copy[key] = env[key] job_description = dict( commands=commands, cwd=cwd, env=env_copy, stdout_path=stdout_path, stderr_path=stderr_path, stdin_path=stdin_path, ) with open(os.path.join(job_dir, "job.json"), "w") as f: json.dump(job_description, f) try: job_script = os.path.join(job_dir, "run_job.bash") with open(job_script, "w") as f: f.write(job_script_contents) job_run = os.path.join(job_dir, "run_job.py") with open(job_run, "w") as f: f.write(PYTHON_RUN_SCRIPT) sp = subprocess.Popen( ["bash", job_script.encode("utf-8")], shell=False, cwd=job_dir, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, ) if sp.stdin: sp.stdin.close() rcode = sp.wait() return rcode finally: shutil.rmtree(job_dir) cwltool-1.0.20170114120503/cwltool/process.py0000644000175000017500000007050213036412052021147 0ustar michaelmichael00000000000000import os import json import copy import logging import pprint import stat import tempfile import glob import urlparse from collections import Iterable import errno import shutil import uuid import hashlib import abc import schema_salad.validate as validate import schema_salad.schema from schema_salad.ref_resolver import Loader, file_uri from schema_salad.sourceline import SourceLine import avro.schema from typing import (Any, AnyStr, Callable, cast, Dict, List, Generator, IO, Text, Tuple, Union) from rdflib import URIRef from rdflib.namespace import RDFS, OWL from rdflib import Graph from pkg_resources import resource_stream from ruamel.yaml.comments import CommentedSeq, CommentedMap from .utils import aslist, get_feature from .stdfsaccess import StdFsAccess from .builder import Builder, adjustFileObjs, adjustDirObjs from .errors import WorkflowException, UnsupportedRequirement from .pathmapper import PathMapper, abspath, normalizeFilesDirs _logger = logging.getLogger("cwltool") supportedProcessRequirements = ["DockerRequirement", "SchemaDefRequirement", "EnvVarRequirement", "ScatterFeatureRequirement", "SubworkflowFeatureRequirement", "MultipleInputFeatureRequirement", "InlineJavascriptRequirement", "ShellCommandRequirement", "StepInputExpressionRequirement", "ResourceRequirement", "InitialWorkDirRequirement"] cwl_files = ( "Workflow.yml", "CommandLineTool.yml", "CommonWorkflowLanguage.yml", "Process.yml", "concepts.md", "contrib.md", "intro.md", "invocation.md") salad_files = ('metaschema.yml', 'metaschema_base.yml', 'salad.md', 'field_name.yml', 'import_include.md', 'link_res.yml', 'ident_res.yml', 'vocab_res.yml', 'vocab_res.yml', 'field_name_schema.yml', 'field_name_src.yml', 'field_name_proc.yml', 'ident_res_schema.yml', 'ident_res_src.yml', 'ident_res_proc.yml', 'link_res_schema.yml', 'link_res_src.yml', 'link_res_proc.yml', 'vocab_res_schema.yml', 'vocab_res_src.yml', 'vocab_res_proc.yml') SCHEMA_CACHE = {} # type: Dict[Text, Tuple[Loader, Union[avro.schema.Names, avro.schema.SchemaParseException], Dict[Text, Any], Loader]] SCHEMA_FILE = None # type: Dict[Text, Any] SCHEMA_DIR = None # type: Dict[Text, Any] SCHEMA_ANY = None # type: Dict[Text, Any] def get_schema(version): # type: (Text) -> Tuple[Loader, Union[avro.schema.Names, avro.schema.SchemaParseException], Dict[Text,Any], Loader] if version in SCHEMA_CACHE: return SCHEMA_CACHE[version] cache = {} version = version.split("#")[-1] if '.dev' in version: version = ".".join(version.split(".")[:-1]) for f in cwl_files: try: res = resource_stream(__name__, 'schemas/%s/%s' % (version, f)) cache["https://w3id.org/cwl/" + f] = res.read() res.close() except IOError: pass for f in salad_files: try: res = resource_stream( __name__, 'schemas/%s/salad/schema_salad/metaschema/%s' % (version, f)) cache["https://w3id.org/cwl/salad/schema_salad/metaschema/" + f] = res.read() res.close() except IOError: pass SCHEMA_CACHE[version] = schema_salad.schema.load_schema( "https://w3id.org/cwl/CommonWorkflowLanguage.yml", cache=cache) return SCHEMA_CACHE[version] def shortname(inputid): # type: (Text) -> Text d = urlparse.urlparse(inputid) if d.fragment: return d.fragment.split(u"/")[-1] else: return d.path.split(u"/")[-1] def checkRequirements(rec, supportedProcessRequirements): # type: (Any, Iterable[Any]) -> None if isinstance(rec, dict): if "requirements" in rec: for i, r in enumerate(rec["requirements"]): with SourceLine(rec["requirements"], i, UnsupportedRequirement): if r["class"] not in supportedProcessRequirements: raise UnsupportedRequirement(u"Unsupported requirement %s" % r["class"]) for d in rec: checkRequirements(rec[d], supportedProcessRequirements) if isinstance(rec, list): for d in rec: checkRequirements(d, supportedProcessRequirements) def adjustFilesWithSecondary(rec, op, primary=None): """Apply a mapping function to each File path in the object `rec`, propagating the primary file associated with a group of secondary files. """ if isinstance(rec, dict): if rec.get("class") == "File": rec["path"] = op(rec["path"], primary=primary) adjustFilesWithSecondary(rec.get("secondaryFiles", []), op, primary if primary else rec["path"]) else: for d in rec: adjustFilesWithSecondary(rec[d], op) if isinstance(rec, list): for d in rec: adjustFilesWithSecondary(d, op, primary) def getListing(fs_access, rec): # type: (StdFsAccess, Dict[Text, Any]) -> None if "listing" not in rec: listing = [] loc = rec["location"] for ld in fs_access.listdir(loc): if fs_access.isdir(ld): ent = {u"class": u"Directory", u"location": ld} getListing(fs_access, ent) listing.append(ent) else: listing.append({"class": "File", "location": ld}) rec["listing"] = listing def stageFiles(pm, stageFunc, ignoreWritable=False): # type: (PathMapper, Callable[..., Any], bool) -> None for f, p in pm.items(): if not os.path.exists(os.path.dirname(p.target)): os.makedirs(os.path.dirname(p.target), 0755) if p.type == "File": stageFunc(p.resolved, p.target) elif p.type == "WritableFile" and not ignoreWritable: shutil.copy(p.resolved, p.target) elif p.type == "CreateFile" and not ignoreWritable: with open(p.target, "w") as n: n.write(p.resolved.encode("utf-8")) def collectFilesAndDirs(obj, out): # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]], List[Dict[Text, Any]]) -> None if isinstance(obj, dict): if obj.get("class") in ("File", "Directory"): out.append(obj) else: for v in obj.values(): collectFilesAndDirs(v, out) if isinstance(obj, list): for l in obj: collectFilesAndDirs(l, out) def relocateOutputs(outputObj, outdir, output_dirs, action): # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]], Text, Set[Text], Text) -> Union[Dict[Text, Any], List[Dict[Text, Any]]] if action not in ("move", "copy"): return outputObj def moveIt(src, dst): if action == "move": for a in output_dirs: if src.startswith(a): _logger.debug("Moving %s to %s", src, dst) shutil.move(src, dst) return _logger.debug("Copying %s to %s", src, dst) shutil.copy(src, dst) outfiles = [] # type: List[Dict[Text, Any]] collectFilesAndDirs(outputObj, outfiles) pm = PathMapper(outfiles, "", outdir, separateDirs=False) stageFiles(pm, moveIt) def _check_adjust(f): f["location"] = file_uri(pm.mapper(f["location"])[1]) if "contents" in f: del f["contents"] if f["class"] == "File": compute_checksums(StdFsAccess(""), f) return f adjustFileObjs(outputObj, _check_adjust) adjustDirObjs(outputObj, _check_adjust) return outputObj def cleanIntermediate(output_dirs): # type: (Set[Text]) -> None for a in output_dirs: if os.path.exists(a) and empty_subtree(a): _logger.debug(u"Removing intermediate output directory %s", a) shutil.rmtree(a, True) def formatSubclassOf(fmt, cls, ontology, visited): # type: (Text, Text, Graph, Set[Text]) -> bool """Determine if `fmt` is a subclass of `cls`.""" if URIRef(fmt) == URIRef(cls): return True if ontology is None: return False if fmt in visited: return False visited.add(fmt) uriRefFmt = URIRef(fmt) for s,p,o in ontology.triples( (uriRefFmt, RDFS.subClassOf, None) ): # Find parent classes of `fmt` and search upward if formatSubclassOf(o, cls, ontology, visited): return True for s,p,o in ontology.triples( (uriRefFmt, OWL.equivalentClass, None) ): # Find equivalent classes of `fmt` and search horizontally if formatSubclassOf(o, cls, ontology, visited): return True for s,p,o in ontology.triples( (None, OWL.equivalentClass, uriRefFmt) ): # Find equivalent classes of `fmt` and search horizontally if formatSubclassOf(s, cls, ontology, visited): return True return False def checkFormat(actualFile, inputFormats, ontology): # type: (Union[Dict[Text, Any], List, Text], Union[List[Text], Text], Graph) -> None for af in aslist(actualFile): if "format" not in af: raise validate.ValidationException(u"Missing required 'format' for File %s" % af) for inpf in aslist(inputFormats): if af["format"] == inpf or formatSubclassOf(af["format"], inpf, ontology, set()): return raise validate.ValidationException(u"Incompatible file format %s required format(s) %s" % (af["format"], inputFormats)) def fillInDefaults(inputs, job): # type: (List[Dict[Text, Text]], Dict[Text, Union[Dict[Text, Any], List, Text]]) -> None for e, inp in enumerate(inputs): with SourceLine(inputs, e, WorkflowException): if shortname(inp[u"id"]) in job: pass elif shortname(inp[u"id"]) not in job and u"default" in inp: job[shortname(inp[u"id"])] = copy.copy(inp[u"default"]) elif shortname(inp[u"id"]) not in job and aslist(inp[u"type"])[0] == u"null": pass else: raise WorkflowException("Missing required input parameter `%s`" % shortname(inp["id"])) def avroize_type(field_type, name_prefix=""): # type: (Union[List[Dict[Text, Any]], Dict[Text, Any]], Text) -> Any """ adds missing information to a type so that CWL types are valid in schema_salad. """ if isinstance(field_type, list): for f in field_type: avroize_type(f, name_prefix) elif isinstance(field_type, dict): if field_type["type"] in ("enum", "record"): if "name" not in field_type: field_type["name"] = name_prefix+Text(uuid.uuid4()) if field_type["type"] == "record": avroize_type(field_type["fields"], name_prefix) if field_type["type"] == "array": avroize_type(field_type["items"], name_prefix) return field_type class Process(object): __metaclass__ = abc.ABCMeta def __init__(self, toolpath_object, **kwargs): # type: (Dict[Text, Any], **Any) -> None """ kwargs: metadata: tool document metadata requirements: inherited requirements hints: inherited hints loader: schema_salad.ref_resolver.Loader used to load tool document avsc_names: CWL Avro schema object used to validate document strict: flag to determine strict validation (fail on unrecognized fields) """ self.metadata = kwargs.get("metadata", {}) # type: Dict[Text,Any] self.names = None # type: avro.schema.Names global SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY # pylint: disable=global-statement if SCHEMA_FILE is None: get_schema("v1.0") SCHEMA_ANY = cast(Dict[Text, Any], SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/salad#Any"]) SCHEMA_FILE = cast(Dict[Text, Any], SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/cwl#File"]) SCHEMA_DIR = cast(Dict[Text, Any], SCHEMA_CACHE["v1.0"][3].idx["https://w3id.org/cwl/cwl#Directory"]) names = schema_salad.schema.make_avro_schema([SCHEMA_FILE, SCHEMA_DIR, SCHEMA_ANY], schema_salad.ref_resolver.Loader({}))[0] if isinstance(names, avro.schema.SchemaParseException): raise names else: self.names = names self.tool = toolpath_object self.requirements = kwargs.get("requirements", []) + self.tool.get("requirements", []) self.hints = kwargs.get("hints", []) + self.tool.get("hints", []) self.formatgraph = None # type: Graph if "loader" in kwargs: self.formatgraph = kwargs["loader"].graph self.doc_loader = kwargs["loader"] self.doc_schema = kwargs["avsc_names"] checkRequirements(self.tool, supportedProcessRequirements) self.validate_hints(kwargs["avsc_names"], self.tool.get("hints", []), strict=kwargs.get("strict")) self.schemaDefs = {} # type: Dict[Text,Dict[Text, Any]] sd, _ = self.get_requirement("SchemaDefRequirement") if sd: sdtypes = sd["types"] av = schema_salad.schema.make_valid_avro(sdtypes, {t["name"]: t for t in avroize_type(sdtypes)}, set()) for i in av: self.schemaDefs[i["name"]] = i avro.schema.make_avsc_object(av, self.names) # Build record schema from inputs self.inputs_record_schema = { "name": "input_record_schema", "type": "record", "fields": []} # type: Dict[Text, Any] self.outputs_record_schema = { "name": "outputs_record_schema", "type": "record", "fields": []} # type: Dict[Text, Any] for key in ("inputs", "outputs"): for i in self.tool[key]: c = copy.copy(i) c["name"] = shortname(c["id"]) del c["id"] if "type" not in c: raise validate.ValidationException(u"Missing `type` in parameter `%s`" % c["name"]) if "default" in c and "null" not in aslist(c["type"]): c["type"] = ["null"] + aslist(c["type"]) else: c["type"] = c["type"] c["type"] = avroize_type(c["type"], c["name"]) if key == "inputs": self.inputs_record_schema["fields"].append(c) elif key == "outputs": self.outputs_record_schema["fields"].append(c) try: self.inputs_record_schema = schema_salad.schema.make_valid_avro(self.inputs_record_schema, {}, set()) avro.schema.make_avsc_object(self.inputs_record_schema, self.names) except avro.schema.SchemaParseException as e: raise validate.ValidationException(u"Got error `%s` while processing inputs of %s:\n%s" % (Text(e), self.tool["id"], json.dumps(self.inputs_record_schema, indent=4))) try: self.outputs_record_schema = schema_salad.schema.make_valid_avro(self.outputs_record_schema, {}, set()) avro.schema.make_avsc_object(self.outputs_record_schema, self.names) except avro.schema.SchemaParseException as e: raise validate.ValidationException(u"Got error `%s` while processing outputs of %s:\n%s" % (Text(e), self.tool["id"], json.dumps(self.outputs_record_schema, indent=4))) def _init_job(self, joborder, **kwargs): # type: (Dict[Text, Text], **Any) -> Builder """ kwargs: eval_timeout: javascript evaluation timeout use_container: do/don't use Docker when DockerRequirement hint provided make_fs_access: make an FsAccess() object with given basedir basedir: basedir for FsAccess docker_outdir: output directory inside docker for this job docker_tmpdir: tmpdir inside docker for this job docker_stagedir: stagedir inside docker for this job outdir: outdir on host for this job tmpdir: tmpdir on host for this job stagedir: stagedir on host for this job select_resources: callback to select compute resources """ builder = Builder() builder.job = cast(Dict[Text, Union[Dict[Text, Any], List, Text]], copy.deepcopy(joborder)) # Validate job order try: fillInDefaults(self.tool[u"inputs"], builder.job) normalizeFilesDirs(builder.job) validate.validate_ex(self.names.get_name("input_record_schema", ""), builder.job) except (validate.ValidationException, WorkflowException) as e: raise WorkflowException("Invalid job input record:\n" + Text(e)) builder.files = [] builder.bindings = CommentedSeq() builder.schemaDefs = self.schemaDefs builder.names = self.names builder.requirements = self.requirements builder.hints = self.hints builder.resources = {} builder.timeout = kwargs.get("eval_timeout") builder.debug = kwargs.get("debug") dockerReq, is_req = self.get_requirement("DockerRequirement") if dockerReq and is_req and not kwargs.get("use_container"): raise WorkflowException("Document has DockerRequirement under 'requirements' but use_container is false. DockerRequirement must be under 'hints' or use_container must be true.") builder.make_fs_access = kwargs.get("make_fs_access") or StdFsAccess builder.fs_access = builder.make_fs_access(kwargs["basedir"]) if dockerReq and kwargs.get("use_container"): builder.outdir = builder.fs_access.realpath(dockerReq.get("dockerOutputDirectory") or kwargs.get("docker_outdir") or "/var/spool/cwl") builder.tmpdir = builder.fs_access.realpath(kwargs.get("docker_tmpdir") or "/tmp") builder.stagedir = builder.fs_access.realpath(kwargs.get("docker_stagedir") or "/var/lib/cwl") else: builder.outdir = builder.fs_access.realpath(kwargs.get("outdir") or tempfile.mkdtemp()) builder.tmpdir = builder.fs_access.realpath(kwargs.get("tmpdir") or tempfile.mkdtemp()) builder.stagedir = builder.fs_access.realpath(kwargs.get("stagedir") or tempfile.mkdtemp()) if self.formatgraph: for i in self.tool["inputs"]: d = shortname(i["id"]) if d in builder.job and i.get("format"): checkFormat(builder.job[d], builder.do_eval(i["format"]), self.formatgraph) builder.bindings.extend(builder.bind_input(self.inputs_record_schema, builder.job)) if self.tool.get("baseCommand"): for n, b in enumerate(aslist(self.tool["baseCommand"])): builder.bindings.append({ "position": [-1000000, n], "datum": b }) if self.tool.get("arguments"): for i, a in enumerate(self.tool["arguments"]): lc = self.tool["arguments"].lc.data[i] fn = self.tool["arguments"].lc.filename builder.bindings.lc.add_kv_line_col(len(builder.bindings), lc) if isinstance(a, dict): a = copy.copy(a) if a.get("position"): a["position"] = [a["position"], i] else: a["position"] = [0, i] builder.bindings.append(a) elif ("$(" in a) or ("${" in a): cm = CommentedMap(( ("position", [0, i]), ("valueFrom", a) )) cm.lc.add_kv_line_col("valueFrom", lc) cm.lc.filename = fn builder.bindings.append(cm) else: cm = CommentedMap(( ("position", [0, i]), ("datum", a) )) cm.lc.add_kv_line_col("datum", lc) cm.lc.filename = fn builder.bindings.append(cm) builder.bindings.sort(key=lambda a: a["position"]) builder.resources = self.evalResources(builder, kwargs) return builder def evalResources(self, builder, kwargs): # type: (Builder, Dict[AnyStr, Any]) -> Dict[Text, Union[int, Text]] resourceReq, _ = self.get_requirement("ResourceRequirement") if resourceReq is None: resourceReq = {} request = { "coresMin": 1, "coresMax": 1, "ramMin": 1024, "ramMax": 1024, "tmpdirMin": 1024, "tmpdirMax": 1024, "outdirMin": 1024, "outdirMax": 1024 } for a in ("cores", "ram", "tmpdir", "outdir"): mn = None mx = None if resourceReq.get(a+"Min"): mn = builder.do_eval(resourceReq[a+"Min"]) if resourceReq.get(a+"Max"): mx = builder.do_eval(resourceReq[a+"Max"]) if mn is None: mn = mx elif mx is None: mx = mn if mn: request[a+"Min"] = mn request[a+"Max"] = mx if kwargs.get("select_resources"): return kwargs["select_resources"](request) else: return { "cores": request["coresMin"], "ram": request["ramMin"], "tmpdirSize": request["tmpdirMin"], "outdirSize": request["outdirMin"], } def validate_hints(self, avsc_names, hints, strict): # type: (Any, List[Dict[Text, Any]], bool) -> None for i, r in enumerate(hints): sl = SourceLine(hints, i, validate.ValidationException) with sl: if avsc_names.get_name(r["class"], "") is not None: plain_hint = dict((key,r[key]) for key in r if key not in self.doc_loader.identifiers) # strip identifiers validate.validate_ex( avsc_names.get_name(plain_hint["class"], ""), plain_hint, strict=strict) else: _logger.info(sl.makeError(u"Unknown hint %s" % (r["class"]))) def get_requirement(self, feature): # type: (Any) -> Tuple[Any, bool] return get_feature(self, feature) def visit(self, op): # type: (Callable[[Dict[Text, Any]], None]) -> None op(self.tool) @abc.abstractmethod def job(self, job_order, output_callbacks, **kwargs): # type: (Dict[Text, Text], Callable[[Any, Any], Any], **Any) -> Generator[Any, None, None] return None def empty_subtree(dirpath): # type: (Text) -> bool # Test if a directory tree contains any files (does not count empty # subdirectories) for d in os.listdir(dirpath): d = os.path.join(dirpath, d) try: if stat.S_ISDIR(os.stat(d).st_mode): if empty_subtree(d) is False: return False else: return False except OSError as e: if e.errno == errno.ENOENT: pass else: raise return True _names = set() # type: Set[Text] def uniquename(stem, names=None): # type: (Text, Set[Text]) -> Text global _names if names is None: names = _names c = 1 u = stem while u in names: c += 1 u = u"%s_%s" % (stem, c) names.add(u) return u def nestdir(base, deps): # type: (Text, Dict[Text, Any]) -> Dict[Text, Any] dirname = os.path.dirname(base) + "/" subid = deps["location"] if subid.startswith(dirname): s2 = subid[len(dirname):] sp = s2.split('/') sp.pop() while sp: nx = sp.pop() deps = { "class": "Directory", "basename": nx, "listing": [deps] } return deps def mergedirs(listing): # type: (List[Dict[Text, Any]]) -> List[Dict[Text, Any]] r = [] # type: List[Dict[Text, Any]] ents = {} # type: Dict[Text, Any] for e in listing: if e["basename"] not in ents: ents[e["basename"]] = e elif e["class"] == "Directory": ents[e["basename"]]["listing"].extend(e["listing"]) for e in ents.itervalues(): if e["class"] == "Directory" and "listing" in e: e["listing"] = mergedirs(e["listing"]) r.extend(ents.itervalues()) return r def scandeps(base, doc, reffields, urlfields, loadref, urljoin=urlparse.urljoin): # type: (Text, Any, Set[Text], Set[Text], Callable[[Text, Text], Any], Callable[[Text, Text], Text]) -> List[Dict[Text, Text]] r = [] # type: List[Dict[Text, Text]] deps = None # type: Dict[Text, Any] if isinstance(doc, dict): if "id" in doc: if doc["id"].startswith("file://"): df, _ = urlparse.urldefrag(doc["id"]) if base != df: r.append({ "class": "File", "location": df }) base = df if doc.get("class") in ("File", "Directory") and "location" in urlfields: u = doc.get("location", doc.get("path")) if u and not u.startswith("_:"): deps = { "class": doc["class"], "location": urljoin(base, u) } if doc["class"] == "Directory" and "listing" in doc: deps["listing"] = doc["listing"] if doc["class"] == "File" and "secondaryFiles" in doc: deps["secondaryFiles"] = doc["secondaryFiles"] deps = nestdir(base, deps) r.append(deps) else: if doc["class"] == "Directory" and "listing" in doc: r.extend(scandeps(base, doc["listing"], reffields, urlfields, loadref, urljoin=urljoin)) elif doc["class"] == "File" and "secondaryFiles" in doc: r.extend(scandeps(base, doc["secondaryFiles"], reffields, urlfields, loadref, urljoin=urljoin)) for k, v in doc.iteritems(): if k in reffields: for u in aslist(v): if isinstance(u, dict): r.extend(scandeps(base, u, reffields, urlfields, loadref, urljoin=urljoin)) else: sub = loadref(base, u) subid = urljoin(base, u) deps = { "class": "File", "location": subid } sf = scandeps(subid, sub, reffields, urlfields, loadref, urljoin=urljoin) if sf: deps["secondaryFiles"] = sf deps = nestdir(base, deps) r.append(deps) elif k in urlfields and k != "location": for u in aslist(v): deps = { "class": "File", "location": urljoin(base, u) } deps = nestdir(base, deps) r.append(deps) elif k not in ("listing", "secondaryFiles"): r.extend(scandeps(base, v, reffields, urlfields, loadref, urljoin=urljoin)) elif isinstance(doc, list): for d in doc: r.extend(scandeps(base, d, reffields, urlfields, loadref, urljoin=urljoin)) if r: normalizeFilesDirs(r) r = mergedirs(r) return r def compute_checksums(fs_access, fileobj): if "checksum" not in fileobj: checksum = hashlib.sha1() with fs_access.open(fileobj["location"], "rb") as f: contents = f.read(1024*1024) while contents != "": checksum.update(contents) contents = f.read(1024*1024) f.seek(0, 2) filesize = f.tell() fileobj["checksum"] = "sha1$%s" % checksum.hexdigest() fileobj["size"] = filesize cwltool-1.0.20170114120503/cwltool/load_tool.py0000644000175000017500000002463613036412052021454 0ustar michaelmichael00000000000000# pylint: disable=unused-import """Loads a CWL document.""" import os import uuid import logging import re import urlparse from typing import Any, AnyStr, Callable, cast, Dict, Text, Tuple, Union from ruamel.yaml.comments import CommentedSeq, CommentedMap from avro.schema import Names import requests.sessions from schema_salad.ref_resolver import Loader, Fetcher, file_uri import schema_salad.validate as validate from schema_salad.validate import ValidationException import schema_salad.schema as schema from schema_salad.sourceline import cmap from . import update from . import process from .process import Process, shortname from .errors import WorkflowException _logger = logging.getLogger("cwltool") def fetch_document(argsworkflow, # type: Union[Text, dict[Text, Any]] resolver=None, # type: Callable[[Loader, Union[Text, dict[Text, Any]]], Text] fetcher_constructor=None # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher] ): # type: (...) -> Tuple[Loader, CommentedMap, Text] """Retrieve a CWL document.""" document_loader = Loader({"cwl": "https://w3id.org/cwl/cwl#", "id": "@id"}, fetcher_constructor=fetcher_constructor) uri = None # type: Text workflowobj = None # type: CommentedMap if isinstance(argsworkflow, basestring): split = urlparse.urlsplit(argsworkflow) if split.scheme: uri = argsworkflow elif os.path.exists(os.path.abspath(argsworkflow)): uri = file_uri(str(os.path.abspath(argsworkflow))) elif resolver: uri = resolver(document_loader, argsworkflow) if uri is None: raise ValidationException("Not found: '%s'" % argsworkflow) if argsworkflow != uri: _logger.info("Resolved '%s' to '%s'", argsworkflow, uri) fileuri = urlparse.urldefrag(uri)[0] workflowobj = document_loader.fetch(fileuri) elif isinstance(argsworkflow, dict): uri = "#" + Text(id(argsworkflow)) workflowobj = cast(CommentedMap, cmap(argsworkflow, fn=uri)) else: raise ValidationException("Must be URI or object: '%s'" % argsworkflow) return document_loader, workflowobj, uri def _convert_stdstreams_to_files(workflowobj): # type: (Union[Dict[Text, Any], List[Dict[Text, Any]]]) -> None if isinstance(workflowobj, dict): if ('class' in workflowobj and workflowobj['class'] == 'CommandLineTool'): if 'outputs' in workflowobj: for out in workflowobj['outputs']: for streamtype in ['stdout', 'stderr']: if out['type'] == streamtype: if 'outputBinding' in out: raise ValidationException( "Not allowed to specify outputBinding when" " using %s shortcut." % streamtype) if streamtype in workflowobj: filename = workflowobj[streamtype] else: filename = Text(uuid.uuid4()) workflowobj[streamtype] = filename out['type'] = 'File' out['outputBinding'] = {'glob': filename} if 'inputs' in workflowobj: for inp in workflowobj['inputs']: if inp['type'] == 'stdin': if 'inputBinding' in inp: raise ValidationException( "Not allowed to specify inputBinding when" " using stdin shortcut.") if 'stdin' in workflowobj: raise ValidationException( "Not allowed to specify stdin path when" " using stdin type shortcut.") else: workflowobj['stdin'] = \ "$(inputs.%s.path)" % \ inp['id'].rpartition('#')[2] inp['type'] = 'File' else: for entry in workflowobj.itervalues(): _convert_stdstreams_to_files(entry) if isinstance(workflowobj, list): for entry in workflowobj: _convert_stdstreams_to_files(entry) def validate_document(document_loader, # type: Loader workflowobj, # type: CommentedMap uri, # type: Text enable_dev=False, # type: bool strict=True, # type: bool preprocess_only=False, # type: bool fetcher_constructor=None # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher] ): # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text] """Validate a CWL document.""" if isinstance(workflowobj, list): workflowobj = { "$graph": workflowobj } if not isinstance(workflowobj, dict): raise ValueError("workflowjobj must be a dict") jobobj = None if "cwl:tool" in workflowobj: jobobj, _ = document_loader.resolve_all(workflowobj, uri) uri = urlparse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1] fileuri = urlparse.urldefrag(uri)[0] if "cwlVersion" in workflowobj: if not isinstance(workflowobj["cwlVersion"], (str, Text)): raise Exception("'cwlVersion' must be a string, got %s" % type(workflowobj["cwlVersion"])) workflowobj["cwlVersion"] = re.sub( r"^(?:cwl:|https://w3id.org/cwl/cwl#)", "", workflowobj["cwlVersion"]) else: _logger.warn("No cwlVersion found, treating this file as draft-2.") workflowobj["cwlVersion"] = "draft-2" if workflowobj["cwlVersion"] == "draft-2": workflowobj = cast(CommentedMap, cmap(update._draft2toDraft3dev1( workflowobj, document_loader, uri, update_steps=False))) if "@graph" in workflowobj: workflowobj["$graph"] = workflowobj["@graph"] del workflowobj["@graph"] (sch_document_loader, avsc_names) = \ process.get_schema(workflowobj["cwlVersion"])[:2] if isinstance(avsc_names, Exception): raise avsc_names processobj = None # type: Union[CommentedMap, CommentedSeq, unicode] document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph, idx=document_loader.idx, cache=sch_document_loader.cache, fetcher_constructor=fetcher_constructor) workflowobj["id"] = fileuri processobj, metadata = document_loader.resolve_all(workflowobj, fileuri) if not isinstance(processobj, (CommentedMap, CommentedSeq)): raise ValidationException("Workflow must be a dict or list.") if not metadata: if not isinstance(processobj, dict): raise ValidationException("Draft-2 workflows must be a dict.") metadata = cast(CommentedMap, cmap({"$namespaces": processobj.get("$namespaces", {}), "$schemas": processobj.get("$schemas", []), "cwlVersion": processobj["cwlVersion"]}, fn=fileuri)) _convert_stdstreams_to_files(workflowobj) if preprocess_only: return document_loader, avsc_names, processobj, metadata, uri schema.validate_doc(avsc_names, processobj, document_loader, strict) if metadata.get("cwlVersion") != update.LATEST: processobj = cast(CommentedMap, cmap(update.update( processobj, document_loader, fileuri, enable_dev, metadata))) if jobobj: metadata[u"cwl:defaults"] = jobobj return document_loader, avsc_names, processobj, metadata, uri def make_tool(document_loader, # type: Loader avsc_names, # type: Names metadata, # type: Dict[Text, Any] uri, # type: Text makeTool, # type: Callable[..., Process] kwargs # type: dict ): # type: (...) -> Process """Make a Python CWL object.""" resolveduri = document_loader.resolve_ref(uri)[0] if isinstance(resolveduri, list): if len(resolveduri) == 1: processobj = resolveduri[0] else: raise WorkflowException( u"Tool file contains graph of multiple objects, must specify " "one of #%s" % ", #".join( urlparse.urldefrag(i["id"])[1] for i in resolveduri if "id" in i)) elif isinstance(resolveduri, dict): processobj = resolveduri else: raise Exception("Must resolve to list or dict") kwargs = kwargs.copy() kwargs.update({ "makeTool": makeTool, "loader": document_loader, "avsc_names": avsc_names, "metadata": metadata }) tool = makeTool(processobj, **kwargs) if "cwl:defaults" in metadata: jobobj = metadata["cwl:defaults"] for inp in tool.tool["inputs"]: if shortname(inp["id"]) in jobobj: inp["default"] = jobobj[shortname(inp["id"])] return tool def load_tool(argsworkflow, # type: Union[Text, Dict[Text, Any]] makeTool, # type: Callable[..., Process] kwargs=None, # type: dict enable_dev=False, # type: bool strict=True, # type: bool resolver=None, # type: Callable[[Loader, Union[Text, dict[Text, Any]]], Text] fetcher_constructor=None # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher] ): # type: (...) -> Process document_loader, workflowobj, uri = fetch_document(argsworkflow, resolver=resolver, fetcher_constructor=fetcher_constructor) document_loader, avsc_names, processobj, metadata, uri = validate_document( document_loader, workflowobj, uri, enable_dev=enable_dev, strict=strict, fetcher_constructor=fetcher_constructor) return make_tool(document_loader, avsc_names, metadata, uri, makeTool, kwargs if kwargs else {}) cwltool-1.0.20170114120503/Makefile0000644000175000017500000001221213036412052017066 0ustar michaelmichael00000000000000# This file is part of cwltool, # https://github.com/common-workflow-language/cwltool/, and is # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # Contact: common-workflow-language@googlegroups.com # make pep8 to check for basic Python code compliance # make autopep8 to fix most pep8 errors # make pylint to check Python code for enhanced compliance including naming # and documentation # make coverage-report to check coverage of the python scripts by the tests MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount python-flake8 VERSION=1.0.$(shell date +%Y%m%d%H%M%S --date=`git log --first-parent \ --max-count=1 --format=format:%cI`) mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) ## all : default task all: ./setup.py develop ## help : print this help message and exit help: Makefile @sed -n 's/^##//p' $< ## install-dep : install most of the development dependencies via pip install-dep: pip install --upgrade $(DEVPKGS) ## install-deb-dep: install most of the dev dependencies via apt-get install-deb-dep: sudo apt-get install $(DEBDEVPKGS) ## install : install the ${MODULE} module and schema-salad-tool install: FORCE ./setup.py build install ## dist : create a module package for distribution dist: dist/${MODULE}-$(VERSION).tar.gz dist/${MODULE}-$(VERSION).tar.gz: $(SOURCES) ./setup.py sdist ## clean : clean up all temporary / machine-generated files clean: FORCE rm -f ${MODILE}/*.pyc tests/*.pyc ./setup.py clean --all || true rm -Rf .coverage rm -f diff-cover.html ## pep8 : check Python code style pep8: $(PYSOURCES) pep8 --exclude=_version.py --show-source --show-pep8 $^ || true pep8_report.txt: $(PYSOURCES) pep8 --exclude=_version.py $^ > pep8_report.txt || true diff_pep8_report: pep8_report.txt diff-quality --violations=pep8 pep8_report.txt ## pep257 : check Python code style pep257: $(PYSOURCES) pep257 --ignore=D100,D101,D102,D103 $^ || true pep257_report.txt: $(PYSOURCES) pep257 setup.py $^ > pep257_report.txt 2>&1 || true diff_pep257_report: pep257_report.txt diff-quality --violations=pep8 pep257_report.txt ## autopep8 : fix most Python code indentation and formatting autopep8: $(PYSOURCES) autopep8 --recursive --in-place --ignore E309 $^ # A command to automatically run astyle and autopep8 on appropriate files ## format : check/fix all code indentation and formatting (runs autopep8) format: autopep8 # Do nothing ## pylint : run static code analysis on Python code pylint: $(PYSOURCES) pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \ $^ || true pylint_report.txt: ${PYSOURCES} pylint --msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \ $^ > pylint_report.txt || true diff_pylint_report: pylint_report.txt diff-quality --violations=pylint pylint_report.txt .coverage: $(PYSOURCES) all export COVERAGE_PROCESS_START=${mkfile_dir}.coveragerc; \ cd ${CWL}; ./run_test.sh RUNNER=cwltool coverage run setup.py test coverage combine ${CWL} ${CWL}/draft-3/ ./ coverage.xml: .coverage python-coverage xml coverage.html: htmlcov/index.html htmlcov/index.html: .coverage python-coverage html @echo Test coverage of the Python code is now in htmlcov/index.html coverage-report: .coverage python-coverage report diff-cover: coverage-gcovr.xml coverage.xml diff-cover coverage-gcovr.xml coverage.xml diff-cover.html: coverage-gcovr.xml coverage.xml diff-cover coverage-gcovr.xml coverage.xml \ --html-report diff-cover.html ## test : run the ${MODULE} test suite test: FORCE ./setup.py test sloccount.sc: ${PYSOURCES} Makefile sloccount --duplicates --wide --details $^ > sloccount.sc ## sloccount : count lines of code sloccount: ${PYSOURCES} Makefile sloccount $^ list-author-emails: @echo 'name, E-Mail Address' @git log --format='%aN,%aE' | sort -u | grep -v 'root' mypy: ${PYSOURCES} rm -Rf typeshed/2.7/ruamel/yaml ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \ typeshed/2.7/ruamel/yaml rm -Rf typeshed/2.7/schema_salad ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \ typeshed/2.7/schema_salad MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls \ --warn-redundant-casts --warn-unused-ignores --fast-parser \ cwltool FORCE: cwltool-1.0.20170114120503/PKG-INFO0000644000175000017500000001040513036412107016526 0ustar michaelmichael00000000000000Metadata-Version: 1.1 Name: cwltool Version: 1.0.20170114120503 Summary: Common workflow language reference implementation Home-page: https://github.com/common-workflow-language/cwltool Author: Common workflow language working group Author-email: common-workflow-language@googlegroups.com License: Apache 2.0 Download-URL: https://github.com/common-workflow-language/cwltool Description: ================================================================== Common workflow language tool description reference implementation ================================================================== CWL Conformance test: |Build Status| This is the reference implementation of the Common Workflow Language. It is intended to be feature complete and provide comprehensive validation of CWL files as well as provide other tools related to working with CWL. This is written and tested for Python 2.7. The reference implementation consists of two packages. The "cwltool" package is the primary Python module containing the reference implementation in the "cwltool" module and console executable by the same name. The "cwlref-runner" package is optional and provides an additional entry point under the alias "cwl-runner", which is the implementation-agnostic name for the default CWL interpreter installed on a host. Install ------- Installing the official package from PyPi (will install "cwltool" package as well):: pip install cwlref-runner If installling alongside another CWL implementation then:: pip install cwltool To install from source:: git clone https://github.com/common-workflow-language/cwltool.git cd cwltool && python setup.py install cd cwlref-runner && python setup.py install # co-installing? skip this Remember, if co-installing multiple CWL implementations then you need to maintain which implementation ``cwl-runner`` points to via a symbolic file system link or [another facility](https://wiki.debian.org/DebianAlternatives). Run on the command line ----------------------- Simple command:: cwl-runner [tool-or-workflow-description] [input-job-settings] Or if you have multiple CWL implementations installed and you want to override the default cwl-runner use:: cwltool [tool-or-workflow-description] [input-job-settings] Import as a module ---------------- Add:: import cwltool to your script. Use with boot2docker -------------------- boot2docker is running docker inside a virtual machine and it only mounts ``Users`` on it. The default behavoir of CWL is to create temporary directories under e.g. ``/Var`` which is not accessible to Docker containers. To run CWL successfully with boot2docker you need to set the ``--tmpdir-prefix`` and ``--tmp-outdir-prefix`` to somewhere under ``/Users``:: $ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json .. |Build Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance :target: https://ci.commonwl.org/job/cwltool-conformance/ Tool or workflow loading from remote or local locations ------------------------------------------------------- ``cwltool`` can run tool and workflow descriptions on both local and remote systems via its support for HTTP[S] URLs. Input job files and Workflow steps (via the `run` directive) can reference CWL documents using absolute or relative local filesytem paths. If a relative path is referenced and that document isn't found in the current directory then the following locations will be searched: http://www.commonwl.org/v1.0/CommandLineTool.html#Discovering_CWL_documents_on_a_local_filesystem Platform: UNKNOWN cwltool-1.0.20170114120503/setup.cfg0000644000175000017500000000035313036412107017253 0ustar michaelmichael00000000000000[flake8] ignore = E124,E128,E129,E201,E202,E225,E226,E231,E265,E271,E302,E303,F401,E402,E501,W503,E731,F811,F821,F841 exclude = cwltool/schemas [easy_install] [egg_info] tag_build = .20170114120503 tag_date = 0 tag_svn_revision = 0 cwltool-1.0.20170114120503/cwltool.egg-info/0000755000175000017500000000000013036412107020606 5ustar michaelmichael00000000000000cwltool-1.0.20170114120503/cwltool.egg-info/PKG-INFO0000644000175000017500000001040513036412107021703 0ustar michaelmichael00000000000000Metadata-Version: 1.1 Name: cwltool Version: 1.0.20170114120503 Summary: Common workflow language reference implementation Home-page: https://github.com/common-workflow-language/cwltool Author: Common workflow language working group Author-email: common-workflow-language@googlegroups.com License: Apache 2.0 Download-URL: https://github.com/common-workflow-language/cwltool Description: ================================================================== Common workflow language tool description reference implementation ================================================================== CWL Conformance test: |Build Status| This is the reference implementation of the Common Workflow Language. It is intended to be feature complete and provide comprehensive validation of CWL files as well as provide other tools related to working with CWL. This is written and tested for Python 2.7. The reference implementation consists of two packages. The "cwltool" package is the primary Python module containing the reference implementation in the "cwltool" module and console executable by the same name. The "cwlref-runner" package is optional and provides an additional entry point under the alias "cwl-runner", which is the implementation-agnostic name for the default CWL interpreter installed on a host. Install ------- Installing the official package from PyPi (will install "cwltool" package as well):: pip install cwlref-runner If installling alongside another CWL implementation then:: pip install cwltool To install from source:: git clone https://github.com/common-workflow-language/cwltool.git cd cwltool && python setup.py install cd cwlref-runner && python setup.py install # co-installing? skip this Remember, if co-installing multiple CWL implementations then you need to maintain which implementation ``cwl-runner`` points to via a symbolic file system link or [another facility](https://wiki.debian.org/DebianAlternatives). Run on the command line ----------------------- Simple command:: cwl-runner [tool-or-workflow-description] [input-job-settings] Or if you have multiple CWL implementations installed and you want to override the default cwl-runner use:: cwltool [tool-or-workflow-description] [input-job-settings] Import as a module ---------------- Add:: import cwltool to your script. Use with boot2docker -------------------- boot2docker is running docker inside a virtual machine and it only mounts ``Users`` on it. The default behavoir of CWL is to create temporary directories under e.g. ``/Var`` which is not accessible to Docker containers. To run CWL successfully with boot2docker you need to set the ``--tmpdir-prefix`` and ``--tmp-outdir-prefix`` to somewhere under ``/Users``:: $ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json .. |Build Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance :target: https://ci.commonwl.org/job/cwltool-conformance/ Tool or workflow loading from remote or local locations ------------------------------------------------------- ``cwltool`` can run tool and workflow descriptions on both local and remote systems via its support for HTTP[S] URLs. Input job files and Workflow steps (via the `run` directive) can reference CWL documents using absolute or relative local filesytem paths. If a relative path is referenced and that document isn't found in the current directory then the following locations will be searched: http://www.commonwl.org/v1.0/CommandLineTool.html#Discovering_CWL_documents_on_a_local_filesystem Platform: UNKNOWN cwltool-1.0.20170114120503/cwltool.egg-info/top_level.txt0000644000175000017500000000001013036412107023327 0ustar michaelmichael00000000000000cwltool cwltool-1.0.20170114120503/cwltool.egg-info/SOURCES.txt0000644000175000017500000001466513036412107022506 0ustar michaelmichael00000000000000MANIFEST.in Makefile README.rst cwltool.py ez_setup.py gittaggers.py setup.cfg setup.py cwltool/__init__.py cwltool/__main__.py cwltool/builder.py cwltool/cwlNodeEngine.js cwltool/cwlrdf.py cwltool/docker.py cwltool/docker_uid.py cwltool/draft2tool.py cwltool/errors.py cwltool/expression.py cwltool/factory.py cwltool/flatten.py cwltool/job.py cwltool/load_tool.py cwltool/main.py cwltool/pack.py cwltool/pathmapper.py cwltool/process.py cwltool/resolver.py cwltool/sandboxjs.py cwltool/stdfsaccess.py cwltool/update.py cwltool/utils.py cwltool/workflow.py cwltool.egg-info/PKG-INFO cwltool.egg-info/SOURCES.txt cwltool.egg-info/dependency_links.txt cwltool.egg-info/entry_points.txt cwltool.egg-info/requires.txt cwltool.egg-info/top_level.txt cwltool.egg-info/zip-safe cwltool/schemas/draft-2/CommonWorkflowLanguage.yml cwltool/schemas/draft-2/cwl-avro.yml cwltool/schemas/draft-3/CommandLineTool-standalone.yml cwltool/schemas/draft-3/CommandLineTool.yml cwltool/schemas/draft-3/CommonWorkflowLanguage.yml cwltool/schemas/draft-3/Process.yml cwltool/schemas/draft-3/README.md cwltool/schemas/draft-3/UserGuide.yml cwltool/schemas/draft-3/Workflow.yml cwltool/schemas/draft-3/concepts.md cwltool/schemas/draft-3/contrib.md cwltool/schemas/draft-3/index.yml cwltool/schemas/draft-3/intro.md cwltool/schemas/draft-3/invocation.md cwltool/schemas/draft-3/userguide-intro.md cwltool/schemas/draft-3/salad/schema_salad/metaschema/field_name.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/field_name_proc.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/field_name_schema.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/field_name_src.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/ident_res.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/ident_res_proc.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/ident_res_schema.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/ident_res_src.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/import_include.md cwltool/schemas/draft-3/salad/schema_salad/metaschema/link_res.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/link_res_proc.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/link_res_schema.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/link_res_src.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/metaschema.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/salad.md cwltool/schemas/draft-3/salad/schema_salad/metaschema/vocab_res.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/vocab_res_proc.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/vocab_res_schema.yml cwltool/schemas/draft-3/salad/schema_salad/metaschema/vocab_res_src.yml cwltool/schemas/v1.0/CommandLineTool-standalone.yml cwltool/schemas/v1.0/CommandLineTool.yml cwltool/schemas/v1.0/CommonWorkflowLanguage.yml cwltool/schemas/v1.0/Process.yml cwltool/schemas/v1.0/README.md cwltool/schemas/v1.0/UserGuide.yml cwltool/schemas/v1.0/Workflow.yml cwltool/schemas/v1.0/concepts.md cwltool/schemas/v1.0/contrib.md cwltool/schemas/v1.0/intro.md cwltool/schemas/v1.0/invocation.md cwltool/schemas/v1.0/userguide-intro.md cwltool/schemas/v1.0/salad/schema_salad/metaschema/field_name.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/field_name_proc.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/field_name_schema.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/field_name_src.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/ident_res.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/ident_res_proc.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/ident_res_schema.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/ident_res_src.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/import_include.md cwltool/schemas/v1.0/salad/schema_salad/metaschema/link_res.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/link_res_proc.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/link_res_schema.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/link_res_src.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/metaschema.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/metaschema_base.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/salad.md cwltool/schemas/v1.0/salad/schema_salad/metaschema/vocab_res.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/vocab_res_proc.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/vocab_res_schema.yml cwltool/schemas/v1.0/salad/schema_salad/metaschema/vocab_res_src.yml cwltool/schemas/v1.1.0-dev1/CommandLineTool-standalone.yml cwltool/schemas/v1.1.0-dev1/CommandLineTool.yml cwltool/schemas/v1.1.0-dev1/CommonWorkflowLanguage.yml cwltool/schemas/v1.1.0-dev1/Process.yml cwltool/schemas/v1.1.0-dev1/README.md cwltool/schemas/v1.1.0-dev1/UserGuide.yml cwltool/schemas/v1.1.0-dev1/Workflow.yml cwltool/schemas/v1.1.0-dev1/concepts.md cwltool/schemas/v1.1.0-dev1/contrib.md cwltool/schemas/v1.1.0-dev1/intro.md cwltool/schemas/v1.1.0-dev1/invocation.md cwltool/schemas/v1.1.0-dev1/userguide-intro.md cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name_proc.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name_schema.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/field_name_src.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res_proc.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res_schema.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/ident_res_src.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/import_include.md cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res_proc.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res_schema.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/link_res_src.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/metaschema.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/metaschema_base.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/salad.md cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res_proc.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res_schema.yml cwltool/schemas/v1.1.0-dev1/salad/schema_salad/metaschema/vocab_res_src.ymlcwltool-1.0.20170114120503/cwltool.egg-info/requires.txt0000644000175000017500000000024513036412107023207 0ustar michaelmichael00000000000000setuptools requests >= 1.0 ruamel.yaml >= 0.12.4 rdflib >= 4.2.0, < 4.3.0 shellescape >= 3.4.1, < 3.5 schema-salad >= 2.2.20170111180227, < 3 typing >= 3.5.2, < 3.6 cwltool-1.0.20170114120503/cwltool.egg-info/zip-safe0000644000175000017500000000000113036412102022231 0ustar michaelmichael00000000000000 cwltool-1.0.20170114120503/cwltool.egg-info/dependency_links.txt0000644000175000017500000000000113036412107024654 0ustar michaelmichael00000000000000 cwltool-1.0.20170114120503/cwltool.egg-info/entry_points.txt0000644000175000017500000000005713036412107024106 0ustar michaelmichael00000000000000[console_scripts] cwltool = cwltool.main:main cwltool-1.0.20170114120503/MANIFEST.in0000644000175000017500000000006613036412052017170 0ustar michaelmichael00000000000000include gittaggers.py ez_setup.py Makefile cwltool.py cwltool-1.0.20170114120503/gittaggers.py0000644000175000017500000000141713036412053020146 0ustar michaelmichael00000000000000from setuptools.command.egg_info import egg_info import subprocess import time class EggInfoFromGit(egg_info): """Tag the build with git commit timestamp. If a build tag has already been set (e.g., "egg_info -b", building from source package), leave it alone. """ def git_timestamp_tag(self): gitinfo = subprocess.check_output( ['git', 'log', '--first-parent', '--max-count=1', '--format=format:%ct', '.']).strip() return time.strftime('.%Y%m%d%H%M%S', time.gmtime(int(gitinfo))) def tags(self): if self.tag_build is None: try: self.tag_build = self.git_timestamp_tag() except subprocess.CalledProcessError: pass return egg_info.tags(self) cwltool-1.0.20170114120503/README.rst0000644000175000017500000000625613036412052017130 0ustar michaelmichael00000000000000================================================================== Common workflow language tool description reference implementation ================================================================== CWL Conformance test: |Build Status| This is the reference implementation of the Common Workflow Language. It is intended to be feature complete and provide comprehensive validation of CWL files as well as provide other tools related to working with CWL. This is written and tested for Python 2.7. The reference implementation consists of two packages. The "cwltool" package is the primary Python module containing the reference implementation in the "cwltool" module and console executable by the same name. The "cwlref-runner" package is optional and provides an additional entry point under the alias "cwl-runner", which is the implementation-agnostic name for the default CWL interpreter installed on a host. Install ------- Installing the official package from PyPi (will install "cwltool" package as well):: pip install cwlref-runner If installling alongside another CWL implementation then:: pip install cwltool To install from source:: git clone https://github.com/common-workflow-language/cwltool.git cd cwltool && python setup.py install cd cwlref-runner && python setup.py install # co-installing? skip this Remember, if co-installing multiple CWL implementations then you need to maintain which implementation ``cwl-runner`` points to via a symbolic file system link or [another facility](https://wiki.debian.org/DebianAlternatives). Run on the command line ----------------------- Simple command:: cwl-runner [tool-or-workflow-description] [input-job-settings] Or if you have multiple CWL implementations installed and you want to override the default cwl-runner use:: cwltool [tool-or-workflow-description] [input-job-settings] Import as a module ---------------- Add:: import cwltool to your script. Use with boot2docker -------------------- boot2docker is running docker inside a virtual machine and it only mounts ``Users`` on it. The default behavoir of CWL is to create temporary directories under e.g. ``/Var`` which is not accessible to Docker containers. To run CWL successfully with boot2docker you need to set the ``--tmpdir-prefix`` and ``--tmp-outdir-prefix`` to somewhere under ``/Users``:: $ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json .. |Build Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance :target: https://ci.commonwl.org/job/cwltool-conformance/ Tool or workflow loading from remote or local locations ------------------------------------------------------- ``cwltool`` can run tool and workflow descriptions on both local and remote systems via its support for HTTP[S] URLs. Input job files and Workflow steps (via the `run` directive) can reference CWL documents using absolute or relative local filesytem paths. If a relative path is referenced and that document isn't found in the current directory then the following locations will be searched: http://www.commonwl.org/v1.0/CommandLineTool.html#Discovering_CWL_documents_on_a_local_filesystem cwltool-1.0.20170114120503/ez_setup.py0000755000175000017500000002625013036412053017651 0ustar michaelmichael00000000000000#!/usr/bin/env python """ Setuptools bootstrapping installer. Run this script to install or upgrade setuptools. """ import os import shutil import sys import tempfile import zipfile import optparse import subprocess import platform import textwrap import contextlib import warnings from distutils import log try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen try: from site import USER_SITE except ImportError: USER_SITE = None DEFAULT_VERSION = "18.5" DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" DEFAULT_SAVE_DIR = os.curdir def _python_cmd(*args): """ Execute a command. Return True if the command succeeded. """ args = (sys.executable,) + args return subprocess.call(args) == 0 def _install(archive_filename, install_args=()): """Install Setuptools.""" with archive_context(archive_filename): # installing log.warn('Installing Setuptools') if not _python_cmd('setup.py', 'install', *install_args): log.warn('Something went wrong during the installation.') log.warn('See the error message above.') # exitcode will be 2 return 2 def _build_egg(egg, archive_filename, to_dir): """Build Setuptools egg.""" with archive_context(archive_filename): # building an egg log.warn('Building a Setuptools egg in %s', to_dir) _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) # returning the result log.warn(egg) if not os.path.exists(egg): raise IOError('Could not build the egg.') class ContextualZipFile(zipfile.ZipFile): """Supplement ZipFile class to support context manager for Python 2.6.""" def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def __new__(cls, *args, **kwargs): """Construct a ZipFile or ContextualZipFile as appropriate.""" if hasattr(zipfile.ZipFile, '__exit__'): return zipfile.ZipFile(*args, **kwargs) return super(ContextualZipFile, cls).__new__(cls) @contextlib.contextmanager def archive_context(filename): """ Unzip filename to a temporary directory, set to the cwd. The unzipped target is cleaned up after. """ tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) with ContextualZipFile(filename) as archive: archive.extractall() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) yield finally: os.chdir(old_wd) shutil.rmtree(tmpdir) def _do_download(version, download_base, to_dir, download_delay): """Download Setuptools.""" egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): archive = download_setuptools(version, download_base, to_dir, download_delay) _build_egg(egg, archive, to_dir) sys.path.insert(0, egg) # Remove previously-imported pkg_resources if present (see # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). if 'pkg_resources' in sys.modules: del sys.modules['pkg_resources'] import setuptools setuptools.bootstrap_install_from = egg def use_setuptools( version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=DEFAULT_SAVE_DIR, download_delay=15): """ Ensure that a setuptools version is installed. Return None. Raise SystemExit if the requested version or later cannot be installed. """ to_dir = os.path.abspath(to_dir) # prior to importing, capture the module state for # representative modules. rep_modules = 'pkg_resources', 'setuptools' imported = set(sys.modules).intersection(rep_modules) try: import pkg_resources pkg_resources.require("setuptools>=" + version) # a suitable version is already installed return except ImportError: # pkg_resources not available; setuptools is not installed; download pass except pkg_resources.DistributionNotFound: # no version of setuptools was found; allow download pass except pkg_resources.VersionConflict as VC_err: if imported: _conflict_bail(VC_err, version) # otherwise, unload pkg_resources to allow the downloaded version to # take precedence. del pkg_resources _unload_pkg_resources() return _do_download(version, download_base, to_dir, download_delay) def _conflict_bail(VC_err, version): """ Setuptools was imported prior to invocation, so it is unsafe to unload it. Bail out. """ conflict_tmpl = textwrap.dedent(""" The required version of setuptools (>={version}) is not available, and can't be installed while this script is running. Please install a more recent version first, using 'easy_install -U setuptools'. (Currently using {VC_err.args[0]!r}) """) msg = conflict_tmpl.format(**locals()) sys.stderr.write(msg) sys.exit(2) def _unload_pkg_resources(): del_modules = [ name for name in sys.modules if name.startswith('pkg_resources') ] for mod_name in del_modules: del sys.modules[mod_name] def _clean_check(cmd, target): """ Run the command to download target. If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) except subprocess.CalledProcessError: if os.access(target, os.F_OK): os.unlink(target) raise def download_file_powershell(url, target): """ Download the file at url to target using Powershell. Powershell will validate trust. Raise an exception if the command cannot complete. """ target = os.path.abspath(target) ps_cmd = ( "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " "[System.Net.CredentialCache]::DefaultCredentials; " "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars() ) cmd = [ 'powershell', '-Command', ps_cmd, ] _clean_check(cmd, target) def has_powershell(): """Determine if Powershell is available.""" if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_powershell.viable = has_powershell def download_file_curl(url, target): cmd = ['curl', url, '--silent', '--output', target] _clean_check(cmd, target) def has_curl(): cmd = ['curl', '--version'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_curl.viable = has_curl def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) def has_wget(): cmd = ['wget', '--version'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_wget.viable = has_wget def download_file_insecure(url, target): """Use Python to download the file, without connection authentication.""" src = urlopen(url) try: # Read all the data in one block. data = src.read() finally: src.close() # Write all the data in one block to avoid creating a partial file. with open(target, "wb") as dst: dst.write(data) download_file_insecure.viable = lambda: True def get_best_downloader(): downloaders = ( download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, ) viable_downloaders = (dl for dl in downloaders if dl.viable()) return next(viable_downloaders, None) def download_setuptools( version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=DEFAULT_SAVE_DIR, delay=15, downloader_factory=get_best_downloader): """ Download setuptools from a specified location and return its filename. `version` should be a valid setuptools version number that is available as an sdist for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. ``downloader_factory`` should be a function taking no arguments and returning a function for downloading a URL to a target. """ # making sure we use the absolute path to_dir = os.path.abspath(to_dir) zip_name = "setuptools-%s.zip" % version url = download_base + zip_name saveto = os.path.join(to_dir, zip_name) if not os.path.exists(saveto): # Avoid repeated downloads log.warn("Downloading %s", url) downloader = downloader_factory() downloader(url, saveto) return os.path.realpath(saveto) def _build_install_args(options): """ Build the arguments to 'python setup.py install' on the setuptools package. Returns list of command line arguments. """ return ['--user'] if options.user_install else [] def _parse_args(): """Parse the command line for options.""" parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, help='install in user site package (requires Python 2.6 or later)') parser.add_option( '--download-base', dest='download_base', metavar="URL", default=DEFAULT_URL, help='alternative URL from where to download the setuptools package') parser.add_option( '--insecure', dest='downloader_factory', action='store_const', const=lambda: download_file_insecure, default=get_best_downloader, help='Use internal, non-validating downloader' ) parser.add_option( '--version', help="Specify which version to download", default=DEFAULT_VERSION, ) parser.add_option( '--to-dir', help="Directory to save (and re-use) package", default=DEFAULT_SAVE_DIR, ) options, args = parser.parse_args() # positional arguments are ignored return options def _download_args(options): """Return args for download_setuptools function from cmdline args.""" return dict( version=options.version, download_base=options.download_base, downloader_factory=options.downloader_factory, to_dir=options.to_dir, ) def main(): """Install or upgrade setuptools and EasyInstall.""" options = _parse_args() archive = download_setuptools(**_download_args(options)) return _install(archive, _build_install_args(options)) if __name__ == '__main__': sys.exit(main()) cwltool-1.0.20170114120503/setup.py0000755000175000017500000000444313036412053017153 0ustar michaelmichael00000000000000#!/usr/bin/env python import ez_setup ez_setup.use_setuptools() import os import sys import shutil import setuptools.command.egg_info as egg_info_cmd from setuptools import setup, find_packages SETUP_DIR = os.path.dirname(__file__) README = os.path.join(SETUP_DIR, 'README.rst') try: import gittaggers tagger = gittaggers.EggInfoFromGit except ImportError: tagger = egg_info_cmd.egg_info setup(name='cwltool', version='1.0', description='Common workflow language reference implementation', long_description=open(README).read(), author='Common workflow language working group', author_email='common-workflow-language@googlegroups.com', url="https://github.com/common-workflow-language/cwltool", download_url="https://github.com/common-workflow-language/cwltool", license='Apache 2.0', packages=["cwltool"], package_data={'cwltool': ['schemas/draft-2/*.yml', 'schemas/draft-3/*.yml', 'schemas/draft-3/*.md', 'schemas/draft-3/salad/schema_salad/metaschema/*.yml', 'schemas/draft-3/salad/schema_salad/metaschema/*.md', 'schemas/v1.0/*.yml', 'schemas/v1.0/*.md', 'schemas/v1.0/salad/schema_salad/metaschema/*.yml', 'schemas/v1.0/salad/schema_salad/metaschema/*.md', 'schemas/v1.1.0-dev1/*.yml', 'schemas/v1.1.0-dev1/*.md', 'schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.yml', 'schemas/v1.1.0-dev1/salad/schema_salad/metaschema/*.md', 'cwlNodeEngine.js']}, install_requires=[ 'setuptools', 'requests >= 1.0', 'ruamel.yaml >= 0.12.4', 'rdflib >= 4.2.0, < 4.3.0', 'shellescape >= 3.4.1, < 3.5', 'schema-salad >= 2.2.20170111180227, < 3', 'typing >= 3.5.2, < 3.6' ], test_suite='tests', tests_require=[], entry_points={ 'console_scripts': [ "cwltool=cwltool.main:main" ] }, zip_safe=True, cmdclass={'egg_info': tagger}, ) cwltool-1.0.20170114120503/cwltool.py0000755000175000017500000000050013036412052017463 0ustar michaelmichael00000000000000#!/usr/bin/env python """Convienance entry point for cwltool. This can be used instead of the recommended method of `./setup.py install` or `./setup.py develop` and then using the generated `cwltool` executable. """ import sys from cwltool import main if __name__ == "__main__": sys.exit(main.main(sys.argv[1:]))