{
  "pipeline": {
    "name": "input"
  },
  "transform": {
    "cmd": [ "sh" ],
    "stdin": [
        "apt-get update -yq && apt-get install -yq --no-install-recommends ca-certificates wget",
        "wget -e robots=off --recursive --level 1 --adjust-extension --no-check-certificate --no-directories --directory-prefix /pfs/out https://en.wikipedia.org/wiki/Main_Page"
    ],
    "acceptReturnCode": [4,5,6,7,8]
  }
}


{
  "pipeline": {
    "name": "input"
  },
  "transform": {
    "cmd": [ "bash" ],
    "stdin": [
        "apt-get update -yq && apt-get install -yq --no-install-recommends ca-certificates wget",
        "FILES=/pfs/urls/*",
        "for f in $FILES",
        "do",
            "wget -e robots=off --recursive --level 1 --adjust-extension --no-check-certificate --no-directories --directory-prefix /pfs/out/$(basename $f) $(cat $f)",
        "done"
    ],
    "acceptReturnCode": [4,5,6,7,8]
  }
}


Update wordcount pipeline to take an input repo
Update wordcount pipeline to take an input repo
