---
pipeline:
  name: tchain_date_gap_filler_limiter
transform:
  image: registry.access.redhat.com/ubi8/ubi-minimal:8.3
  cmd: ["/bin/bash"]
  env:
    # START_DATE and END_DATE indicate the date range (inclusive) to create the /Y/M/D folder structure
    # START_DATE must be set, format "YYYY-MM-DD"
    # END_DATE can be set or unset (remove line entirely to unset). If unset, end date will be yesterday. 
    OUT_PATH: /pfs/out
    START_DATE: "2020-01-01"
    END_DATE: "2020-01-03"
  stdin:
  - "#!/bin/bash"
  - "# If END_DATE unset, set to yesterday"
  - "[  -z $END_DATE ] && END_DATE=$(date -u -d '1 day ago' +%Y/%m/%d)"
  - "# Create daily folder structure from start date to end date"
  - "for d in $(seq $(date -u +%s -d $START_DATE) +86400 $(date -u +%s -d $END_DATE)) ; do"
  - "date_path=$(date -u +%Y/%m/%d -d @$d)"
  - "echo $OUT_PATH/$date_path"
  - "mkdir -p $OUT_PATH/$date_path"
  - "touch $OUT_PATH/$date_path/empty"
  - "done"
input:
  # Currently the input is set to a cron 
  # In production this should be linked to the daily ingest trigger
  pfs: 
    repo: cron_daily_tchain_tick
    glob: /*
    empty_files: true
resource_requests:
  memory: 64M
  cpu: 0.5
autoscaling: true
scheduling_spec:
  node_selector:
    cloud.google.com/gke-ephemeral-storage-local-ssd: "true"
    nodepool.neonscience.org/pipeline: "yes"
    cloud.google.com/gke-spot: "true"
pod_spec: |-
  { "tolerations": [
    {
      "key": "nodepool.neonscience.org/pipeline",
      "operator": "Exists"
    },
    {
      "effect": "NoSchedule",
      "key": "cloud.google.com/gke-spot",
      "operator": "Exists"
    }  
  ] }
pod_patch: |-
  [
    { "op": "replace",
      "path":"/containers/1/resources/requests/memory",
      "value":"1G"
    },
    { "op": "replace",
      "path": "/containers/1/resources/requests/cpu",
      "value": "0.5"
    }
  ]
