---
pipeline:
  name: par_level1_group
transform:
#  image_pull_secrets: [battelleecology-quay-read-all-pull-secret]
  image: us-central1-docker.pkg.dev/neon-shared-service/neonscience/neon-is-filt-join:v1.0.0
  cmd: ["/bin/bash"]
  stdin:
  - "#!/bin/bash"
  - "python3 -m filter_joiner.filter_joiner_main"
  env:
    CONFIG: |
      ---
      # In Pachyderm root will be index 0, 'pfs' index 1, and the repo name index 2.
      # Metadata indices will typically begin at index 3.
      input_paths:
        - path:
            name: REGULARIZED_PATH
            glob_pattern: /pfs/STATISTICS_PATH/**
            # Join on context group (already joined by day below)
            join_indices: [6]
        - path:
            name: THRESHOLD_PATH
            glob_pattern: /pfs/QUALITY_METRICS_PATH/**
            # Join on context group (already joined by day below)
            join_indices: [6]
    OUT_PATH: /pfs/out
    LOG_LEVEL: DEBUG
    RELATIVE_PATH_INDEX: "3"
input:
  join:
  - pfs:
      name: STATISTICS_PATH
      repo: par_statistics
      glob: /(*)/(*)/(*)
      joinOn: $1/$2/$3
      empty_files: true
  - pfs:
      name: QUALITY_METRICS_PATH
      repo: par_quality_metrics
      glob: /(*)/(*)/(*)
      joinOn: $1/$2/$3
      empty_files: true
parallelism_spec:
  constant: "1"
enable_stats: false
standby: true
