text:
<file_sep>#!/usr/bin/env bash
# test.sh
#
# Test picardmetrics
#
# data/project1/sample1.bam     10,000 paired-end 101 bp reads
# data/project1/sample2.bam     similar reads, but the header is shuffled

OK=1

main() {
  # Clean up previous runs.
  rm -rf out

  echo -e "$(DATE)\tSTART test_default"
  test_default
  echo -e "$(DATE)\tDONE test_default\n"

  echo -e "$(DATE)\tSTART test_rnaseq"
  test_rnaseq
  echo -e "$(DATE)\tDONE test_rnaseq\n"

  if [[ ! $OK = 1 ]];then
    echo -e "$(DATE)\tFailed"
    exit 1
  fi
}

test_default() {
  out=out/default
  mkdir -p $out

  run_log=$out/picardmetrics-run.log
  collate_log=$out/picardmetrics-collate.log
  rm -f $run_log $collate_log

  # Run multiple metrics on each BAM file.
  for f in data/project1/sample?/sample?.bam
  do
    ./picardmetrics run -f ~/picardmetrics.conf -o $out $f 2>&1 >> $run_log
  done

  # Collate the generated tables.
  ./picardmetrics collate $out $out &> $collate_log

  # Confirm that the final output is correct.
  checksum=$(md5sum ${out}-all-metrics.tsv | cut -f1 -d' ')
  if [[ ! "244cffd02775ab1517578147aee39ab1" = "$checksum" ]]; then
    cat $run_log >&2
    cat $collate_log >&2
    OK=0
  fi
}

test_rnaseq() {
  out=out/rnaseq
  mkdir -p $out

  run_log=$out/picardmetrics-run.log
  collate_log=$out/picardmetrics-collate.log
  rm -f $run_log $collate_log

  # Run multiple metrics on each BAM file.
  for f in data/project1/sample?/sample?.bam
  do
    ./picardmetrics run -k -r -f ~/picardmetrics.conf -o $out $f 2>&1 >> $run_log

    # Ensure that the -k option worked.
    dup_bam=$out/$(basename ${f%*.bam}.MarkDuplicates.bam)
    if [[ ! -f $dup_bam ]]; then
      echo "ERROR: Missing file '$dup_bam'" >&2
      OK=0
    fi
  done

  # Collate the generated tables.
  ./picardmetrics collate $out $out &> $collate_log

  # Confirm that the final output is correct.
  checksum=$(md5sum ${out}-all-metrics.tsv | cut -f1 -d' ')
  if [[ ! "3482de25bcf68887d71ed1803505ec66" = "$checksum" ]]; then
    cat $run_log >&2
    cat $collate_log >&2
    OK=0
  fi
}

DATE() {
  command date +'%Y-%m-%d %H:%M:%S'
}

main
<file_sep># Travis will do the following:
#
#   git clone https://github.com/slowkow/picardmetrics
#   cd picardmetrics

addons:
    apt:
        packages:
        - r-base

before_install:
    - wget https://zenodo.org/record/18116/files/picardmetrics-data.tar.xz
    - tar xf picardmetrics-data.tar.xz
    - export PATH="$HOME/.local/bin:$PATH"
    - make get-deps

install:
    - perl -pe '$x = qx{pwd}; chomp $x; s{/PATH/TO}{$x/data}' picardmetrics.conf > ~/picardmetrics.conf
    - cat ~/picardmetrics.conf

script: ./test/test.sh

<file_sep># Change Log

# 0.2.4 - 2016-07-06

## Fixed

-   Test that option `-k` works as it should.

# 0.2.3 - 2016-06-06

## Fixed

-   Delete the intermediate collated files only after they are successfully
    collated into one master file.

-   Prefer `picardmetrics.conf` in the working directory if it exists.

## Changed

-   Increase compatibility:

    -   Sample name is now the filename without the file extension.

    -   Use `find` to discover Picard output files in directories.

    -   Use the BAM dirname as part of the `TEMP_DIR`.

    -   Call the output BAM `file.MarkDuplicates.bam`

    -   Search for `"rRNA"` instead of `gene_type "rRNA"` to find ribosomal
        intervals in a GTF file.

# 0.2.2 - 2015-06-12

## Fixed

-   Set `TMP_DIR` option in all invocations to Picard tools.

# 0.2.1 - 2015-06-05

## Added

-   Add `PREFIX-insert-size-histogram.tsv` to the collated outputs.

-   Add `PREFIX-base-distribution-by-cycle-histogram.tsv` to the collated
	outputs.

## Changed

-   Use `## METRICS` and `## HISTOGRAM` as landmarks to find the correct
    lines in the Picard output files. This is more robust than using line
    numbers.

-   Delete the collated output files that are successfully integrated into the
    `PREFIX-all-metrics.tsv` file. They are redundant.

# 0.2.0 - 2015-05-28

## Added

-   Added two commands:

    -    refFlat

        -   This creates a file for the REF_FLAT argument of
            CollectRnaSeqMetrics.

    -    rRNA

        -   This creates a file for the RIBOSOMAL_INTERVALS argument of
            CollectRnaSeqMetrics.

-   Travis continuous integration testing. Each time I push new changes to the
    repo, Travis will automatically check that the picardmetrics output is
    correct.

## Fixed

-   This was harmless, but some of the collated output files were missing the
    trailing tab character at the end of the line. Some also had extra lines
    without data. Now all of the collated output files should be perfectly
    formatted.

-   Bash functions return 1 when problems are encountered.

-   The `-k` option is checked.

## Changed

-   `picardmetrics run` now puts the newly generated BAM file in `/tmp` and
    deletes it after successfully running, unless the `-k` option is used.

-   `picardmetrics run` now runs CreateSequenceDictionary and ReorderSam on
    the BAM file to avoid issues where the sequence names in the header of the
    BAM file is not in the same order as the sequence names in the reference
    sequence FASTA file.

-   Shuffled the header for `data/project1/sample2.bam` so we can ensure that
    BAM files with unordered headers work correctly.

-   Rearranged output files so you can see the extra metrics contributed by
    CollectRnaSeqMetrics:

    -   Compare the files in `out/default` with `out/rnaseq`.

## Removed

-   Removed files that are no longer needed:

    -   `scripts/make_refFlat`
    -   `scripts/make_rRNA_intervals`
    -   `scripts/plot_picardmetrics.R`

# 0.1.4 - 2015-05-22

## Fixed

-   The `picard collate` command now tolerates missing files and will still
    collate the existing files.

-   The collated output file `PREFIX-alignment-metrics.tsv` no longer has
    extra blanks lines.

-   Fix the invocation of `md5sum -c` in `test/test.sh`.

-   Some systems like CentOS don't have an up-to-date version of coreutils.
    So, I replaced the invocation of `realpath` with a Bash function that does
    produces the same output.

## Changed

-   The `rnaseq_metrics` function will generate a sequence dictionary for the
    rRNA intervals_list file on-the-fly by taking it from the BAM file. The
    sequence dictionary consists of the BAM header lines starting with '@SQ'.

    -   The script `scripts/make_rRNA_intervals` no longer downloads
        chromosome sizes from UCSC.

-   The `sort_sam` function checks if the output BAM is correct.

-   The reference genome sequence is no longer copied to RAM.

-   The script `scripts/make_refFlat` no longer downloads the reference
    genome. The user must use their own anyway.

-   In `scripts/install_deps`, roll back to Picard 1.126 because of a
    conflicting change introduced in Picard 1.130. See here for more details:
    https://github.com/broadinstitute/picard/issues/212

# 0.1.3 - 2015-04-23

## Fixed

-   The `collate_` functions did not correctly check for the number of metrics
    files. This affects output generated using `picardmetrics` without `-r`.

## Changed

-   The output BAM file is now deleted after a successful run.

-   Added option `-k` to optionally keep the output BAM file.

-   The configuration file is now called `picardmetrics.conf` instead of
    `.picardmetricsrc`.

## Added

-   Added extra commands to the Makefile to ease installation of dependencies
    and download of human reference data.

    -   Now h38 is downloaded by default instead of hg19. The user can modify
        the script to get different versions.

# 0.1.2 - 2015-04-09

## Fixed

-   In `collate_all`, check for `prefix-rnaseq-metrics.tsv` file before
    attempting to merge the files into `prefix-all-metrics.tsv`.

    -   Warnings are no longer printed when a user tries to collate output
        files after running picardmetrics without the '-r' option.

-   The tests have been updated: the md5sum of final output files is checked.
    This is more robust than checking content of log files.

## Changed

-   Removed regular expression from `is_bam` function, so the script will run
    with Bash version 2 released prior to 2004.

-   Previously, output files were written to a directory called
    'picardmetrics' created in the BAM file's directory. Currently, the output
    files are written to a user-specified directory or the current directory
    by default.

-   The `collate_` functions previously accepted the BAM files output by
    picardmetrics as an argument. Now, the `collate_` functions accept a
    directory instead and search for the appropriately named files. This means
    that the BAM files created by Picard may be deleted after running
    picardmetrics.

# 0.1.1 - 2015-03-20

## Changed

-   Write output files to `project1/sample1/picardmetrics` instead of
    `project1/sample1/`.

# 0.1 - 2015-03-16

-   Initial release.
<file_sep>#!/usr/bin/env bash
# install_deps.sh
# <NAME>
#
# Install dependencies for picardmetrics.

# Executables, libraries, and manpages will be put here with 'make install'.
if [[ "$1" == "" ]]; then
  PREFIX="$HOME/.local"
else
  PREFIX="$1"
fi

# Source code downloaded from the web will be put here.
SRC="$HOME/src"

main() {
  mkdir -p $PREFIX/bin
  mkdir -p $SRC

  cd $SRC

  # Picard
  VERSION=1.126
  if [[ ! -e "$SRC/picard-tools-${VERSION}/picard.jar" ]]; then
    URL="https://github.com/broadinstitute/picard/releases/download"
    if ! curl -k -L -O $URL/${VERSION}/picard-tools-${VERSION}.zip; then
      echo "Failed to download Picard"
      exit 1
    fi
    unzip picard-tools-${VERSION}.zip

    echo "Edit your picardmetrics.conf file to include:"
    echo "  PICARD_JAR=\"$SRC/picard-tools-${VERSION}/picard.jar\""
  fi

  # gtfToGenePred
  if [[ ! -e $PREFIX/bin/gtfToGenePred ]]; then
    url='http://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/gtfToGenePred'
    wget $url -O $PREFIX/bin/gtfToGenePred && chmod +x $PREFIX/bin/gtfToGenePred
  fi

  # stats
  if [[ ! -e $PREFIX/bin/stats ]]; then
    git_clone arq5x/filo

    cd filo \
      && make \
      && cp bin/stats $PREFIX/bin/ \
      && cd ..
  fi

  # samtools
  if [[ ! -e $PREFIX/bin/samtools ]]; then
    git_clone samtools/htslib

    cd htslib \
      && make \
      && make prefix=$PREFIX install \
      && cd ..

    git_clone samtools/samtools

    cd samtools \
      && make \
      && make prefix=$PREFIX install \
      && cd ..
  fi

  if ! which gtfToGenePred || ! which stats || ! which samtools ; then
    echo
    echo "Add this to your .bashrc or .bash_profile:"
    echo "    PATH=\"\$HOME/.local/bin:\$PATH\""
  fi
}

git_clone() {
  USER_REPO=$1
  REPO=$(basename $USER_REPO)
  if [[ ! -d $REPO ]]; then
    if ! git clone https://github.com/${USER_REPO}; then
      echo "ERROR: Failed to download ${USER_REPO}"
      exit 1
    fi
  fi
}

main
<file_sep># picardmetrics

Run [Picard] tools and collate multiple metrics files. Check the quality of
your sequencing data.

[![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.17142.svg)](http://dx.doi.org/10.5281/zenodo.17142)
[![Build Status](https://travis-ci.org/slowkow/picardmetrics.svg?branch=master)](https://travis-ci.org/slowkow/picardmetrics)

## Summary

Run `picardmetrics` like this:

```bash
for bam in data/project1/sample?/sample?.bam
do
  # -k keeps the BAM file with marked duplicate reads
  # -r runs RNA-seq Picard metrics
  # -o specifies where to put the output files
  picardmetrics run -k -r -o out/rnaseq $bam
done

# The final output file will be called "project1-all-metrics.tsv"
picardmetrics collate project1 out/rnaseq
```

`picardmetrics` runs up to [12 Picard tools][manual] on each [BAM] file and
collates all of the output files into a single table with up to [90 different
metrics][definitions]. It also automatically creates the `.refFlat` and
`.rRNA.list` files required for [CollectRnaSeqMetrics].

See the [picardmetrics manual][manual] for more details.

Next, plot and explore the metrics in R:

```r
library(ggplot2)

dat <- read.delim("project1-all-metrics.tsv", stringsAsFactors = FALSE)

ggplot(dat) +
  geom_point(aes(PF_READS, PF_ALIGNED_BASES))
```

See two example BAM files in the [data/][data] folder. The
[test/test.sh][test] script illustrates the usage of `picardmetrics` and tests
that it works correctly. See the outputs in the [out/][out] folder. You can
also download the [reference files][reference] used to test `picardmetrics`.

[scripts]: https://github.com/slowkow/picardmetrics/tree/master/scripts
[data]: https://github.com/slowkow/picardmetrics/tree/master/data
[test]: https://github.com/slowkow/picardmetrics/tree/master/test/test.sh
[out]: https://github.com/slowkow/picardmetrics/tree/master/out

[manual]: https://slowkow.github.io/picardmetrics
[reference]: http://dx.doi.org/10.5281/zenodo.18116

[definitions]: https://broadinstitute.github.io/picard/picard-metric-definitions.html
[CollectRnaSeqMetrics]: https://broadinstitute.github.io/picard/command-line-overview.html#CollectRnaSeqMetrics

## Example

![Genes detected vs. Mean MAPQ and Percent of bases vs. Sample][example]

[example]: https://github.com/slowkow/picardmetrics/blob/master/man/picardmetrics-banner.png

Use Picard to assess the quality of your sequencing data. This example shows
RNA-seq data from [hundreds of glioblastoma cells and gliomasphere cell
lines][Patel2014].

On the left, each point represents an RNA-seq sample. We see that samples
with high mean mapping quality have the greatest number of detected genes.
Further, the color reveals variation in the percent of reads per sample
that are assigned to exons.

On the right, each bar represents an RNA-seq sample. Each sample is broken
down into the percent of sequenced bases coming from different genomic
regions. We see that many samples have few sequenced bases coming from
coding regions relative to intergenic regions.

[Patel2014]: http://www.ncbi.nlm.nih.gov/bioproject/PRJNA248302

## Installation

```bash
# Download the code.
git clone https://github.com/slowkow/picardmetrics

cd picardmetrics

# Download and install the dependencies.
make get-deps PREFIX=~/.local

# Install picardmetrics and the man page.
make install PREFIX=~/.local

# Edit the configuration file for your project.
vim ~/picardmetrics.conf
```

If you wish, you can manually install the dependencies:

-   [Picard]
-   [samtools], which depends on [htslib]
-   [stats]
-   [gtfToGenePred]

[BAM]: http://samtools.github.io/hts-specs/SAMv1.pdf
[Gencode]: http://www.gencodegenes.org/

[Picard]: https://broadinstitute.github.io/picard/
[samtools]: https://github.com/samtools/samtools
[htslib]: https://github.com/samtools/htslib
[stats]: https://github.com/arq5x/filo
[gtfToGenePred]: http://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/

## Contributing

Please [submit an issue][issues] to report bugs or ask questions.

Please contribute bug fixes or new features with a [pull request][pull] to this repository.

[issues]: https://github.com/slowkow/picardmetrics/issues
[pull]: https://help.github.com/articles/using-pull-requests/

## Related work

[RNA-SeQC][rnaseqc]

> RNA-SeQC is a java program which computes a series of quality control
> metrics for RNA-seq data. The input can be one or more BAM files. The output
> consists of HTML reports and tab delimited files of metrics data. This
> program can be valuable for comparing sequencing quality across different
> samples or experiments to evaluate different experimental parameters. It can
> also be run on individual samples as a means of quality control before
> continuing with downstream analysis.

[RSeQC][rseqc]

> RSeQC package provides a number of useful modules that can comprehensively
> evaluate high throughput sequence data especially RNA-seq data. Some basic
> modules quickly inspect sequence quality, nucleotide composition bias, PCR
> bias and GC bias, while RNA-seq specific modules evaluate sequencing
> saturation, mapped reads distribution, coverage uniformity, strand
> specificity, etc.

[QoRTs][qorts]

> The QoRTs software package is a fast, efficient, and portable multifunction
> toolkit designed to assist in the analysis, quality control, and data
> management of RNA-Seq datasets. Its primary function is to aid in the
> detection and identification of errors, biases, and artifacts produced by
> paired-end high-throughput RNA-Seq technology. In addition, it can produce
> count data designed for use with differential expression and differential
> exon usage tools 2, as well as individual-sample and/or group-summary
> genome track files suitable for use with the UCSC genome browser (or any
> compatible browser).

[rnaseqc]: http://www.broadinstitute.org/cancer/cga/rna-seqc
[rseqc]: http://rseqc.sourceforge.net/
[qorts]: https://github.com/hartleys/QoRTs
<file_sep>#!/usr/bin/env bash
# picardmetrics
# <NAME>

PICARDMETRICS_VERSION="0.2.4 2016-07-06"

# Main functions. ------------------------------------------------------------

main() {
  echo "picardmetrics version $PICARDMETRICS_VERSION"

  if [[ "$1" == "collate" ]]; then
    shift
    main_collate $@
  elif [[ "$1" == "run" ]]; then
    shift
    main_run $@
  elif [[ "$1" == "refFlat" ]]; then
    main_refFlat $@
  elif [[ "$1" == "rRNA" ]]; then
    main_rRNA $@
  else
    echo "Usage: picardmetrics COMMAND"
    echo "  run         Run the Picard tools on a given BAM file."
    echo "  collate     Collate metrics files for multiple BAM files."
    echo "  refFlat     Create a .refFlat file for CollectRnaSeqMetrics."
    echo "  rRNA        Create a .rRNA.list file for CollectRnaSeqMetrics."
  fi
}

main_run() {
  # Ensure that the script was called with correct arguments.
  if [[ "$1" == "" ]]; then
    echo "Usage: picardmetrics run [-f FILE] [-o DIR] [-r] <file.bam>"
    echo "  -f FILE     The configuration file. (Default: ./picardmetrics.conf)"
    echo "  -o DIR      Write output files in this directory. (Default: .)"
    echo "  -r          The BAM file has RNA-seq reads. (Default: false)"
    echo "  -k          Keep the output BAM file. (Default: false)"
    return 0
  fi

  # The configuration file for picardmetrics.
  PICARDMETRICS_CONFIG=""

  # By default, write output to the user's current working directory.
  OUT_DIR="."

  # Assume the input .bam file does not come from an RNA-seq experiment.
  RNASEQ=""

  # By default, the sorted and deduplicated .bam file will be deleted after
  # metrics are computed. This option allows you to keep it instead.
  KEEP=""

  while getopts ":f:o:rk" opt; do
    case "$opt" in
      f)
        PICARDMETRICS_CONFIG="$OPTARG"
        ;;
      o)
        OUT_DIR="$OPTARG"
        ;;
      r)
        RNASEQ=1
        ;;
      k)
        KEEP=1
        ;;
      \?)
        warn "Invalid option: -$OPTARG"
        exit 1
        ;;
      :)
        warn "Option -$OPTARG requires an argument."
        ;;
    esac
  done

  # Load the configuration variables.
  configure_run

  # Get the first positional argument after the option flags.
  bam=${@:$OPTIND:1}

  # Ensure the BAM file is valid.
  if ! is_bam "$bam"; then
    warn "ERROR: Not a .bam file: '$bam'"
    exit 1
  elif bam_incomplete "$bam"; then
    warn "ERROR: Missing EOF marker: '$bam'"
    exit 1
  fi

  # Get the absolute path, but do not resolve symlinks.
  bam="$(bash_realpath $bam)"

  # Make the ouput directory if it does not exist.
  if ! mkdir -p "$OUT_DIR"; then
    warn "ERROR: mkdir -p '$OUT_DIR'"
    exit 1
  fi

  # Perform all actions in the output directory.
  if ! cd "$OUT_DIR"; then
    warn "ERROR: cd '$OUT_DIR'"
    exit 1
  fi

  warn -e "$(DATE)\tSTART\t$bam"

  # Create a sequence dictionary needed for ReorderSam.
  create_sequence_dictionary "$REFERENCE_SEQUENCE"

  # Temporary directory with intermediate files.
  if [[ ! -d "$TEMP_DIR" ]]; then
    warn "Setting TEMP_DIR=/tmp"
    warn "Please ensure disk space is sufficient with 'df -h /tmp'"
    TEMP_DIR=/tmp
  fi

  # Temporary directory mimics BAM file directory.
  TEMP_DIR="$TEMP_DIR/picardmetrics_${USER}_${$}/$(dirname $bam)"
  if ! mkdir -p "$TEMP_DIR"; then
    warn "ERROR: mkdir -p '$TEMP_DIR'"
    exit 1
  fi

  # Temporary BAM file that is deleted by default.
  local temp_bam="$TEMP_DIR/$(basename $bam)"

  if ! ln -sf "$bam" "$temp_bam"; then
    warn "ERROR: ln -sf '$bam' '$temp_bam'"
    exit 1
  fi

  # Run all of the Picard tools.
  compute_metrics "$temp_bam" "$RNASEQ"

  if [[ "$KEEP" = "" ]]; then
    # Delete the temporary BAM file after computing metrics.
    if ! rm -f "$temp_bam"; then
      warn "ERROR: rm -f '$temp_bam'"
      exit 1
    fi
  else
    # Otherwise, move it to the output folder.
    dup_bam="$(basename ${bam%.bam}.MarkDuplicates.bam)"
    if ! mv "$temp_bam" "$dup_bam" ;then
      warn "ERROR: mv '$temp_bam' '$dup_bam'"
      exit 1
    fi
  fi
  # Delete the temporary directory.
  if ! rm -rf "$TEMP_DIR"; then
    warn "ERROR: rm -rf '$TEMP_DIR'"
    exit 1
  fi

  warn -e "$(DATE)\tDONE\t$bam"
}

main_collate() {
  if [[ "$1" == "" || "$2" == "" ]]; then
    echo "Usage: picardmetrics collate PREFIX DIR"
    echo "  Find all picardmetrics output files in DIR and collate them"
    echo "  into a file named 'PREFIX-all-metrics.tsv'."
    return 0
  fi

  local prefix="$1"
  local dir="$2"

  if [[ ! -d "$dir" ]]; then
    warn "ERROR: Not a directory: '$dir'"
    exit 1
  fi

  if ! mkdir -p $(dirname "$prefix"); then
    warn "ERROR: mkdir -p '$(dirname $prefix)'"
    exit 1
  fi

  warn -e "$(DATE)\tSTART\t$prefix"

  # CollectAlignmentMetrics
  collate_alignment_metrics "$prefix-alignment-metrics.tsv" "$dir"

  # CollectQualityMetrics
  collate_quality_metrics "$prefix-quality-histogram.tsv" "$dir"

  # CollectRnaSeqMetrics
  collate_rnaseq_metrics "$prefix-rnaseq-metrics.tsv" "$dir"
  collate_rnaseq_coverage "$prefix-rnaseq-coverage.tsv" "$dir"

  # CollectGcBiasMetrics
  collate_gc_bias_metrics "$prefix-gc-bias-metrics.tsv" "$dir"
  collate_gc_bias_histogram "$prefix-gc-bias-histogram.tsv" "$dir"

  # MarkDuplicates
  collate_duplicate_metrics "$prefix-duplicate-metrics.tsv" "$dir"

  # CollectInsertSizeMetrics
  collate_insert_size_metrics "$prefix-insert-size-metrics.tsv" "$dir"
  collate_insert_size_histogram "$prefix-insert-size-histogram.tsv" "$dir"

  # BaseDistributionByCycleMetrics
  collate_base_distribution_by_cycle \
    "$prefix-base-distribution-by-cycle-histogram.tsv" "$dir"

  # EstimateLibraryComplexity
  collate_library_complexity "$prefix-library-complexity.tsv" "$dir"
  collate_library_complexity_histogram \
    "$prefix-library-complexity-histogram.tsv" "$dir"

  # MAPQ statistics.
  collate_mapq_stats "$prefix-mapq-stats.tsv" "$dir"

  # Merge all collated files into one master file "$prefix-all-metrics.tsv".
  collate_all "$prefix"

  warn -e "$(DATE)\tDONE\t$prefix"
}

main_refFlat() {
  local gtf="$2"
  # Ensure that the script was called with correct arguments.
  if [[ ! ("$gtf" == *.gtf || "$gtf" == *.gtf.gz) ]]; then
    echo "Usage: picardmetrics refFlat <file.gtf[.gz]>"
    echo "  Create 'file.refFlat' for CollectRnaSeqMetrics."
    return 0
  fi

  if [[ ! -f "$gtf" ]]; then
    warn "ERROR: File not found '$gtf'"
    exit 1
  fi

  local out="${gtf%.gtf*}.refFlat"
  if [[ -f "$out" ]]; then
    echo "$out"
    return 0
  fi

  local in=$(mktemp)

  if [[ "$gtf" == *.gtf.gz ]] ; then
    gunzip -c "$gtf" > "$in"
  else
    cat "$gtf" > "$in"
  fi

  if ! which gtfToGenePred &> /dev/null; then
    warn "ERROR: gtfToGenePred is not installed."
    warn "  Install gtfToGenePred to a directory in your PATH"
    warn "  http://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/gtfToGenePred"
    exit 1
  fi

  gtfToGenePred -ignoreGroupsWithoutExons "$in" "$out" \
    && perl -i -lane 'print join "\t", ($F[0], @F)' "$out" \
    && rm -f "$in"
  local ERR=$?

  if [[ "$ERR" != 0 && "$ERR" != 255 ]]; then
      warn "Unexpected error $ERR from gtfToGenePred"
      exit 1
  fi

  echo "$out"
}

main_rRNA() {
  local gtf="$2"
  # Ensure that the script was called with correct arguments.
  if [[ ! ("$gtf" == *.gtf || "$gtf" == *.gtf.gz) ]]; then
    echo "Usage: picardmetrics rRNA <file.gtf[.gz]>"
    echo "  Create 'file.rRNA.list' for CollectRnaSeqMetrics."
    return 0
  fi

  if [[ ! -f "$gtf" ]]; then
    warn "ERROR: File not found '$gtf'"
    exit 1
  fi

  local out="${gtf%.gtf*}.rRNA.list"
  if [[ -f "$out" ]]; then
    echo "$out"
    return 0
  fi

  local in=$(mktemp)

  if [[ "$gtf" == *.gtf.gz ]] ; then
    gunzip -c "$gtf" > "$in"
  else
    cat "$gtf" > "$in"
  fi

  grep '"rRNA"' "$in" | \
      awk '$3 == "transcript"' | \
      cut -f1,4,5,7,9 | \
      perl -lane '
          /transcript_id "([^"]+)"/ or die "no transcript_id on $.";
          print join "\t", (@F[0,1,2,3], $1)
      ' | \
      sort -k1V -k2n -k3n \
  > "$out" && \
  rm -f "$in"
  local ERR=$?

  if [[ "$ERR" != 0 ]]; then
      warn "ERROR: Unexpected error $ERR while making '$out'"
      exit 1
  fi

  echo "$out"
}

# Check for correct configuration. -------------------------------------------

configure_run() {
  # In order of preference:
  #     1. The user's specified config.
  #     2. The config in the current directory.
  #     3. The config in the home directory.
  if [[ -f "$PICARDMETRICS_CONFIG" ]]; then
    PICARDMETRICS_CONFIG="$PICARDMETRICS_CONFIG"
  elif [[ -f picardmetrics.conf ]]; then
    PICARDMETRICS_CONFIG=picardmetrics.conf
  elif [[ -f ~/picardmetrics.conf ]]; then
    PICARDMETRICS_CONFIG=~/picardmetrics.conf
  else
    warn "WARNING: Config file '$PICARDMETRICS_CONFIG' not found"
  fi
  warn -e "$(DATE)\tpicardmetrics config: '$PICARDMETRICS_CONFIG'"
  source "$PICARDMETRICS_CONFIG"

  # All jobs will be launched with niceness=20 by default.
  if [[ -z "$NICENESS" ]]; then
    warn "Setting NICENESS=20"
    NICENESS=20
  fi
  NICE="nice -n$NICENESS"

  if [[ ! -e "$PICARD_JAR" ]]; then
    warn "ERROR: Please download the 'picard.jar' file"
    warn "         https://broadinstitute.github.io/picard/"
    warn "       and define PICARD_JAR=/path/to/picard.jar"
    exit 1
  fi

  if [[ -z "$PICARD" ]]; then
    warn "ERROR: Please define the variable PICARD"
    warn '       For example: PICARD="java -jar $PICARD_JAR"'
  fi

  if ! which samtools &> /dev/null; then
    warn "ERROR: Please install the 'samtools' program"
    warn "         https://github.com/samtools/samtools"
    exit 1
  fi

  if ! which stats &> /dev/null; then
    warn "ERROR: Please install the 'stats' program"
    warn "         https://github.com/arq5x/filo"
    exit 1
  fi

  if [[ ! -f "$REFERENCE_SEQUENCE" ]]; then
    warn "ERROR: FASTA file not found: '$REFERENCE_SEQUENCE'"
    warn "       Please define REFERENCE_SEQUENCE"
    warn '       For example: REFERENCE_SEQUENCE=~/data/hg19.fasta'
    exit 1
  fi

  # If the input BAM file comes from an RNA-seq experiment.
  if [[ "$RNASEQ" ]]; then
    if [[ "$REF_FLAT" && ! -f "$REF_FLAT" ]]; then
      warn "ERROR: refFlat file not found: '$REF_FLAT'"
      exit 1
    elif [[ -f "$GTF" ]]; then
      REF_FLAT=$(main_refFlat 1 "$GTF")
    fi

    if [[ "$RIBOSOMAL_INTERVALS" && ! -f "$RIBOSOMAL_INTERVALS" ]]; then
      warn "ERROR: Ribosomal intervals file not found: '$RIBOSOMAL_INTERVALS'"
      exit 1
    elif [[ -f "$GTF" ]]; then
      RIBOSOMAL_INTERVALS=$(main_rRNA 1 "$GTF")
    fi
  fi
}

# Call the Picard tools. -----------------------------------------------------

compute_metrics() {
  local bam="$1" rnaseq="$2"
  if [[ ! -f "$bam" ]]; then
    warn "BAM file not found '$bam'"
    return 1
  fi

  # Reorder the BAM header to match the header of the reference sequence.
  reorder_sam "$bam"

  # Sort the reads by coordinate.
  sort_sam "$bam"

  local prefix=$(basename "$bam" .bam)

  # Check if the BAM file is newer or if the metrics file is absent.
  mapq_stats="${prefix}.mapq_stats"
  if [[ "$bam" -nt "$mapq_stats" || ! -f "$mapq_stats" ]]; then
    if ! ( $NICE samtools view "$bam" | cut -f5 | stats > "$mapq_stats" );
    then
      warn "ERROR: mapq_stats failed"
    fi
  fi

  metrics="${prefix}.duplicate_metrics"
  if [[ "$bam" -nt "$metrics" || ! -f "$metrics" ]]; then
    # featureCounts can ignore the marked duplicate reads with --ignoreDup
    mark_duplicates "$bam"
  fi

  metrics="${prefix}.alignment_summary_metrics"
  if [[ "$bam" -nt "$metrics" || ! -f "$metrics" ]]; then
    multiple_metrics "$bam"
  fi

  if [[ "$rnaseq" ]]; then
    metrics="${prefix}.rnaseq_metrics"
    if [[ "$bam" -nt "$metrics" || ! -f "$metrics" ]]; then
      rnaseq_metrics "$bam"
    fi
  fi

  metrics="${prefix}.gc_bias_metrics"
  if [[ "$bam" -nt "$metrics" || ! -f "$metrics" ]]; then
    gcbias_metrics "$bam"
  fi

  metrics="${prefix}.library_complexity"
  if [[ "$bam" -nt "$metrics" || ! -f "$metrics" ]]; then
    library_complexity "$bam"
  fi
}

# Picard tools. --------------------------------------------------------------

run_picard() {
  local tool="$1"; shift
  local args=($@)
  local bam=${args[0]}
  local log=$(basename ${bam#*=} .bam).${tool}.log
  warn -e "$(DATE)\t$tool"
  if ! $NICE $PICARD $tool ${args[*]} &> $log; then
    warn "ERROR: $tool failed. See: $(pwd)/$log"
    cat "$(pwd)/$log" >&2
  fi
}

create_sequence_dictionary() {
  local ref="$1"
  local dict="${1%.*}.dict"
  if [[ ! -e "$dict" ]]; then
    local args=(
      CreateSequenceDictionary
      REFERENCE="$1"
      OUTPUT="$dict"
      TMP_DIR="$TEMP_DIR"
    )
    run_picard ${args[*]}
  fi
}

reorder_sam() {
  local bam="$1"
  local out="${bam%.bam}.ReorderSam.bam"
  local args=(
    ReorderSam
    INPUT="$bam"
    OUTPUT="$out"
    REFERENCE="${REFERENCE_SEQUENCE}"
    VALIDATION_STRINGENCY=LENIENT
    TMP_DIR="$TEMP_DIR"
  )
  run_picard ${args[*]}
  if ! mv -f "$out" "$bam"; then
    warn "ERROR: Failed to move '$out' to '$bam'"
    exit 1
  fi
}

sort_sam() {
  local bam="$1"
  local out="${bam%.bam}.SortSam.bam"
  if ! bam_sorted "$bam"; then
    local args=(
      SortSam
      INPUT="$bam"
      OUTPUT="$out"
      SORT_ORDER=coordinate
      VALIDATION_STRINGENCY=LENIENT
      TMP_DIR="$TEMP_DIR"
    )
    run_picard ${args[*]}
    if ! mv -f "$out" "$bam"; then
      warn "ERROR: Failed to move '$out' to '$bam'"
      exit 1
    fi
  fi
}

mark_duplicates() {
  local bam="$1"
  local out="${bam%.bam}.MarkDuplicates.bam"
  local prefix=$(basename "$bam" .bam)
  local args=(
    MarkDuplicates
    INPUT="$bam"
    OUTPUT="$out"
    METRICS_FILE="${prefix}.duplicate_metrics"
    REMOVE_DUPLICATES=false
    READ_NAME_REGEX=null
    VALIDATION_STRINGENCY=LENIENT
    TMP_DIR="$TEMP_DIR"
  )
  run_picard ${args[*]}
  if ! mv -f "$out" "$bam"; then
    warn "ERROR: Failed to move '$out' to '$bam'"
    exit 1
  fi
}

multiple_metrics() {
  local bam="$1"
  local prefix=$(basename "$bam" .bam)
  local args=(
    CollectMultipleMetrics
    INPUT="$bam"
    OUTPUT="$prefix"
    REFERENCE_SEQUENCE="$REFERENCE_SEQUENCE"
    VALIDATION_STRINGENCY=LENIENT
    PROGRAM=CollectAlignmentSummaryMetrics
    PROGRAM=CollectInsertSizeMetrics
    PROGRAM=QualityScoreDistribution
    TMP_DIR="$TEMP_DIR"
  )
  run_picard ${args[*]}
}

rnaseq_metrics() {
  local bam="$1"
  local prefix=$(basename "$bam" .bam)
  # Ignore the sequence dictionary present in the .list file,
  # and instead take the sequence dictionary from the BAM file.
  local t="$TEMP_DIR/${prefix}.list"
  if ! (
    samtools view -H "$bam" | grep '^@SQ' > "$t" \
      && grep -v '^@SQ' "$RIBOSOMAL_INTERVALS" >> "$t"
  ); then
    warn "ERROR: Could not retrieve header from '$bam'"
    exit 1
  fi
  local args=(
    CollectRnaSeqMetrics
    INPUT="$bam"
    OUTPUT="${prefix}.rnaseq_metrics"
    CHART_OUTPUT="${prefix}.rnaseq_metrics.pdf"
    REFERENCE_SEQUENCE="$REFERENCE_SEQUENCE"
    REF_FLAT="$REF_FLAT"
    RIBOSOMAL_INTERVALS="$t"
    STRAND_SPECIFICITY=NONE
    VALIDATION_STRINGENCY=LENIENT
    TMP_DIR="$TEMP_DIR"
  )
  run_picard ${args[*]}
  # Remove the temporary file after we're done with it.
  if ! rm -f "$t"; then
    warn "WARNING: Could not delete '$t'"
  fi
}

gcbias_metrics() {
  local bam="$1"
  local prefix=$(basename "$bam" .bam)
  local args=(
    CollectGcBiasMetrics
    INPUT="$bam"
    OUTPUT="${prefix}.gc_bias_histogram"
    CHART_OUTPUT="${prefix}.gc_bias_histogram.pdf"
    SUMMARY_OUTPUT="${prefix}.gc_bias_metrics"
    REFERENCE_SEQUENCE="$REFERENCE_SEQUENCE"
    VALIDATION_STRINGENCY=LENIENT
    TMP_DIR="$TEMP_DIR"
  )
  run_picard ${args[*]}
}

library_complexity() {
  local bam="$1"
  local prefix=$(basename "$bam" .bam)
  local args=(
    EstimateLibraryComplexity
    INPUT="$bam"
    OUTPUT="${prefix}.library_complexity"
    VALIDATION_STRINGENCY=LENIENT
    TMP_DIR="$TEMP_DIR"
  )
  run_picard ${args[*]}
}

# Collate functions. ---------------------------------------------------------

collate_alignment_metrics() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.alignment_summary_metrics' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} alignment_summary_metrics files"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## METRICS' "${files[0]}" | tail -n1 | cut -f1-22
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A1000000 '## METRICS' "$f" | tail -n+3 | \
        grep -Pv '^$' | cut -f1-22 | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_rnaseq_metrics() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.rnaseq_metrics' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} rnaseq_metrics files (summary)"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## METRICS' "${files[0]}" | tail -n1 | cut -f1-22
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A2 '## METRICS' "$f" | tail -n1 | cut -f1-22 \
        | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_rnaseq_coverage() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.rnaseq_metrics' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} rnaseq_metrics files (coverage)"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## HISTOGRAM' "${files[0]}" | tail -n1 | perl -pe 's/\s+$/\n/'
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A1000000 '## HISTOGRAM' "$f" | tail -n+3 \
        | grep -Pv '^$' | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_quality_metrics() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.quality_distribution_metrics' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} quality_distribution_metrics files"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## HISTOGRAM' "${files[0]}" | tail -n1 | perl -pe 's/\s+$/\n/'
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A1000000 '## HISTOGRAM' "$f" | tail -n+3 | grep -Pv '^$' \
        | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_gc_bias_histogram() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.gc_bias_histogram' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} gc_bias_histogram files"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## METRICS' "${files[0]}" | tail -n1 | perl -pe 's/\s+$/\n/'
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A1000000 '## METRICS' "$f" | tail -n+3 | grep -Pv '^$' \
        | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_gc_bias_metrics() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.gc_bias_metrics' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} gc_bias_metrics files"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## METRICS' "${files[0]}" | tail -n1 | perl -pe 's/\s+$/\n/'
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A2 '## METRICS' "$f" | tail -n1 \
        | perl -pe 's{^}{'$sample'\t};s/\s+$/\n/'
    done
  ) > "$out"
}

collate_insert_size_metrics() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.insert_size_metrics' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} insert_size_metrics files"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## METRICS' "${files[0]}" | tail -n1 | cut -f1-18
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A2 '## METRICS' "$f" | tail -n1 | cut -f1-18 \
        | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_insert_size_histogram() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.insert_size_metrics' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} insert_size_metrics files (histogram)"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## HISTOGRAM' "${files[0]}" | tail -n1 | perl -pe 's/\s+$/\n/'
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A1000000 '## HISTOGRAM' "$f" | tail -n+3 \
        | grep -Pv '^$' | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_base_distribution_by_cycle() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.base_distribution_by_cycle_metrics' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} base_distribution_by_cycle files"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## METRICS' "${files[0]}" | tail -n1 | cut -f1-7
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A1000000 '## METRICS' "$f" | tail -n+3 | cut -f1-7 \
        | grep -Pv '^$' | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_duplicate_metrics() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.duplicate_metrics' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} duplicate_metrics files"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## METRICS' "${files[0]}" | tail -n1 | cut -f1-9
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A2 '## METRICS' "$f" | tail -n1 | cut -f1-9 \
        | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_library_complexity() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.library_complexity' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} library_complexity files"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## METRICS' "${files[0]}" | tail -n1 | cut -f1-9
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A2 '## METRICS' "$f" | tail -n1 | cut -f1-9 \
        | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_library_complexity_histogram() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.library_complexity' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} library_complexity files (histogram)"
  (
    echo -ne "SAMPLE\t"
    grep -A1 '## HISTOGRAM' "${files[0]}" | tail -n1 | cut -f1 \
      | perl -pe 's/\s+$/\n/; s/(\S+)/$1\tfragments/;'
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      grep -A1000000 '## HISTOGRAM' "$f" | tail -n+3 \
        | grep -Pv '^$' | perl -pe 's{^}{'$sample'\t}'
    done
  ) > "$out"
}

collate_mapq_stats() {
  local out="$1" dir="$2"
  local files=($(find "$dir" -name '*.mapq_stats' | sort))
  [[ ${#files[@]} -eq 0 ]] && return 1
  warn -e "$(DATE)\tCollating ${#files[@]} mapq_stats files"
  (
    echo -e \
    "SAMPLE\tMAPQ_Mean\tMAPQ_Median\tMAPQ_SD\tMAPQ_Mode\tMAPQ_ModePercent"
    local f
    for f in ${files[*]}
    do
      local sample=$(sample_name "$f")
      echo -ne "$sample\t"
      cat $f | perl -lne '
      BEGIN{ $n = 0; $mean = 0; $median = 0; $sd = 0; $mode = 0; $m = 0; }
      /Total lines:\s+(\S+)/  && ($n = $1);
      /Ari. Mean:\s+(\S+)/  && ($mean = $1);
      /Median:\s+(\S+)/     && ($median = $1);
      /^Mode:\s+(\S+) \(N=(\d+)\)/  && ($mode = $1, $m = 100 * $2 / $n);
      /StdDev:\s+(\S+)/     && ($sd = $1);
      END{ print join "\t", ($mean, $median, $sd, $mode, $m) }
      '
    done
  ) > "$out"
}

collate_all() {
  local prefix="$1"
  warn -e "$(DATE)\tJoining all files into '$prefix-all-metrics.tsv'"
  Rscript --vanilla <(echo '
main <- function() {
  args <- commandArgs(trailingOnly = TRUE)
  prefix <- args[1]
  dat <- read_metrics(prefix)
  write_tsv(x = dat, file = sprintf("%s-all-metrics.tsv", prefix))
}

write_tsv <- function(...) {
  write.table(sep = "\t", quote = FALSE, row.names = FALSE, ...)
}

read_tsv <- function(filename, ...) {
  if (!file.exists(filename)) {
    warning("File does not exist: ", filename)
    return(NULL)
  }
  dat <- read.delim(filename, stringsAsFactors = FALSE, ...)
  return(dat)
}

read_metrics <- function(prefix) {
  dat_align_metrics = read_tsv(
    sprintf("%s-alignment-metrics.tsv", prefix)
  )
  # This file must be present. The others are optional.
  if (is.null(dat_align_metrics)) {
    stop(sprintf("Failed to read %s-alignment-metrics.tsv", prefix))
  }

  dat_duplicate_metrics = read_tsv(
    sprintf("%s-duplicate-metrics.tsv", prefix)
  )
  dat_gcbias_metrics = read_tsv(
    sprintf("%s-gc-bias-metrics.tsv", prefix)
  )
  dat_insert_size_metrics = read_tsv(
    sprintf("%s-insert-size-metrics.tsv", prefix)
  )
  dat_library_complexity = read_tsv(
    sprintf("%s-library-complexity.tsv", prefix)
  )
  is_rnaseq <- file.exists(sprintf("%s-rnaseq-metrics.tsv", prefix))
  if (is_rnaseq) {
    dat_rnaseq_metrics = read_tsv(
      sprintf("%s-rnaseq-metrics.tsv", prefix)
    )
  }
  dat_mapq_stats <- read_tsv(
    sprintf("%s-mapq-stats.tsv", prefix)
  )

  # Exclude FIRST_OF_PAIR and SECOND_OF_PAIR.
  idx = !dat_align_metrics$CATEGORY %in% c("FIRST_OF_PAIR", "SECOND_OF_PAIR")
  dat_align_metrics = dat_align_metrics[idx, ]

  if (is_rnaseq && !is.null(dat_rnaseq_metrics)) {
    # alignment_metrics and rnaseq_metrics both have this column.
    idx <- colnames(dat_rnaseq_metrics) == "PF_ALIGNED_BASES"
    colnames(dat_rnaseq_metrics)[idx] <- "PF_ALIGNED_BASES_rnaseq_metrics"

    stopifnot( all(dat_align_metrics$SAMPLE == dat_rnaseq_metrics$SAMPLE) )

    dat_align_metrics <- merge(dat_align_metrics, dat_rnaseq_metrics, by = "SAMPLE")
  }

  if (!is.null(dat_duplicate_metrics)) {
    stopifnot( all(dat_align_metrics$SAMPLE == dat_duplicate_metrics$SAMPLE) )
    dat_align_metrics <- merge(dat_align_metrics, dat_duplicate_metrics, by = "SAMPLE")
  }

  if (!is.null(dat_gcbias_metrics)) {
    stopifnot( all(dat_align_metrics$SAMPLE == dat_gcbias_metrics$SAMPLE) )
    dat_align_metrics <- merge(dat_align_metrics, dat_gcbias_metrics, by = "SAMPLE")
  }

  if (!is.null(dat_insert_size_metrics)) {
    stopifnot( all(dat_align_metrics$SAMPLE == dat_insert_size_metrics$SAMPLE) )
    dat_align_metrics <- merge(dat_align_metrics, dat_insert_size_metrics, by = "SAMPLE")
  }

  if (!is.null(dat_library_complexity)) {
    # duplicate_metrics and library_complexity share columns
    cnames <- colnames(dat_library_complexity)
    cnames[2:length(cnames)] <- paste(
      cnames[2:length(cnames)], "_library_complexity",
      sep = ""
    )
    colnames(dat_library_complexity) <- cnames

    stopifnot( all(dat_align_metrics$SAMPLE == dat_library_complexity$SAMPLE) )
    dat_align_metrics <- merge(dat_align_metrics, dat_library_complexity, by = "SAMPLE")
  }

  if (!is.null(dat_mapq_stats)) {
    stopifnot( all(dat_align_metrics$SAMPLE == dat_mapq_stats$SAMPLE) )
    dat_align_metrics <- merge(dat_align_metrics, dat_mapq_stats, by = "SAMPLE")
  }

  # Delete intermediate files that are merged into "%s-all-metrics.tsv".
  filenames <- sprintf(c(
    "%s-alignment-metrics.tsv",
    "%s-duplicate-metrics.tsv",
    "%s-gc-bias-metrics.tsv",
    "%s-insert-size-metrics.tsv",
    "%s-library-complexity.tsv",
    "%s-rnaseq-metrics.tsv",
    "%s-mapq-stats.tsv"
  ), prefix)
  sapply(filenames, function(filename) {
    if (file.exists(filename)) {
      unlink(filename, force = TRUE)
    }
  })

  return(dat_align_metrics)
}

main()
  ') $prefix
}

# Extra functions. -----------------------------------------------------------

warn() {
  echo $@ >&2
}

DATE() {
  command date +'%Y-%m-%d %H:%M:%S'
}

sample_name() {
  # Remove the file extension. 'a.b.c.def' -> 'a.b.c'
  echo "${1%.*}"
}

is_bam() {
  [[ -f "$1" && "$1" == *.bam ]]
}

bam_incomplete() {
  if [[ ! -s "$1" ]]; then
    return 1
  fi
  samtools view -H "$1" 2>&1 | grep -q 'EOF marker is absent'
  return "$?"
}

bam_sorted() {
  if [[ ! -f "$1" ]]; then
    return 1
  fi
  samtools view -H "$1" 2>&1 | grep -q 'SO:coordinate'
  return "$?"
}

bash_realpath() {
  if [[ -f "$1" ]]; then
    local f=$(basename "$1")
    local d=$(dirname "$1")
    if ! (cd "$d" && echo "$(pwd)/$f"); then
      warn "ERROR: Failed to cd '$d'"
      exit 1
    fi
  elif [[ -d "$1" ]]; then
    if ! (cd "$1" && echo "$(pwd)"); then
      warn "ERROR: Failed to cd '$1'"
      exit 1
    fi
  fi
}

main "$@"
<file_sep>PREFIX = ~/.local

.PHONY: all install man get-deps

all:
	@echo "Run 'make get-deps' to download and install dependencies in $(PREFIX)"
	@echo "Run 'make install' to install picardmetrics files in $(PREFIX)"

install:
	mkdir -p $(PREFIX)/bin/
	mkdir -p $(PREFIX)/share/man/man1/
	cp picardmetrics $(PREFIX)/bin/
	cp man/picardmetrics.1 $(PREFIX)/share/man/man1/
	cp picardmetrics.conf ~/

man: man/picardmetrics.1.ronn
	ronn \
		--manual="picardmetrics manual" \
		--organization="picardmetrics-0.2.4" \
		--date="$(shell date +%Y-%m-%d)" \
		--style=toc \
		$?

get-deps:
	./scripts/install_deps $(PREFIX)
<file_sep># picardmetrics(1) -- Run Picard tools and collate multiple metrics files

## SYNOPSIS

`picardmetrics` run [-f <FILE>] [-o <DIR>] [-r] [-k] <file.bam>

`picardmetrics` collate <PREFIX> <DIR>

`picardmetrics` refFlat <file.gtf[.gz]>

`picardmetrics` rRNA <file.gtf[.gz]>

<br>
`# Example with provided data:`

`picardmetrics` run -r -o out data/project1/sample1/sample1.bam

`picardmetrics` run -r -o out data/project1/sample2/sample2.bam

`picardmetrics` collate out/project1 out

## DESCRIPTION

**Picardmetrics** is a Bash script that simplifies calling Picard tools and
collates the different output files generated by Picard. It also has functions
for generating the two input files required by CollectRnaSeqMetrics.

In order, **picardmetrics run** will do the following:

  1.    Automatically create a sequence dictionary using your reference sequence.
  2.    Create a new temporary BAM file that you can keep with option `-k`.
  2.    Reorder the header of the BAM file to match the reference.
  3.    Sort the reads in the BAM file by coordinate.
  4.    Mark duplicates in the BAM file and report duplicate metrics.
  5.    Run up to 8 additional Picard tools.

After running the tools, use **picardmetrics collate** to merge all of the
generated metrics from multiple BAM files into tab-delimited files.
Additionally, all of these tab-delimited files are consolidated into a single
file with all metrics from all BAM files and all Picard tools.

These are the tools called by **picardmetrics**:

  *   [CreateSequenceDictionary]
  *   [ReorderSam]
  *   [SortSam]
  *   [MarkDuplicates]
  *   [CollectMultipleMetrics]
      *   [CollectAlignmentSummaryMetrics]
      *   [CollectBaseDistributionByCycle]
      *   [CollectInsertSizeMetrics]
      *   [MeanQualityByCycle]
      *   [QualityScoreDistribution]
  *   [CollectRnaSeqMetrics]
  *   [CollectGcBiasMetrics]
  *   [EstimateLibraryComplexity]

Read about the meaning of each metric: [Picard metrics definitions][definitions].

## COMMANDS AND OPTIONS

### run

`picardmetrics` run [-f <FILE>] [-o <DIR>] [-r] <file.bam>

  * `-f` <FILE>

    The configuration file with specified variables. In order of preference,
    `picardmetrics` will use (1) the specified file, (2) a file named
    `picardmetrics.conf` in the current directory or (3) in the user's HOME
    directory. If no file is found, an error will be thrown and the program
    will exit.

  * `-o` <DIR>

    Write 21 output files in this directory. By default, write to the
    current directory. The output files include:

    (with option -k) 1 BAM file sorted by coordinate and with duplicates marked<br>
    6 log files for the Picard tools<br>
    4 PDF files with plots generated by Picard tools<br>
    10 text files with metrics and histograms

  * `-r`

    The BAM file is from an RNA-seq experiment. By default, this is not true.
    When this option is used, the CollectRnaSeqMetrics Picard tool is run.
    This option requires the variables `REF_FLAT` and `RIBOSOMAL_INTERVALS` to
    be set. You should specify them in the configuration file.

  * `-k`

    Keep the output BAM file. By default, delete it after metrics are created.

### collate

`picardmetrics` collate <PREFIX> <DIR>

Collate output metrics files in <DIR> into one file with all metrics from all
Picard tools and all BAM files:

<PREFIX>-all-metrics.tsv

Also write 5 collated histogram files:

<PREFIX>-base-distribution-by-cycle-histogram.tsv<br>
<PREFIX>-gc-bias-histogram.tsv<br>
<PREFIX>-insert-size-histogram.tsv<br>
<PREFIX>-library-complexity-histogram.tsv<br>
<PREFIX>-quality-histogram.tsv

### refFlat

`picardmetrics` refFlat <file.gtf[.gz]>

Create <file.refFlat> for the `REF_FLAT` argument of the
CollectRnaSeqMetrics tool. Run this command on your optionally gzipped GTF
file, and the output file will be written to the same directory as the GTF
file.

`picardmetrics run` will automatically create the .refFlat file for you if you
define the `GTF` variable in the configuration file.

### rRNA

`picardmetrics` rRNA <file.gtf[.gz]>

Create <file.rRNA.list> for the `RIBOSOMAL_INTERVALS` argument of the
CollectRnaSeqMetrics tool. Run this command on your optionally gzipped GTF
file, and the output file will be written to the same directory as the GTF
file.

`picardmetrics run` will automatically create the .rRNA.list file for you if
you define the `GTF` variable in the configuration file.

## CONFIGURATION FILE

The `picardmetrics.conf` file must define the following variables:

  * `TEMP_DIR`

    A directory where temporary files will be written.

    The sequence dictionary is taken from the BAM header and written to a
    .list file in this directory. This file is used as the header of the
    `RIBOSOMAL_INTERVALS` file passed to CollectRnaSeqMetrics.

    A copy of the input BAM file is written to the temporary directory. Then
    ReorderSam, SortSam, and MarkDuplicates are run on this copy. By default,
    it is deleted after `picardmetrics` is done. Use option `-k` to move the
    sorted and deduplicated BAM file to the output folder `-o` instead.

  * `NICENESS`

    A number between 0 and 20 specifying the niceness to use for all jobs. Use
    a number greater than 0 to avoid interrupting interactive jobs such as
    vim or emacs.

  * `PICARD_JAR`

    The full path to a downloaded picard.jar file. Get the file here:
    https://broadinstitute.github.io/picard/index.html

  * `PICARD`

    Your preferred way to invoke Java to call Picard. For example:

    `PICARD="java -Xms5g -Xmx5g -jar $PICARD_JAR"`

  * `REFERENCE_SEQUENCE`

    The full path to the organism's genome sequence in FASTA format. Required
    for: [CollectMultipleMetrics], [CollectRnaSeqMetrics],
    [CollectGcBiasMetrics].

  * `GTF`

    Ful path to a .gtf or .gtf.gz file with gene annotations. `picardmetrics`
    will use this to automatically create a .refFlat file and .rRNA.list file.

  * `REF_FLAT` (overrides `GTF`)

    Full path to a text file with annotations of all gene features in UCSC
    format. Can be generated from a GFF or GTF file. Required for:
    [CollectRnaSeqMetrics].

  * `RIBOSOMAL_INTERVALS` (overrides `GTF`)

    Full path to a text file with genomic coordinates of all ribosomal RNA
    genes in Picard format. Required for [CollectRnaSeqMetrics].

## EXAMPLES

Here are three examples of how you can run the program:

1. Run **picardmetrics** sequentially in a for loop on multiple BAM files.

2. Run in parallel with [GNU parallel][parallel], using multiple processors or
   multiple servers.

3. Run in parallel with an [LSF] queue, distributing jobs to multiple servers.

### Example 1: Sequential

Run **picardmetrics** on the provided example BAM files:

```bash
for f in data/project1/sample?/sample?.bam; do
  picardmetrics run -r -o out $f
done
```

Collate the generated metrics files:

```bash
picardmetrics collate out/project1 out
```

Next, use the file `out/project1-all-metrics.tsv` to explore the metrics.

### Example 2: GNU parallel

Run 2 jobs in parallel:

```bash
parallel -j2 \
  picardmetrics run -o /path/to/out -r {} ::: data/project1/sample?/sample?.bam
```

If you have many files, or if you want to run jobs on multiple servers, it's
a good idea to put the full paths in a text file.

Here, we have ssh access to `server1` and `server2`. We're launching 16 jobs
on `server1` and 8 jobs on `server2`. You'll have to make sure that
**picardmetrics** is in your `PATH` on all servers.

```bash
ls /full/path/to/data/project1/sample*/sample*.bam > bams.txt
parallel -S 16/server1,8/server2 \
  picardmetrics run -r -o /path/to/out {} :::: bams.txt
```

### Example 3: LSF

I recommend you install and use `asub` (see below) to submit jobs easily. This
command will submit a job for each BAM file to the `myqueue` LSF queue.

```bash
cat bams.txt | xargs -i echo picardmetrics run -r -o /path/to/out {} \
  | asub -j picardmetrics_jobs -q myqueue
```

## SOURCE CODE

Find the source code here:<br>
<https://github.com/slowkow/picardmetrics>

## BUGS

Please report issues here:<br>
<https://github.com/slowkow/picardmetrics/issues>

## AUTHOR

<NAME> from Harvard University wrote picardmetrics. Many developers
at the Broad Institute wrote Picard. <NAME> from the Sanger Institute wrote
samtools. <NAME> from the University of Utah wrote stats.

## SEE ALSO

[Picard]<br>
[samtools]<br>
[stats]<br>
[GNU parallel][parallel]<br>
[LSF]<br>
[asub]

[parallel]: https://www.gnu.org/software/parallel/parallel_tutorial.html
[LSF]: http://www.vub.ac.be/BFUCC/LSF/
[asub]: https://github.com/lh3/asub
[picardmetrics]: https://github.com/slowkow/picardmetrics
[Picard]: https://broadinstitute.github.io/picard
[samtools]: https://github.com/samtools/samtools
[stats]: https://github.com/arq5x/filo

[definitions]: https://broadinstitute.github.io/picard/picard-metric-definitions.html

[CreateSequenceDictionary]: https://broadinstitute.github.io/picard/command-line-overview.html#CreateSequenceDictionary
[ReorderSam]: https://broadinstitute.github.io/picard/command-line-overview.html#ReorderSam
[SortSam]: https://broadinstitute.github.io/picard/command-line-overview.html#SortSam
[MarkDuplicates]: https://broadinstitute.github.io/picard/command-line-overview.html#MarkDuplicates
[CollectMultipleMetrics]: https://broadinstitute.github.io/picard/command-line-overview.html#CollectMultipleMetrics
[CollectAlignmentSummaryMetrics]: https://broadinstitute.github.io/picard/command-line-overview.html#CollectAlignmentSummaryMetrics
[CollectBaseDistributionByCycle]: https://broadinstitute.github.io/picard/command-line-overview.html#CollectBaseDistributionByCycle
[CollectInsertSizeMetrics]: https://broadinstitute.github.io/picard/command-line-overview.html#CollectInsertSizeMetrics
[MeanQualityByCycle]: https://broadinstitute.github.io/picard/command-line-overview.html#MeanQualityByCycle
[QualityScoreDistribution]: https://broadinstitute.github.io/picard/command-line-overview.html#QualityScoreDistribution
[CollectRnaSeqMetrics]: https://broadinstitute.github.io/picard/command-line-overview.html#CollectRnaSeqMetrics
[CollectGcBiasMetrics]: https://broadinstitute.github.io/picard/command-line-overview.html#CollectGcBiasMetrics
[EstimateLibraryComplexity]: https://broadinstitute.github.io/picard/command-line-overview.html#EstimateLibraryComplexity

languages: Makefile Markdown Shell YAML

repo_language: Makefile


text:
<repo_name>Marachi/TryShare_v1<file_sep>/src/javaGame/View.java
package javaGame;

/**
 * Created by User on 17.03.2016.
 */
public class View {
    // Text's constants
    public static final String INPUT_INT_DATA_BETWEEN = "Input INT value between ";
    public static final String CONGRATULATION = "Congratulations! The number is guessed!!!";
    public static final String KEEP_BORDER = "Please, TRY input your value in a specified range.";
    public static final String FAIL = "Try one more  ";

    public static final String WRONG_INPUT_INT_DATA = "Wrong input! Repeat please! ";
    //int constants
    public static final int RAND_MAX = 101;
    //console print
    public void printMessage(String message){
        System.out.println(message);
    }
    public void printMessageAndTwoInt(String message, int value1,int value2){
        System.out.println(message + value1+" and " +value2);
    }

}
<file_sep>/src/javaGame/Model.java
package javaGame;

import java.util.Random;

public class Model {

    private int start; //start of range
    private int end;    //end of range
    private int goal;   //number which should be guessed by user

    // The Program logic

    /**
     * in this method goal field will be set by
     * value
     * @param goal
     * @return sum
     */
    private void setGoal(int goal) {
        this.goal = goal;
    }

    /**
     * in this method goal field will be set by
     * random value with setGoal(int goal).
     * Value is generated randomly  in range [0,100]
     */
    public void rand(){
        this.start=0;
        this.end=100;
        Random random = new Random();
        setGoal(random.nextInt(View.RAND_MAX));
    }

    /**
     * in this method goal field will be set by
     * random value.
     * Value is generated randomly  in range [start,end]
     * @param start
     * @param end
     */
    public void rand(int start, int end){
        this.start=start;
        this.end=end;
        setGoal(start + (int)(Math.random() * ((end - start) + 1)));

    }

    /**
     * this method compares user's number
     * and goal.
     * If user failed, range will be
     * shortened considering user's number
     * @param userPoint
     * @return result
     */
    public boolean tryGuess(int userPoint){
        if(userPoint==goal){
            return true;
        }
        else{
            if(userPoint>goal)
                this.end=userPoint;
            else
                this.start=userPoint;
            return false;
        }
    }

    /**
     * this method returns start
     * @return start
     */
    public int getStart() {
        return start;
    }

    /**
     * this method returns end
     * @return end
     */
    public int getEnd() {
        return end;
    }

}

branch_name: refs/heads/master

repo_language: Java


text:
<repo_name>lnmpy/api.lnmpy.com<file_sep>/.gitattributes
#pattern  filter=crypt diff=crypt
.secret.json filter=crypt diff=crypt
<file_sep>/Makefile
all: serve

install:
	yarn install --frozen-lockfile

deploy:
	@node_modules/.bin/sls deploy --conceal

serve:
	@node_modules/.bin/sls offline start

remove:
	@node_modules/.bin/sls remove

lint:
	@node_modules/.bin/eslint .
<file_sep>/.github/workflows/npm.yml
name: Npm

on:
  push:
    branches:
    - master

jobs:

  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-node@v3
        with:
          node-version: 18
      - run: npm i yarn@latest --no-save
      - run: make install
      - run: git stash
      - run: wget https://raw.githubusercontent.com/elasticdog/transcrypt/master/transcrypt
      - run: bash ./transcrypt -c aes-256-cbc -y -p ${{secrets.TRANSCRYPT_KEY}}
      - run: make lint
      - run: make deploy
        env:
          AWS_ACCESS_KEY_ID: ${{secrets.AWS_ACCESS_KEY_ID}}
          AWS_SECRET_ACCESS_KEY: ${{secrets.AWS_SECRET_ACCESS_KEY}}

<file_sep>/utils.js
exports.ServiceNotFound = (event, context, callback) => {
  callback(null, {
    statusCode: 404,
    headers: {
      'Access-Control-Allow-Origin': '*',
      'Content-Type': 'application/json; charset=utf-8',
    },
    body: JSON.stringify({
      msg: `Resouce ${event.path} not exist`,
    }),
  });
};

exports.ServiceDenied = (event, context, callback) => {
  callback(null, {
    statusCode: 401,
    headers: {
      'Access-Control-Allow-Origin': '*',
      'Content-Type': 'application/json; charset=utf-8',
    },
    body: JSON.stringify({
      msg: `Resouce ${event.path} denied`,
    }),
  });
};

exports.RandomId = (length) => {
  let result = '';
  const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
  const charactersLength = characters.length;
  for (let i = 0; i < length; i++) { // eslint-disable-line no-plusplus
    result += characters.charAt(Math.floor(Math.random() * charactersLength));
  }
  return result;
};

exports.SafeParseJson = (target, attr) => {
  if (typeof (target[attr]) === 'string') {
    target[attr] = JSON.parse(target[attr] || '{}');
  }
};
<file_sep>/apis/url.js
const AWS = require('aws-sdk'); // eslint-disable-line import/no-extraneous-dependencies
const Utils = require('../utils');
const SECRET = require('../.secret').url.LIST_SECRET; // eslint-disable-line import/extensions

const dynamoDb = new AWS.DynamoDB.DocumentClient();

function POST(event, context, callback) {
  const timestamp = new Date().getTime();
  const data = event.body;
  if (typeof data.url !== 'string') {
    callback(null, {
      statusCode: 400,
      headers: { 'Content-Type': 'text/plain' },
      body: 'Couldn\'t create the url item.',
    });
    return;
  }

  const params = {
    TableName: process.env.DYNAMODB_TABLE,
    Item: {
      id: Utils.RandomId(8),
      url: data.url,
      checked: false,
      createdAt: timestamp,
      updatedAt: timestamp,
    },
  };

  // write the url to the database
  dynamoDb.put(params, (error) => {
    // handle potential errors
    if (error) {
      callback(null, {
        statusCode: error.statusCode || 501,
        headers: { 'Content-Type': 'text/plain' },
        body: 'Couldn\'t create the url item.',
      });
      return;
    }

    // create a response
    const response = {
      statusCode: 200,
      body: JSON.stringify(params.Item),
    };
    callback(null, response);
  });
}

function LIST(event, context, callback) {
  if (event.headers.Authorization !== SECRET) {
    return Utils.ServiceDenied(event, context, callback);
  }
  const params = {
    TableName: process.env.DYNAMODB_TABLE,
  };

  // fetch all urls from the database
  dynamoDb.scan(params, (error, result) => {
    // handle potential errors
    if (error) {
      callback(null, {
        statusCode: error.statusCode || 501,
        headers: { 'Content-Type': 'text/plain' },
        body: 'Couldn\'t fetch the urls.',
      });
      return;
    }

    // create a response
    const response = {
      statusCode: 200,
      body: JSON.stringify(result.Items),
    };
    callback(null, response);
  });
}

function GET(event, context, callback) {
  if (event.pathParameters == null) {
    return LIST(event, context, callback);
  }
  const params = {
    TableName: process.env.DYNAMODB_TABLE,
    Key: {
      id: event.pathParameters.id,
    },
  };

  // fetch url from the database
  dynamoDb.get(params, (error, result) => {
    // handle potential errors
    if (error) {
      callback(null, {
        statusCode: error.statusCode || 501,
        headers: { 'Content-Type': 'text/plain' },
        body: 'Couldn\'t fetch the url item.',
      });
      return;
    }
    if (result.Item == null) {
      Utils.ServiceNotFound(event, context, callback);
      return;
    }

    // create a response
    const response = {
      statusCode: 301,
      headers: {
        Location: result.Item.url,
      },
    };
    callback(null, response);
  });
}

function DELETE(event, context, callback) {
  if (event.headers.Authorization !== SECRET) {
    return Utils.ServiceDenied(event, context, callback);
  }
  const params = {
    TableName: process.env.DYNAMODB_TABLE,
    Key: {
      id: event.pathParameters.id,
    },
  };

  // delete the url from the database
  dynamoDb.delete(params, (error) => {
    // handle potential errors
    if (error) {
      callback(null, {
        statusCode: error.statusCode || 501,
        headers: { 'Content-Type': 'text/plain' },
        body: 'Couldn\'t remove the url item.',
      });
      return;
    }

    // create a response
    const response = {
      statusCode: 200,
      body: JSON.stringify({}),
    };
    callback(null, response);
  });
}

const SERVICE_MAPPING = {
  GET,
  POST,
  DELETE,
};

module.exports = (event, context, callback) => {
  const serviceFunction = SERVICE_MAPPING[event.httpMethod] || Utils.ServiceNotFound;
  event.body = JSON.parse(event.body);
  return serviceFunction(event, context, callback);
};

languages:
Git Attributes
Makefile
JavaScript
YAML

repo_language: Git Attributes


text:
<file_sep>varying float dist;

void main () {
  float red = dist;
  float blue = 1.0 - dist;
  gl_FragColor = vec4(red, 0.5, blue, 1.0);
}<file_sep>uniform float time;
varying float dist;

void main () {
  vec4 offset = vec4(position, 1.0);
  dist = sin(time) * 0.5 + 0.5;
  offset.xyz += normal * dist;
  gl_Position = projectionMatrix * modelViewMatrix * offset;
}
<file_sep>var THREE = require('three');
var OrbitControls = require('three-orbit-controls')(THREE);

var vertShader = require('webpack-glsl-loader!./shaders/shader.vert');
var fragShader = require('webpack-glsl-loader!./shaders/shader.frag');

function createIcosahedron() {
  // create icosahedron geometry
  var icosahedronGeometry = new THREE.IcosahedronGeometry();

  // this should be commented out after we start using the shaders
  var icosahedronMaterial = new THREE.MeshNormalMaterial();

  // create icosahedron material
  // var icosahedronMaterial = new THREE.ShaderMaterial({
  //   vertexShader: vertShader,
  //   fragmentShader: fragShader,
  //   uniforms: {
  //     time: { type: 'f', value: 0 }
  //   }
  });

  // create icosahedron mesh and add to scene
  var icosahedron = new THREE.Mesh(icosahedronGeometry, icosahedronMaterial);

  // return the icosahedron
  return icosahedron;
}

(function init() {
  var time = 0;

  // create scene
  var scene = new THREE.Scene();

  // create camera
  var fieldOfView = 45;
  var aspectRatio = window.innerWidth / window.innerHeight;
  var nearClipping = 1;
  var farClipping = 1000;
  var camera = new THREE.PerspectiveCamera(
    fieldOfView,
    aspectRatio,
    nearClipping,
    farClipping
  );
  camera.position.z = 10;

  // create controls
  var controls = new OrbitControls(camera);

  // create icosahedron
  var icosahedron = createIcosahedron();
  scene.add(icosahedron);

  // create light
  var pointLight = new THREE.PointLight(0xffffff, 1.2);
  pointLight.position.x = 10;
  pointLight.position.y = 10;
  pointLight.position.z = 10;
  scene.add(pointLight);

  // create renderer
  var renderer = new THREE.WebGLRenderer();
  renderer.setSize(window.innerWidth, window.innerHeight);
  document.getElementById('root').appendChild(renderer.domElement);

  // start render loop
  animate(scene, camera, controls, renderer, time, icosahedron);
}());

function animate(scene, camera, controls, renderer, time, icosahedron) {
  // update time variable for shader
  time += 0.05;

  // update rotation for cool spin effect
  icosahedron.rotation.x += 0.005;
  icosahedron.rotation.y += 0.005;
  icosahedron.rotation.z += 0.005;

  // update the shader time attribute
  // icosahedron.material.uniforms.time.value = time;

  requestAnimationFrame(function() {
    animate(scene, camera, controls, renderer, time, icosahedron);
  });

  controls.update();
  renderer.render(scene, camera);
}<file_sep># Exercise 1: The Basics

In this exercise, we'll setup a scene, an environment map, a bump texture map and explore materials.

## What we're building

To see the finished product:

1. Download the archive.

2. Unzip it and navigate to the unzipped folder in your terminal then run `npm install`.

3. Run `npm run part1-finish`. This will open the completed project.

## Getting Started

Run `npm run part1-start` in your terminal to start the development server.

The files have been setup with Three.js already installed.
If you look at the console, you can see the `THREE` object has been logged.

## Creating the Scene Object

A Scene represents the 3D world we will be building. It’s a container for all our 3D objects.

> ‼ Remember to include a capital 'S' in `THREE.Scene()` and a lowercase 's' for the variable name.

```javascript
var scene = new THREE.Scene();
```

## Creating a Camera

The camera is one of two things that will determine how we will see our 3D scene. (The other is the `renderer` and we'll create that next)

* The first parameter is the field of view,
* second is the aspect ratio,
* third is the near clipping plane and
* fourth is the far clipping plane.

> 💡 Anything outside the clipping planes won’t be visible to our camera and won’t be included in the calculations used to generate the image.
> Heres a picture of what these parameters set for a THREE.js camera:
> http://sots.brookes.ac.uk/webmsc/p00700/pngs/threejs_camera_model.png

```javascript
var camera = new THREE.PerspectiveCamera(
    45,
    window.innerWidth / window.innerHeight,
    1,
    1000
);
```

## Creating the Renderer

Now we'll need to render what the camera sees. Three.js comes with a few different kinds of renderers.
Today, we will be using the WebGL renderer but there are also the canvas and SVG renderers.

```javascript
var renderer = new THREE.WebGLRenderer();
```

We also need to set the size of the renderer, which will determine the size of the rendered image.

```javascript
renderer.setSize(window.innerWidth, window.innerHeight);
```

And then append the `renderer`’s `domElement` attribute to the `#root` div, already present in our `index.HTML` file.

```javascript
document.getElementById("root").appendChild(renderer.domElement);
```

And now we can call the `render()` method of the `renderer` object to get it to display something.

```javascript
renderer.render(scene, camera);
```

If we run this now, we should see a black screen.
This is good! This is our 3D world, there’s just nothing to look at yet.

## Creating a Sphere

When creating a 3D object in Three.js, we need to define the object's shape and material separately.
Then they get combined to create a 3D object, and finally it needs to be added to the scene.

Before the `renderer` code, we will add the following code.

```javascript
var sphereGeometry = new THREE.SphereGeometry();
var sphereMaterial = new THREE.MeshBasicMaterial({ color: 0xffbb00 });
var sphere = new THREE.Mesh(sphereGeometry, sphereMaterial);
scene.add(sphere);
```

If we look at our scene, we won’t be able to see anything yet.
This is because the `camera` and the `sphere` we just created are overlapping each other in the center of the scene.

Let’s fix this by moving the camera towards us, on the z axis.

```javascript
camera.position.z = 10;
```

When no parameters are set for a `SphereGeometry`, it defaults to a polygonal sphere and aren't very pretty.
Let’s add some more vertices to it to make it look more round.

* The first parameter is the size (defaults to 1),
* The second parameter is the width segments and
* the third is the height segments.

Add parameters to match the following code.

```javascript
var sphereGeometry = new THREE.SphereGeometry(1, 48, 48);
```

> 💡 To demonstrate how segments work, go to the three.js docs and try playing with the example.
> https://threejs.org/docs/#api/geometries/SphereGeometry

## The MeshBasicMaterial

Right now, the ball just looks like a flat circle.
It’s 3D, but because the `MeshBasicMaterial` isn’t affected by light, it’s always 100% lit up in all directions and so it looks flat.

## The MeshLambertMaterial

Let's change the material from a `MeshBasicMaterial` to `MeshLambertMaterial`. The lambert material has a matt finish, like rubber or clay.

```javascript
var sphereMaterial = new THREE.MeshLambertMaterial({ color: 0xffbb00 });
```

If we check our scene, everything should be black again. This is good! It's back because there's no light to illuminate our sphere.

## Creating a Light

There are 6 different kinds of lights in Three.js.
Today, we'll create a `PointLight`. The point light is like a light bulb: It shines light in all directions, away from a single point.

We’ll create a variable called `pointLight` (lowercase `p` in the `var` name), set its color, intensity and position, and finally we'll add it to the scene.

```javascript
var pointLight = new THREE.PointLight(0xffffff, 1.2);
pointLight.position.x = 10;
pointLight.position.y = 10;
pointLight.position.z = 10;
scene.add(pointLight);
```

So now we can see our ball again with light shining on it little bit from the side.
It looks 3D, but it would be better if we could see it from different angles.

## Setting up OrbitControls

`OrbitControls` is a package made for Three.js to set up basic click and drag controls for a scene.
We'll need to do a couple things for it to work. The first is initializing the controls.
After the camera code, add the following code.

```javascript
var controls = new OrbitControls(camera);
```

## Setting up the Render Loop

There’s one more thing we’ll need to set up before we can have the controls working.

Right now, the `renderer.render()` is being run once, to display a single static image.
To be able to update the scene, we’ll need to update it recursively (over and over) for each frame.
We could use a `setInterval()` function, but `requestAnimationFrame()` has optimizations for this specific purpose, like pausing when we're not viewing the webpage.

```javascript
animate();

function animate() {
    requestAnimationFrame(function() {
        animate();
    });
    controls.update();
    renderer.render(scene,camera);
}
```

We can take out the old `render.render()` that's outside the `animate()` function.
Now, the controls are updated with each frame and the renderer renders the image.

If we look at our scene, we should be able to drag around and zoom in and out to get a better view of what’s going on.

Now that we have `orbitcontrols`, lets play around with another kind of material.

## The MeshPhongMaterial

So far, we’ve looked at the `MeshBasicMaterial`, which is always fully illuminated and the `MeshLambertMaterial` which is has a dull, matt finish.

Lets now try the `MeshPhongMaterial` which adds a `shininess` parameter.

Modify the `sphereMaterial` to be a `MeshPhongMaterial` and add a `shininess` parameter, after the color, and set it to 10.

```javascript
var sphereMaterial = new THREE.MeshPhongMaterial({ color: 0xffbb00, shininess: 10 });
```

If we look at our scene, the sphere looks a little shiny. Let's jack up the `shininess` to `1000`.

Now our ball very shiny, like a pool ball or something.

Our ball is looking like the moon here with its one dark side which isn't a bad thing but let's see what we can do about it.

If we add a couple more properties, `emissive` and `emissiveIntensity` we can have the sphere look like it's emitting light on its own, similar to the `MeshBasicMaterial`.

`emissive` is the colour the material will emit. This defaults to black. `emissiveIntensity` will change how string it is, and defaults to 1.

```javascript
var sphereMaterial = new THREE.MeshPhongMaterial({
  color: 0xffbb00,
  shininess: 1000,
  emissive: 0xffbb00,
  emissiveIntensity: 0.2
});
```

Now we have the best of `MeshBasicMaterial` and `MeshPhongMaterial` going on.

## Creating the init() Function

For organization, many Three.js developers decide to put all the code that has to do with the initial setup of the scene in an `init()` function.

Doing this will make our code neater and takes all our variables out of the global namespace which is always a good thing.

```javascript
init();

function init() {
    // all the code, excluding the require() statements and the animate() function goes in here.
}
```

Make sure to include the `animate()` function call at the bottom of the `init()` function declaration.

We could leave it as-is, with `init()` being called before its declared, as JavaScript ‘hoists’ all functions to the top of their block scope.
But, because we only need to call it once, I like to make the function self-calling with the following code.

```javascript
(function init() {
    // all the code, excluding the require() statements and the animate() function declaration goes in here.
}());
```

This way, `init()` is called automatically.

Now that all our variables are not in the global scope, we will need to pass some of them as parameters to the `animate()` function.

At the end of the `init()` function add the following code.

```javascript
animate(controls, renderer, scene, camera);
```

And now modify the `animate()` function to match the following code.

```javascript
function animate(controls, renderer, scene, camera) {
    requestAnimationFrame(function() {
        animate(controls, renderer, scene, camera);
    });
    controls.update();
    renderer.render(scene, camera);
};
```

Now that that’s all done, everything is looking clean and organized.

## Rotations and Creating a Plane

Before we can setup shadows, we first need something for the sphere to cast a shadow onto. So let’s make a plane under the sphere.

For now we'll use a `meshLambertMaterial`, but feel free to use a `meshPhongMaterial` and a shininess level to see what it looks like.
We'll be using yet another material later though.

The only new thing we're doing here is setting a `rotation` property.

> 💡 Three.js uses radians (not degrees) to calculate angles. Radians are based on the mathematical constant Pi.
> A rotation value of `Math.PI` is equal to 180 degrees. So the below rotation value rotates the plane by -90 degrees.

Add the following code under the sphere code.

```javascript
var planeGeometry = new THREE.PlaneGeometry(10, 10);
var planeMaterial = new THREE.MeshLambertMaterial({ color: 0x0055ff });
var plane = new THREE.Mesh(planeGeometry, planeMaterial);
plane.position.y = -1;
plane.rotation.x = -Math.PI / 2;
scene.add(plane);
```

## Setting up Shadows

To render shadows, we need do a few different things.

First, we need to tell the renderer that shadows will be rendered.

Add the following code just under the `renderer` code.

```javascript
renderer.shadowMap.enabled = true;
```

Next, we'll tell the light to cast shadows.

Add the following code just under the `pointLight` code.

```javascript
pointLight.castShadow = true;
```

Finally, we'll set which objects should cast shadows and which objects should receive them.

Add the following code just under the `plane` code.

```javascript
plane.receiveShadow = true;
```

And finally, add the following code just under the `sphere` code.

```javascript
sphere.castShadow = true;
```

The shadows are looking a little pixely, so let’s increase their resolution.

The default is 512 so let’s try doubling it. If you’re feeling confident, you can quadruple it. Or make it whatever you want.

Add the following code just under the `pointLight` code.

```javascript
pointLight.shadow.mapSize.width = 1024;
pointLight.shadow.mapSize.height = 1024;
```

## The MeshStandardMaterial

There’s one more material we’ll try out and it’s called the standard material. This material is considered an industry standard, hence its name.

It has `roughness`, `metalness`, and `opacity` parameters, along with `color`. Let’s try them out on the sphere.

`Roughness`, `metalness` and `opacity` all take a numbers from 0 to 1. Let’s set `roughness` and `metalness` to `0` and `opacity` to `1`.

Where we have declared the `sphereMaterial`, replace it with this code.

```javascript
var sphereMaterial = new THREE.MeshStandardMaterial({
    color: 0xffbb00,
    roughness: 0,
    metalness: 0
});
```

Try playing with the `roughness` and `metalness` values to see how it affects the look of the sphere.

## Adding Textures

Let’s try adding a texture to our plane. We’ll first create an instance of `TextureLoader` so we can use it more than once.

Add the following variable above the `scene` variable.

```javascript
var textureLoader = new THREE.TextureLoader();
```

Now we can load in a texture and set it as an overlay on our `planeMaterial`.

There's a concrete texture called `rock.jpg` inside the assets folder. Let's use that.
(it's named rock because I kept spelling concrete wrong :s)

```javascript
planeMaterial.map = textureLoader.load('assets/rock.jpg');
```

That looks cool, but it’s a little flat looking. We can fix that with a bump map.

We’ll need to change our plane material to a `MeshStandardMaterial`.
This is the only material bump maps are available for.
Let’s also set it’s `roughness` to `0.8` for a realistic concrete look.

```javascript
var planeMaterial = new THREE.MeshStandardMaterial({
  color: 0xffffff,
  roughness: 0.8
});
planeMaterial.bumpMap = textureLoader.load('assets/rock.jpg');
```

Okay, that’s a liiiittle too bumpy. Let’s set its `bumpScale` value to 0.02.

```javascript
planeMaterial.bumpScale = 0.02;
```

Now if we pan our scene around, the plane looks just the right amount of bumpy and rough to look like concrete.

## Setting up the Background

Backgrounds (or environment maps, or cube maps) are created from 6 images that are put together in a cube shape that will surround the scene.

You can convert any panoramic or spherical photo into a cube map to import into three.js for use as an environment map.
For today, there’s already an example cube map for us to use as our background inside the assets folder.

First we need to import all of the faces of the cube from our assets folder. Let’s do that above the `scene` variable.

```javascript
var cubeTexture = new THREE.CubeTextureLoader().load([
    'assets/cube/px.png', 'assets/cube/nx.png',
    'assets/cube/py.png', 'assets/cube/ny.png',
    'assets/cube/pz.png', 'assets/cube/nz.png'
]);
```

Now that that’s done, we can set the scene background to the `cubeTexture`.

```javascript
scene.background = cubeTexture;
```

And we can set the `envMap` (environment map) property of our `sphereMaterial` to the `cubeTexture`, to give it something to reflect.

```javascript
sphereMaterial.envMap = cubeTexture;
```

Try playing with the `metalness` and `roughness` values of the `sphereMaterial` to see how it affects the reflection of the background.

And we're done with this exercise!

Now it's time to learn the basics of shaders!!

# Exercise 2: Shaders

Shaders are custom materials that you can make with the power of math!

## What we're building

Run `npm run part2-finish` to check out the finished product.

This is a very simple example of the power of shaders.
The colour and position of each face of the object is being updated each frame based on a variables being passed into the shader code from Three.js

## Getting Started

Run `npm run part2-start` in your terminal.

## What we're working with

The `main.js` in `threejs_tutorial/shaders/start` has a basic scene already set up.
This is so we can see what the object looks like before our custom shaders are applied.

Right now the object has a `MeshNormalMaterial` applied to it.
This material is mainly used for testing and doesn't require light to be visible.

### 🤔 What are Shaders?

Shaders are custom materials.
Shaders are written in the OpenGL Shading Language (GLSL).
There are a few different kinds of shaders, today we'll be using vertex shaders and fragment shaders.

### 🤔 What are Vertex Shaders?

A vertex is where two or more lines meet. So for example, a triangle as three vertexes.

A vertex shader is run once for each vertex of a mesh its applied to.
Also, Three.js will pass in special variables for vertex shaders, including ones that have to do with the vertex position.

### 🤔 What are Fragment Shaders?

A fragment is similar to a pixel. Fragments are calculated when the 3D object is being rendered as a 2D image.
The fragment shader is applied to each 'fragment' the 2D image takes up of the entire rendered image.
The size covered by a fragment is related to the pixel area, but rasterization can produce multiple fragments
from the same object per-pixel, depending on various multisampling parameters and OpenGL state.

## Setting up the main.js

We need to do a couple things to our `main.js` before we start writing our shader code.

First, we need to uncomment the `ShaderMaterial` code (lines 16 to 22).

We should also comment out the `MeshNormalMaterial`, now that we're not using it.

Second, in the `animate()` function at the bottom, uncomment the following code.

```javascript
icosahedron.material.uniforms.time.value = time;
```

`time` is a variable that's been setup to start at 0 and add 0.05 each frame.
This line of code passes that variable into the vertex shader.
You can see the value of `time` and how often it's updated by checking the console, where it's being printed each frame.

In the `main.js`, the `ShaderMaterial` has a `uniforms` value that's an object.
The object contains two properties: `type` and `value`. The type is 'f' for float, and the value is set to 0 to start.

This is needed to define exactly what the shader should expect to have passed into it.

### 🤔 What is the main() function?

Both Fragment and vertex shaders need a `main()` function.

The vertex shader `main()` function gets called once for every vertex of the mesh it's applied to.

The fragment shader `main()` function gets called once* for every fragment the 3D object takes up of the final 2D rendered image.

\* or more depending on various multisampling parameters and OpenGL state.

## Adding Motion to the Faces

With the `time` variable and a little math, we are going to make the faces of the mesh move outward and back inward.

In `shader.vert` We first need to setup some variables we'll be using.

```glsl
uniform float time;
```

This initializes the `time` variable that we passed in.

### 🤔 What's a Uniform?

A `uniform` variable is used to communicate with your vertex shader from "outside".
They are variables passed into shaders by us, from our code.

Now, we'll add the following code under the `uniform` we just made, a the top of the `shader.vert`.

```glsl
varying float dist;
```

### 🤔 What's a Varying?

A `varying` variable is for passing values between vertex and fragment shaders. We'll be using `dist` soon in our fragment shader.

Now, we'll set the value of `dist`. Add the following code inside the `main()` function in `shader.vert`

```glsl
dist = sin(time) * 0.5 + 0.5;
```

If you'd like to see what that calculation does, open your terminal, write `node` then drag in the `timeSineOutput.js` file that's inside `threejs_tutorial/part2/start`.
It's a simple program that contains the same calculation and logs the output to the console.
That's what we'll be using to set the position of each face.

Now we'll add the following line of code to initialize a a variable named `offset`, which will be used in calculating the final position of the faces.

Add the following code to inside the `shader.vert`s `main()` function.

```glsl
vec4 offset = vec4(position, 1.0);
```

A `vec4` variable is a variable that contains 4 numbers.

`position` is passed in automatically by Three.js. It's a `vec3` (it contains 3 numbers).

And now, we'll add the following code to update the value stored in `offset`.

```glsl
offset.xyz += normal * dist;
```

In 3D rendering, a normal is a perpendicular line sticking out of the center of a face.
The `normal` variable is passed in from Three.js.

`.xyz` is used to only target the first three values of the `vec4` variable.

And finally, we'll add the following code to set an internal variable called `gl_Position`. Add it to the `main()` function.

```glsl
gl_Position = projectionMatrix * modelViewMatrix * offset;
```

`gl_Position` is the output position of the current vertex.

`projectionMatrix` and `modelViewMatrix` are built-in variables passed in by ThreeJS.

We won't be able to see the object yet. This is because we haven't given it any colour.

In the `shader.frag` we'll add the following code inside the `main()` function.

```glsl
gl_FragColor = vec4(1.0, 1.0, 1.0, 1.0);
```

`gl_FragColor` is the variable that determines the colour of the fragment. So the above code is setting each face to white.

And now we can see the object with it's animation.

## Updating the colour

To be able to update the colour, we'll need use the `varying` variable we setup earlier.

Add the following code to the tops of the `shader.frag`.

```glsl
varying float dist;
```

Now, the value of `dist` is shared between the two files.

Add the following code to inside the `main()` function of the `shader.frag`.

```glsl
float red = dist;
float blue = 1.0 - dist;
gl_FragColor = vec4(red, 0.5, blue, 1.0);
```

And now the colour is updating!

And we're done!

languages: JavaScript Markdown GLSL

repo_language: JavaScript