import AssemblyKeys._

seq(assemblySettings: _*)

name := "googlegenomics-spark-examples"

version := "1.0"

scalaVersion := "2.10.4"

scalacOptions += "-target:jvm-1.7"

val sparkVersion = "1.3.1"

val genomicsUtilsVersion = "v1beta2-0.33-SNAPSHOT"

libraryDependencies ++= Seq(
  "com.google.cloud.genomics" % "google-genomics-utils" % genomicsUtilsVersion excludeAll(
      ExclusionRule(organization = "org.mortbay.jetty", name = "servlet-api"),
      ExclusionRule(organization = "com.google.guava", name = "guava-jdk5"),
      ExclusionRule(organization = "com.fasterxml.jackson.core", name = "jackson-core"),
      ExclusionRule(organization = "javax", name="javaee-api")),
  "org.apache.spark" %% "spark-core" % sparkVersion % "provided",
  "org.apache.spark" %% "spark-mllib" % sparkVersion % "provided",
  "org.rogach" %% "scallop" % "0.9.5",
  "org.scalatest" %% "scalatest" % "2.2.1" % "test"
)

resolvers += Resolver.mavenLocal

mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
  {
    case PathList("META-INF", "io.netty.versions.properties") => MergeStrategy.first
    case x => old(x)
  }
}

import AssemblyKeys._

seq(assemblySettings: _*)

name := "googlegenomics-spark-examples"

version := "1.0"

scalaVersion := "2.10.4"

scalacOptions += "-target:jvm-1.7"

val sparkVersion = "1.3.1"

val genomicsUtilsVersion = "v1beta2-0.32"

libraryDependencies ++= Seq(
  "com.google.cloud.genomics" % "google-genomics-utils" % genomicsUtilsVersion excludeAll(
      ExclusionRule(organization = "org.mortbay.jetty", name = "servlet-api"),
      ExclusionRule(organization = "com.google.guava", name = "guava-jdk5"),
      ExclusionRule(organization = "com.fasterxml.jackson.core", name = "jackson-core"),
      ExclusionRule(organization = "javax", name="javaee-api")),
  "org.apache.spark" %% "spark-core" % sparkVersion % "provided",
  "org.apache.spark" %% "spark-mllib" % sparkVersion % "provided",
  "org.rogach" %% "scallop" % "0.9.5",
  "org.scalatest" %% "scalatest" % "2.2.1" % "test"
)

resolvers += Resolver.mavenLocal

mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) =>
  {
    case PathList("META-INF", "io.netty.versions.properties") => MergeStrategy.first
    case x => old(x)
  }
}

Use the published version of utils-java

Use the published version of utils-java


import play.PlayImport.PlayKeys._

name := "lib-play"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

crossScalaVersions := Seq("2.11.8")

version := "0.0.55"

lazy val root = project
  .in(file("."))
  .enablePlugins(PlayScala)
  .settings(
    libraryDependencies ++= Seq(
      ws,
      "com.jason-goodwin" %% "authentikat-jwt" % "0.4.1",
      "org.scalatest" %% "scalatest" % "2.2.6" % "test",
      "org.scalatestplus" %% "play" % "1.4.0" % "test"
    )
)

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

import play.PlayImport.PlayKeys._

name := "lib-play"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

crossScalaVersions := Seq("2.11.8")

version := "0.1.0"

lazy val root = project
  .in(file("."))
  .enablePlugins(PlayScala)
  .settings(
    libraryDependencies ++= Seq(
      ws,
      "com.jason-goodwin" %% "authentikat-jwt" % "0.4.1",
      "org.scalatest" %% "scalatest" % "2.2.6" % "test",
      "org.scalatestplus" %% "play" % "1.4.0" % "test"
    )
)

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

autocommit: Update configuration to record version 0.1.0


name := "sbt-robovm"

organization := "org.roboscala"

val roboVersion = "1.1.0" //When changing, change also RoboVMVersion in sbtrobovm.RobovmPlugin

version := roboVersion + "-cleanup-SNAPSHOT"

scalacOptions ++= Seq("-unchecked", "-deprecation", "-Xcheckinit", "-Xfatal-warnings")

javacOptions ++= Seq("-source","6","-target","6")

publishMavenStyle := false

publishTo := {
  val scalasbt = "http://repo.scala-sbt.org/scalasbt/"
  val (name, url) = if (version.value.contains("-SNAPSHOT"))
    ("sbt-plugin-snapshots", scalasbt+"sbt-plugin-snapshots")
  else
    ("sbt-plugin-releases", scalasbt+"sbt-plugin-releases")
  Some(Resolver.url(name, new URL(url))(Resolver.ivyStylePatterns))
}

libraryDependencies += "org.robovm" % "robovm-dist-compiler" % roboVersion

libraryDependencies += "org.robovm" % "robovm-maven-resolver" % roboVersion

sbtPlugin := true

name := "sbt-robovm"

organization := "org.roboscala"

val roboVersion = "1.1.0" //When changing, change also RoboVMVersion in sbtrobovm.RobovmPlugin

version := roboVersion + "-SNAPSHOT"

scalacOptions ++= Seq("-unchecked", "-deprecation", "-Xcheckinit", "-Xfatal-warnings")

javacOptions ++= Seq("-source","6","-target","6")

publishMavenStyle := false

publishTo := {
  val scalasbt = "http://repo.scala-sbt.org/scalasbt/"
  val (name, url) = if (version.value.contains("-SNAPSHOT"))
    ("sbt-plugin-snapshots", scalasbt+"sbt-plugin-snapshots")
  else
    ("sbt-plugin-releases", scalasbt+"sbt-plugin-releases")
  Some(Resolver.url(name, new URL(url))(Resolver.ivyStylePatterns))
}

libraryDependencies += "org.robovm" % "robovm-dist-compiler" % roboVersion

libraryDependencies += "org.robovm" % "robovm-maven-resolver" % roboVersion

sbtPlugin := true

Remove -cleanup tag from version

Remove -cleanup tag from version


name := "lib-reference-scala"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "org.scalatest" %% "scalatest" % "3.0.3" % Test
    )
  )

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}
version := "0.1.19"

name := "lib-reference-scala"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "org.scalatest" %% "scalatest" % "3.0.3" % Test
    )
  )

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}
version := "0.1.20"

autocommit: Update configuration to record version 0.1.20


import com.github.retronym.SbtOneJar._

oneJarSettings

name := "api-build"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

version := "0.1.27"

exportJars := true

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "com.typesafe.play" %% "play-json" % "2.5.13",
      "com.ning" % "async-http-client" % "1.9.40",
      "org.scalatest" %% "scalatest" % "3.0.1" % Test
    )
  )

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

import com.github.retronym.SbtOneJar._

oneJarSettings

name := "api-build"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

version := "0.1.28"

exportJars := true

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "com.typesafe.play" %% "play-json" % "2.5.13",
      "com.ning" % "async-http-client" % "1.9.40",
      "org.scalatest" %% "scalatest" % "3.0.1" % Test
    )
  )

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

autocommit: Update configuration to record version 0.1.28


name := "lib-reference-scala"

organization := "io.flow"

scalaVersion in ThisBuild := "2.12.8"

crossScalaVersions := Seq("2.12.8")

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "org.scalatest" %% "scalatest" % "3.0.5" % Test
    ),
    credentials += Credentials(
      "Artifactory Realm",
      "flow.jfrog.io",
      System.getenv("ARTIFACTORY_USERNAME"),
      System.getenv("ARTIFACTORY_PASSWORD")
    )
  )

publishTo := {
  val host = "https://flow.jfrog.io/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

version := "0.2.14"

name := "lib-reference-scala"

organization := "io.flow"

scalaVersion in ThisBuild := "2.12.8"

crossScalaVersions := Seq("2.12.8")

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "org.scalatest" %% "scalatest" % "3.0.5" % Test
    ),
    credentials += Credentials(
      "Artifactory Realm",
      "flow.jfrog.io",
      System.getenv("ARTIFACTORY_USERNAME"),
      System.getenv("ARTIFACTORY_PASSWORD")
    )
  )

publishTo := {
  val host = "https://flow.jfrog.io/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

version := "0.2.15"

autocommit: Update configuration to record version 0.2.15


import com.github.retronym.SbtOneJar._

oneJarSettings

name := "api-build"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

version := "0.0.73"

exportJars := true

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "com.typesafe.play" %% "play-json" % "2.5.5",
      "com.ning" % "async-http-client" % "1.9.39",
      "org.scalatest" %% "scalatest" % "2.2.6" % Test
    )
  )

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

import com.github.retronym.SbtOneJar._

oneJarSettings

name := "api-build"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

version := "0.0.74"

exportJars := true

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "com.typesafe.play" %% "play-json" % "2.5.5",
      "com.ning" % "async-http-client" % "1.9.39",
      "org.scalatest" %% "scalatest" % "2.2.6" % Test
    )
  )

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

autocommit: Update configuration to record version 0.0.74


organization := "com.github.nilsga"

name := "akka-persistence-elasticsearch"

version := "1.0-SNAPSHOT"

scalaVersion := "2.11.6"

libraryDependencies ++= Seq(
  "org.elasticsearch"  % "elasticsearch"             % "1.7.1",
  "com.sksamuel.elastic4s" %% "elastic4s-streams" % "1.7.4",
  "com.typesafe.akka"      %% "akka-persistence"                  % "2.4.0-RC2",
  "com.typesafe.akka"      %% "akka-slf4j"                        % "2.4.0-RC2",
  "com.typesafe.akka"      %% "akka-persistence-tck"              % "2.4.0-RC2"  % "test",
  "ch.qos.logback"         % "logback-classic" % "1.1.3" % "test"
)

organization := "com.github.nilsga"

name := "akka-persistence-elasticsearch"

version := "1.0-SNAPSHOT"

scalaVersion := "2.11.6"

licenses += ("MIT", url("http://opensource.org/licenses/MIT"))

credentials += Credentials(
  "Artifactory Realm",
  "oss.jfrog.org",
  sys.env.getOrElse("OSS_JFROG_USER", ""),
  sys.env.getOrElse("OSS_JFROG_PASS", "")
)

publishTo := {
  val jfrog = "https://oss.jfrog.org/artifactory/"
  if (isSnapshot.value)
    Some("OJO Snapshots" at jfrog + "oss-snapshot-local")
  else
    Some("OJO Releases" at jfrog + "oss-release-local")
}

libraryDependencies ++= Seq(
  "org.elasticsearch"  % "elasticsearch"             % "1.7.1",
  "com.sksamuel.elastic4s" %% "elastic4s-streams" % "1.7.4",
  "com.typesafe.akka"      %% "akka-persistence"                  % "2.4.0-RC2",
  "com.typesafe.akka"      %% "akka-slf4j"                        % "2.4.0-RC2",
  "com.typesafe.akka"      %% "akka-persistence-tck"              % "2.4.0-RC2"  % "test",
  "ch.qos.logback"         % "logback-classic" % "1.1.3" % "test"
)

Add publishing to jfrog oss

Add publishing to jfrog oss


name := "gcal-slack-update"
organization := "hugocf"
maintainer := "hugo@ferreira.cc"

scalaVersion := "2.12.2"

scalacOptions += "-deprecation"
scalacOptions += "-feature" // https://blog.threatstack.com/useful-scala-compiler-options-part-2-advanced-language-features
scalacOptions += "-Ypartial-unification" // https://typelevel.org/cats/

libraryDependencies ++= Seq(
  "com.typesafe" % "config" % "1.3.3",
  "org.scalaj" %% "scalaj-http" % "2.4.2",

  "org.scalatest" %% "scalatest" % "3.0.5" % Test withSources(),
  "org.scalacheck" %% "scalacheck" % "1.14.0" % Test withSources(),
  "org.mockito" % "mockito-core" % "2.23.4" % Test withSources())

enablePlugins(BuildInfoPlugin)
buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion)
buildInfoOptions += BuildInfoOption.BuildTime

enablePlugins(JavaAppPackaging)
mappings in Universal := (mappings in Universal).value.filter { case(jar, _) => !jar.getName.contains("scala-reflect") }

ghreleaseAssets := Seq((packageBin in Universal).value)
ghreleaseNotes := { tag => s"""See CHANGELOG [$tag](../master/CHANGELOG.md#${tag.stripPrefix("v")}) for details.""" }

name := "gcal-slack-update"
organization := "hugocf"
maintainer := "hugo@ferreira.cc"

scalaVersion := "2.12.2"

scalacOptions += "-deprecation"
scalacOptions += "-feature" // https://blog.threatstack.com/useful-scala-compiler-options-part-2-advanced-language-features
scalacOptions += "-Ypartial-unification" // https://typelevel.org/cats/

libraryDependencies ++= Seq(
  "com.typesafe" % "config" % "1.3.3",
  "org.scalaj" %% "scalaj-http" % "2.4.2",

  "org.scalatest" %% "scalatest" % "3.0.5" % Test withSources(),
  "org.scalacheck" %% "scalacheck" % "1.14.0" % Test withSources(),
  "org.mockito" % "mockito-core" % "2.23.4" % Test withSources())

enablePlugins(BuildInfoPlugin)
buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion)
buildInfoOptions += BuildInfoOption.BuildTime

enablePlugins(JavaAppPackaging)
mappings in Universal := (mappings in Universal).value.filter { case(jar, _) => !jar.getName.contains("scala-reflect") }

ghreleaseAssets := Seq((packageBin in Universal).value)
ghreleaseNotes := { tag => s"""See CHANGELOG [$tag](../master/CHANGELOG.md#${tag.replaceAll("[v.]", "")}) for details.""" }

docs: Fix release links to the changelog versions


name := "lib-apidoc-json-validation"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

version := "0.0.36"

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "com.typesafe.play" %% "play-json" % "2.5.13",
      "org.scalatest" %% "scalatest" % "3.0.1" % Test
    )
  )

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

name := "lib-apidoc-json-validation"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

version := "0.0.37"

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "com.typesafe.play" %% "play-json" % "2.5.13",
      "org.scalatest" %% "scalatest" % "3.0.1" % Test
    )
  )

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

autocommit: Update configuration to record version 0.0.37


package com.anchortab.snippet

import net.liftweb.common._
import net.liftweb.util._
  import Helpers._
import net.liftweb.mongodb.BsonDSL._

import com.anchortab.model._
import com.stripe

trait AccountDeletion {
  def deleteAccount(user: User) = {
    for {
      tabDelete <- tryo(Tab.delete("userId" -> user._id)) // Nuke tabs.
      stripeCustomerId <- (user.stripeCustomerId: Box[String]) ?~! "No stripe ID."
      stripeCustomerDelete <- tryo(stripe.Customer.retrieve(stripeCustomerId).delete) // Delete Stripe customer
      userDelete <- tryo(user.delete)
    } yield {
      true
    }
  }
}

package com.anchortab.snippet

import net.liftweb.common._
import net.liftweb.util._
  import Helpers._
import net.liftweb.mongodb.BsonDSL._

import com.anchortab.model._
import com.stripe

trait AccountDeletion extends Loggable {
  def deleteStripeCustomer(stripeCustomerId: Box[String]) = {
    for (stripeCustomerId <- stripeCustomerId) {
      tryo(stripe.Customer.retrieve(stripeCustomerId).delete) match {
        case error: EmptyBox =>
          logger.error("Error encountered while deleting Stripe customer: " + error)

        case _ =>
      }
    }
  }

  def deleteAccount(user: User) = {
    for {
      tabDelete <- tryo(Tab.delete("userId" -> user._id)) // Nuke tabs.
      _ = deleteStripeCustomer(user.stripeCustomerId)
      userDelete <- tryo(user.delete)
    } yield {
      true
    }
  }
}

Make account deletion fault-tolerent on Stripe failures.


package turksem.qasrl

object QASRLSettings {

  // used as URL parameters that indicate to the client which interface to use

  val generationTaskKey = "generation"
  val validationTaskKey = "validation"
  val dashboardTaskKey = "dashboard"

  // annotation pipeline hyperparameters

  val generationRewardCents = 7
  val generationReward = generationRewardCents * 0.01

  final def generationBonus(nValidQAs: Int) = {
    // no bonus for the first question, hence -1
    val cents = (0 until (nValidQAs - 1)).map(_ + generationRewardCents).sum
    cents * 0.01
  }

  val validationReward = 0.10
  val validationBonusPerQuestion = 0.02
  val validationBonusThreshold = 4

  final def validationBonus(numQuestions: Int) =
    math.max(0.0, validationBonusPerQuestion * (numQuestions - validationBonusThreshold))

  val generationCoverageQuestionsPerVerbThreshold = 2.0
  val generationCoverageBlockingThreshold = math.floor(generationCoverageQuestionsPerVerbThreshold * 10).toInt
  val generationCoverageGracePeriod = 15

  val generationAccuracyBlockingThreshold = 0.75
  val generationAccuracyGracePeriod = 15

  val validationAgreementBlockingThreshold = 0.70
  val validationAgreementGracePeriod = 15
}

package turksem.qasrl

object QASRLSettings {

  // used as URL parameters that indicate to the client which interface to use

  val generationTaskKey = "generation"
  val validationTaskKey = "validation"
  val dashboardTaskKey = "dashboard"

  // annotation pipeline hyperparameters

  val generationRewardCents = 5
  val generationReward = generationRewardCents * 0.01

  final def generationBonus(nValidQAs: Int) = {
    // no bonus for the first question, hence -1
    val cents = (0 until (nValidQAs - 1)).map(_ + generationRewardCents).sum
    cents * 0.01
  }

  val validationReward = 0.08
  val validationBonusPerQuestion = 0.02
  val validationBonusThreshold = 4

  final def validationBonus(numQuestions: Int) =
    math.max(0.0, validationBonusPerQuestion * (numQuestions - validationBonusThreshold))

  val generationCoverageQuestionsPerVerbThreshold = 2.0
  val generationCoverageBlockingThreshold = math.floor(generationCoverageQuestionsPerVerbThreshold * 10).toInt
  val generationCoverageGracePeriod = 15

  val generationAccuracyBlockingThreshold = 0.75
  val generationAccuracyGracePeriod = 15

  val validationAgreementBlockingThreshold = 0.70
  val validationAgreementGracePeriod = 15
}

Reduce reward back to original since speed issues might be fixed


package com.oreilly.learningsparkexamples.scala.logs;

import org.apache.spark._
import org.apache.spark.SparkContext._
import org.apache.spark.streaming._
import org.apache.spark.streaming.dstream._
import com.oreilly.learningsparkexamples.java.logs.ApacheAccessLog

/**
 * Computes various pieces of information on a sliding window form the log input
 */
object LogAnalyzerWindowed {
  def processAccessLogs(accessLogsDStream: DStream[ApacheAccessLog], opts: Config) {
    val ipAddressesDStream = accessLogsDStream.map{entry => entry.getIpAddress()}
    val ipAddressRequestCount = ipAddressesDStream.countByValueAndWindow(
      opts.getWindowDuration(), opts.getSlideDuration())
    val requestCount = accessLogsDStream.countByWindow(opts.getWindowDuration(), opts.getSlideDuration())
    requestCount.print()
    ipAddressRequestCount.print()
  }
}

package com.oreilly.learningsparkexamples.scala.logs;

import org.apache.spark._
import org.apache.spark.rdd._
import org.apache.spark.SparkContext._
import org.apache.spark.streaming._
import org.apache.spark.streaming.dstream._
import com.oreilly.learningsparkexamples.java.logs.ApacheAccessLog

/**
 * Computes various pieces of information on a sliding window form the log input
 */
object LogAnalyzerWindowed {
  def responseCodeCount(accessLogRDD: RDD[ApacheAccessLog]) = {
    accessLogRDD.map(log => (log.getResponseCode(), 1)).reduceByKey((x, y) => x + y)
  }

  def processAccessLogs(accessLogsDStream: DStream[ApacheAccessLog], opts: Config) {
    val ipAddressesDStream = accessLogsDStream.map{entry => entry.getIpAddress()}
    val ipAddressRequestCount = ipAddressesDStream.countByValueAndWindow(
      opts.getWindowDuration(), opts.getSlideDuration())
    val requestCount = accessLogsDStream.countByWindow(opts.getWindowDuration(), opts.getSlideDuration())
    requestCount.print()
    ipAddressRequestCount.print()
    val accessLogsWindow = accessLogsDStream.window(
      opts.getWindowDuration(), opts.getSlideDuration())
    accessLogsWindow.transform(rdd => responseCodeCount(rdd)).print()
  }
}

Add a scala transform example

Add a scala transform example


import sbt.Keys.version
import sbt._

object Common {
  val versionString = "1.3"
  val scalaVersionString = "2.10.3"
  val organisationString = "dvla"
  val organisationNameString = "Driver & Vehicle Licensing Agency"
  val nexus = "http://rep002-01.skyscape.preview-dvla.co.uk:8081/nexus/content/repositories"

  val scalaOptionsSeq = Seq(
    "-deprecation",
    "-unchecked",
    "-feature",
    "-Xlint",
    "-language:reflectiveCalls",
    "-Xmax-classfile-name", "128"
  )

  val projectResolvers = Seq(
    "typesafe repo" at "http://repo.typesafe.com/typesafe/releases",
    "spray repo" at "http://repo.spray.io/",
    "local nexus snapshots" at s"$nexus/snapshots",
    "local nexus releases" at s"$nexus/releases"
  )

  val publishResolver: sbt.Def.Initialize[Option[sbt.Resolver]] = version { v: String =>
    if (v.trim.endsWith("SNAPSHOT"))
      Some("snapshots" at s"$nexus/snapshots")
    else
      Some("releases" at s"$nexus/releases")
  }

  val sbtCredentials = Credentials(Path.userHome / ".sbt/.credentials")
}

import sbt.Keys.version
import sbt._

object Common {
  val versionString = "1.0-SNAPSHOT"
  val scalaVersionString = "2.10.3"
  val organisationString = "dvla"
  val organisationNameString = "Driver & Vehicle Licensing Agency"
  val nexus = "http://rep002-01.skyscape.preview-dvla.co.uk:8081/nexus/content/repositories"

  val scalaOptionsSeq = Seq(
    "-deprecation",
    "-unchecked",
    "-feature",
    "-Xlint",
    "-language:reflectiveCalls",
    "-Xmax-classfile-name", "128"
  )

  val projectResolvers = Seq(
    "typesafe repo" at "http://repo.typesafe.com/typesafe/releases",
    "spray repo" at "http://repo.spray.io/",
    "local nexus snapshots" at s"$nexus/snapshots",
    "local nexus releases" at s"$nexus/releases"
  )

  val publishResolver: sbt.Def.Initialize[Option[sbt.Resolver]] = version { v: String =>
    if (v.trim.endsWith("SNAPSHOT"))
      Some("snapshots" at s"$nexus/snapshots")
    else
      Some("releases" at s"$nexus/releases")
  }

  val sbtCredentials = Credentials(Path.userHome / ".sbt/.credentials")
}

Revert version to non-release snapshot

Revert version to non-release snapshot


package org.allenai.plugins

import sbt._
import sbt.Keys._

import scalariform.formatter.preferences.{ DoubleIndentClassDeclaration, FormattingPreferences }
import scalariform.sbt.ScalariformPlugin

object CoreSettingsPlugin extends AutoPlugin {

  // Automatically add the StylePlugin and VersionInjectorPlugin
  override def requires: Plugins = ScalariformPlugin && StylePlugin && VersionInjectorPlugin

  // Automatically enable the plugin (no need for projects to `enablePlugins(CoreSettingsPlugin)`)
  override def trigger: PluginTrigger = allRequirements

  object autoImport {
    val CoreResolvers = CoreRepositories.Resolvers
    val PublishTo = CoreRepositories.PublishTo
  }

  // These settings will be automatically applied to projects
  override def projectSettings: Seq[Setting[_]] =
    Seq(
      fork := true, // Forking for run, test is required sometimes, so fork always.
      scalaVersion := CoreDependencies.defaultScalaVersion,
      scalacOptions ++= Seq("-target:jvm-1.7", "-Xlint", "-deprecation", "-feature"),
      javacOptions ++= Seq("-source", "1.8", "-target", "1.8"),
      conflictManager := ConflictManager.strict,
      resolvers ++= CoreRepositories.Resolvers.defaults,
      dependencyOverrides ++= CoreDependencies.loggingDependencyOverrides,
      dependencyOverrides += "org.scala-lang" % "scala-library" % scalaVersion.value,
      // Override default scalariform settings.
      ScalariformPlugin.autoImport.scalariformPreferences := {
        FormattingPreferences().setPreference(DoubleIndentClassDeclaration, true)
      }
    )
}

package org.allenai.plugins

import sbt._
import sbt.Keys._

import scalariform.formatter.preferences.{ DoubleIndentClassDeclaration, FormattingPreferences }
import scalariform.sbt.ScalariformPlugin

object CoreSettingsPlugin extends AutoPlugin {

  // Automatically add the StylePlugin and VersionInjectorPlugin
  override def requires: Plugins = ScalariformPlugin && StylePlugin && VersionInjectorPlugin

  // Automatically enable the plugin (no need for projects to `enablePlugins(CoreSettingsPlugin)`)
  override def trigger: PluginTrigger = allRequirements

  object autoImport {
    val CoreResolvers = CoreRepositories.Resolvers
    val PublishTo = CoreRepositories.PublishTo
  }

  override val projectConfigurations = Seq(Configurations.IntegrationTest)

  // These settings will be automatically applied to projects
  override def projectSettings: Seq[Setting[_]] =
    Defaults.itSettings ++ Seq(
      fork := true, // Forking for run, test is required sometimes, so fork always.
      scalaVersion := CoreDependencies.defaultScalaVersion,
      scalacOptions ++= Seq("-target:jvm-1.7", "-Xlint", "-deprecation", "-feature"),
      javacOptions ++= Seq("-source", "1.8", "-target", "1.8"),
      resolvers ++= CoreRepositories.Resolvers.defaults,
      dependencyOverrides ++= CoreDependencies.loggingDependencyOverrides,
      dependencyOverrides += "org.scala-lang" % "scala-library" % scalaVersion.value,
      // Override default scalariform settings.
      ScalariformPlugin.autoImport.scalariformPreferences := {
        FormattingPreferences().setPreference(DoubleIndentClassDeclaration, true)
      }
    )
}

Use default ConflictManager instead of strict

SBT will report when a dependency has been evicted that it suspects may
be binary incompatible. The default ConflictManager will choose the
latest version of the dependency. This may become an issue when we move
to semantic versioning.


latestVersion in ThisBuild := "1.0.1"

unreleasedModuleNames in ThisBuild := Set()

binaryCompatibleVersions in ThisBuild := Set(
  "1.0.0",
  "1.0.1"
)

latestVersion in ThisBuild := "1.0.2"

unreleasedModuleNames in ThisBuild := Set()

binaryCompatibleVersions in ThisBuild := Set(
  "1.0.0",
  "1.0.1",
  "1.0.2"
)

Set latest version to 1.0.2

Set latest version to 1.0.2


latestVersion in ThisBuild := "0.10.0"
latestBinaryCompatibleVersion in ThisBuild := Some("0.10.0")
unreleasedModuleNames in ThisBuild := Set()

latestVersion in ThisBuild := "0.10.1"
latestBinaryCompatibleVersion in ThisBuild := Some("0.10.1")
unreleasedModuleNames in ThisBuild := Set()

Set latest version to 0.10.1

Set latest version to 0.10.1


package libs

import play.api.{Play, Application, PlayException}
import play.api.Play.current
import se.radley.plugin.salat.SalatPlugin
import com.mongodb.casbah.gridfs.GridFS

package object mongo {
  def gridFS(collectionName: String, sourceName:String = "default")(implicit app: Application): GridFS = {
    app.plugin[SalatPlugin].map(plugin => GridFS(plugin.source(sourceName).connection)).getOrElse(throw PlayException("SalatPlugin is not registered.", "You need to register the plugin with \"500:se.radley.plugin.salat.SalatPlugin\" in conf/play.plugins"))
  }
}

package libs

import play.api.{Play, Application, PlayException}
import play.api.Play.current
import se.radley.plugin.salat.SalatPlugin
import com.mongodb.casbah.gridfs.GridFS
import com.mongodb.casbah.{MongoDB, MongoConnection}

package object mongo {
  def gridFS(collectionName: String, sourceName:String = "default")(implicit app: Application): GridFS = {
    app.plugin[SalatPlugin].map({ plugin =>
      val connection: MongoConnection = plugin.source(sourceName).connection
      val db: MongoDB = connection(plugin.source(sourceName).db)
      GridFS(db)
    }).getOrElse(throw PlayException("SalatPlugin is not registered.", "You need to register the plugin with \"500:se.radley.plugin.salat.SalatPlugin\" in conf/play.plugins"))
  }
}

Use MongoDB not MongoConnection for gridfs


// The Typesafe repository
resolvers += "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/"

resolvers += Resolver.sonatypeRepo("public")

// Use the Play sbt plugin for Play projects
addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.4.8")

addSbtPlugin("com.typesafe.sbt" %% "sbt-web" % "1.0.0")

addSbtPlugin("com.typesafe.sbt" % "sbt-less" % "1.0.0")

addSbtPlugin("com.typesafe.sbt" % "sbt-rjs" % "1.0.1")

addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5")

addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.8.2")

Add dependency-tree plugin to be able to diagnose classpath issues.


package controllers

import play.api._
import play.api.mvc._
import play.api.libs.ws.WS
import play.api.libs.concurrent.Execution.Implicits._
import scala.concurrent.Future

object Application extends Controller {

  def index = Action {
    Ok(views.html.index())
  }

  def api(apiUrl: String) = Action.async {
    WS.url("http://localhost/api/" + apiUrl)
      .get()
      .map {
        response =>
          val contentType = response.header("Content-Type").getOrElse("text/plain")
          Ok(response.body).as(contentType)
      }
  }

}

package controllers

import play.api._
import play.api.mvc._
import play.api.libs.ws.WS
import play.api.libs.concurrent.Execution.Implicits._
import scala.concurrent.Future

object Application extends Controller {

  def index = Action {
    Ok(views.html.index())
  }

  def api(apiUrl: String) = Action.async {
    WS.url("http://localhost:8080/api/" + apiUrl)
      .get()
      .map {
        response =>
          val contentType = response.header("Content-Type").getOrElse("text/plain")
          Ok(response.body).as(contentType)
      }
  }

}

Add port for web service call

Add port for web service call


import sbt.Keys.version
import sbt._

object Common {
  val versionString = "9.16-SNAPSHOT"
  val scalaVersionString = "2.10.3"
  val organisationString = "dvla"
  val organisationNameString = "Driver & Vehicle Licensing Agency"
  val nexus = "http://rep002-01.skyscape.preview-dvla.co.uk:8081/nexus/content/repositories"

  val scalaOptionsSeq = Seq(
    "-deprecation",
    "-unchecked",
    "-feature",
    "-Xlint",
    "-language:reflectiveCalls",
    "-Xmax-classfile-name", "128"
  )

  val projectResolvers = Seq(
    "typesafe repo" at "http://repo.typesafe.com/typesafe/releases",
    "spray repo" at "http://repo.spray.io/",
    "local nexus snapshots" at s"$nexus/snapshots",
    "local nexus releases" at s"$nexus/releases"
  )

  val publishResolver: sbt.Def.Initialize[Option[sbt.Resolver]] = version { v: String =>
    if (v.trim.endsWith("SNAPSHOT"))
      Some("snapshots" at s"$nexus/snapshots")
    else if (v.trim.endsWith("LOCAL"))
      None // Do not publish local test versions that are accidentally committed.
    else
      Some("releases" at s"$nexus/releases")
  }

  val sbtCredentials = Credentials(Path.userHome / ".sbt/.credentials")

  val testProjectName = "common-test"
}

import sbt.Keys.version
import sbt._

object Common {
  val versionString = "2.16-SNAPSHOT"
  val scalaVersionString = "2.10.3"
  val organisationString = "dvla"
  val organisationNameString = "Driver & Vehicle Licensing Agency"
  val nexus = "http://rep002-01.skyscape.preview-dvla.co.uk:8081/nexus/content/repositories"

  val scalaOptionsSeq = Seq(
    "-deprecation",
    "-unchecked",
    "-feature",
    "-Xlint",
    "-language:reflectiveCalls",
    "-Xmax-classfile-name", "128"
  )

  val projectResolvers = Seq(
    "typesafe repo" at "http://repo.typesafe.com/typesafe/releases",
    "spray repo" at "http://repo.spray.io/",
    "local nexus snapshots" at s"$nexus/snapshots",
    "local nexus releases" at s"$nexus/releases"
  )

  val publishResolver: sbt.Def.Initialize[Option[sbt.Resolver]] = version { v: String =>
    if (v.trim.endsWith("SNAPSHOT"))
      Some("snapshots" at s"$nexus/snapshots")
    else if (v.trim.endsWith("LOCAL"))
      None // Do not publish local test versions that are accidentally committed.
    else
      Some("releases" at s"$nexus/releases")
  }

  val sbtCredentials = Credentials(Path.userHome / ".sbt/.credentials")

  val testProjectName = "common-test"
}

US2287: Set the version back to something sensible.


import sbt._
import Keys._

object Versions {
  val dropwizard = "0.7.1"
  val jackson = "2.3.2"
}

import sbt._
import Keys._

object Versions {
  val dropwizard = "0.7.1"
  val jackson = "2.3.3"
}

Update Jackson dependency to 2.3.3

Update Jackson dependency to 2.3.3

This is the version used by upstream Dropwizard.


package com.twitter.util

import org.scalatest.WordSpec

class CancellableTest extends WordSpec {
  "CancellableSink" should {
    "cancel once" in {
      var count = 0
      def increment: Unit = count += 1
      val s = new CancellableSink(increment)
      s.cancel()
      assert(count == 1)
      s.cancel()
      assert(count == 1)
    }
  }
}

package com.twitter.util

import org.scalatest.FunSuite

class CancellableTest extends FunSuite {

  test("Cancellable.nil is always cancelled") {
    assert(!Cancellable.nil.isCancelled)
  }

  test("Cancellable.nil cancel method") {
    Cancellable.nil.cancel()
    assert(!Cancellable.nil.isCancelled)
  }

  test("Cancellable.nil linkTo method") {
    Cancellable.nil.linkTo(Cancellable.nil)
    assert(!Cancellable.nil.isCancelled)
  }

  test("CancellableSink cancel normal case") {
    var count = 2
    def square: Unit = count *= count
    val s = new CancellableSink(square)
    assert(count == 2)
    s.cancel()
    assert(count == 4)
  }

  test("CancellableSink cancel once") {
    var count = 0
    def increment: Unit = count += 1
    val s = new CancellableSink(increment)
    s.cancel()
    assert(count == 1)
    s.cancel()
    assert(count == 1)
  }

  test("CancellableSink has not cancelled") {
    var count = 0
    def increment: Unit = count += 1
    val s = new CancellableSink(increment)
    assert(!s.isCancelled)
    assert(count == 0)
  }

  test("CancellableSink confirm cancelled") {
    var count = 0
    def increment: Unit = count += 1
    val s = new CancellableSink(increment)
    s.cancel()
    assert(s.isCancelled)
    assert(count == 1)
  }

  test("CancellableSink not support linking") {
    var count = 1
    def multiply: Unit = count *= 2
    val s1 = new CancellableSink(multiply)
    val s2 = new CancellableSink(multiply)
    assertThrows[Exception] {
      s1.linkTo(s2)
    }
    assertThrows[Exception] {
      s2.linkTo(s1)
    }
  }

}

Add more tests for Cancellable

util: Add more tests for Cancellable

Signed-off-by: Ryan O'Neill <ryano@twitter.com>

JIRA Issues: CSL-9502

Differential Revision: https://phabricator.twitter.biz/D447374


package spark

private[spark] abstract class Dependency[T](val rdd: RDD[T], val isShuffle: Boolean) extends Serializable

private[spark] abstract class NarrowDependency[T](rdd: RDD[T]) extends Dependency(rdd, false) {
  def getParents(outputPartition: Int): Seq[Int]
}

private[spark] class ShuffleDependency[K, V, C](
    val shuffleId: Int,
    @transient rdd: RDD[(K, V)],
    val aggregator: Aggregator[K, V, C],
    val partitioner: Partitioner)
  extends Dependency(rdd, true)

private[spark] class OneToOneDependency[T](rdd: RDD[T]) extends NarrowDependency[T](rdd) {
  override def getParents(partitionId: Int) = List(partitionId)
}

private[spark] class RangeDependency[T](rdd: RDD[T], inStart: Int, outStart: Int, length: Int)
  extends NarrowDependency[T](rdd) {

  override def getParents(partitionId: Int) = {
    if (partitionId >= outStart && partitionId < outStart + length) {
      List(partitionId - outStart + inStart)
    } else {
      Nil
    }
  }
}

package spark

abstract class Dependency[T](val rdd: RDD[T], val isShuffle: Boolean) extends Serializable

abstract class NarrowDependency[T](rdd: RDD[T]) extends Dependency(rdd, false) {
  def getParents(outputPartition: Int): Seq[Int]
}

class ShuffleDependency[K, V, C](
    val shuffleId: Int,
    @transient rdd: RDD[(K, V)],
    val aggregator: Aggregator[K, V, C],
    val partitioner: Partitioner)
  extends Dependency(rdd, true)

class OneToOneDependency[T](rdd: RDD[T]) extends NarrowDependency[T](rdd) {
  override def getParents(partitionId: Int) = List(partitionId)
}

class RangeDependency[T](rdd: RDD[T], inStart: Int, outStart: Int, length: Int)
  extends NarrowDependency[T](rdd) {

  override def getParents(partitionId: Int) = {
    if (partitionId >= outStart && partitionId < outStart + length) {
      List(partitionId - outStart + inStart)
    } else {
      Nil
    }
  }
}

Make dependency classes public - used by spark


/*
 * Copyright 2017-2022 Viktor Lövgren
 *
 * SPDX-License-Identifier: MIT
 */

package ciris

import com.comcast.ip4s.Host
import com.comcast.ip4s.Port
import org.http4s.Uri

package object http4s {
  implicit final val hostConfigDecoder: ConfigDecoder[String, Host] =
    ConfigDecoder[String].mapOption("Host")(Host.fromString)

  implicit final val portConfigDecoder: ConfigDecoder[String, Port] =
    ConfigDecoder[String].mapOption("Host")(Port.fromString)

  implicit final val uriConfigDecoder: ConfigDecoder[String, Uri] =
    ConfigDecoder[String].mapOption("Uri")(Uri.fromString(_).toOption)
}

/*
 * Copyright 2017-2022 Viktor Lövgren
 *
 * SPDX-License-Identifier: MIT
 */

package ciris

import com.comcast.ip4s.Host
import com.comcast.ip4s.Port
import org.http4s.Uri

package object http4s {
  implicit final val hostConfigDecoder: ConfigDecoder[String, Host] =
    ConfigDecoder[String].mapOption("Host")(Host.fromString)

  implicit final val portConfigDecoder: ConfigDecoder[String, Port] =
    ConfigDecoder[String].mapOption("Port")(Port.fromString)

  implicit final val uriConfigDecoder: ConfigDecoder[String, Uri] =
    ConfigDecoder[String].mapOption("Uri")(Uri.fromString(_).toOption)
}

Fix port decoder type name argument

Fix port decoder type name argument


import sbt._, Keys._
import com.typesafe.tools.mima.plugin.MimaPlugin
import com.typesafe.tools.mima.plugin.MimaKeys._

object MimaSettings {

  val mimaSettings = MimaPlugin.mimaDefaultSettings ++ Seq(
    previousArtifact := {
      CrossVersion.partialVersion(scalaVersion.value) match {
        case Some((2, scalaMajor)) if scalaMajor <= 11 =>
          Some(organization.value % s"${name.value}_${scalaBinaryVersion.value}" % "3.3.0")
        case _ =>
          None
      }
    }
  )

}

import sbt._, Keys._
import com.typesafe.tools.mima.plugin.MimaPlugin
import com.typesafe.tools.mima.plugin.MimaKeys._

object MimaSettings {

  val mimaSettings = MimaPlugin.mimaDefaultSettings ++ Seq(
    previousArtifact := {
      CrossVersion.partialVersion(scalaVersion.value) match {
        case Some((2, scalaMajor)) if scalaMajor <= 11 =>
          // NOTE: when we start 3.4.0 release, enable this mima setting
          // previousArtifact must always be the previous version (e.g.) verify 3.4.2 for 3.4.3 build
          // Some(organization.value % s"${name.value}_${scalaBinaryVersion.value}" % "3.4.0")
          None
        case _ =>
          None
      }
    }
  )

}

Disable MiMa for development branch

Disable MiMa for development branch


package controllers.disposal_of_vehicle

import helpers.common.CookieHelper
import play.api.test.FakeRequest
import play.api.test.Helpers._
import helpers.disposal_of_vehicle.CookieFactoryForUnitSpecs
import helpers.UnitSpec
import helpers.WithApplication
import pages.disposal_of_vehicle.DisposeFailurePage
import CookieHelper._
import scala.Some
import play.api.Play

final class DisposeFailureUnitSpec extends UnitSpec {
  "present" should {
    "display the page" in new WithApplication {
      whenReady(present) {
        r => r.header.status should equal(OK)
      }
    }

    "not display progress bar" in new WithApplication {
      contentAsString(present) should not include "Step "
    }

    "display prototype message when config set to true" in new WithApplication {
      contentAsString(present) should include("""<div class="prototype">""")
    }
  }

  private lazy val present = {
    val disposeFailure = injector.getInstance(classOf[DisposeFailure])
    val request = FakeRequest().
      withCookies(CookieFactoryForUnitSpecs.traderDetailsModel()).
      withCookies(CookieFactoryForUnitSpecs.vehicleDetailsModel()).
      withCookies(CookieFactoryForUnitSpecs.disposeFormModel()).
      withCookies(CookieFactoryForUnitSpecs.disposeTransactionId())
    disposeFailure.present(request)
  }
}

package controllers.disposal_of_vehicle

import common.ClientSideSessionFactory
import helpers.common.CookieHelper
import org.mockito.Mockito._
import play.api.test.FakeRequest
import play.api.test.Helpers._
import helpers.disposal_of_vehicle.CookieFactoryForUnitSpecs
import helpers.UnitSpec
import helpers.WithApplication
import pages.disposal_of_vehicle.DisposeFailurePage
import CookieHelper._
import utils.helpers.Config
import scala.Some
import play.api.Play

final class DisposeFailureUnitSpec extends UnitSpec {
  "present" should {
    "display the page" in new WithApplication {
      whenReady(present) {
        r => r.header.status should equal(OK)
      }
    }

    "not display progress bar" in new WithApplication {
      contentAsString(present) should not include "Step "
    }

    "display prototype message when config set to true" in new WithApplication {
      contentAsString(present) should include("""
""")
    }

    "not display prototype message when config set to false" in new WithApplication {
      val request = FakeRequest()
      implicit val clientSideSessionFactory = injector.getInstance(classOf[ClientSideSessionFactory])
      implicit val config: Config = mock[Config]
      when(config.isPrototypeBannerVisible).thenReturn(false)
      val disposeFailurePrototypeNotVisible = new DisposeFailure()

      val result = disposeFailurePrototypeNotVisible.present(request)
      contentAsString(result) should not include """
"""
    }
  }

  private lazy val present = {
    val disposeFailure = injector.getInstance(classOf[DisposeFailure])
    val request = FakeRequest().
      withCookies(CookieFactoryForUnitSpecs.traderDetailsModel()).
      withCookies(CookieFactoryForUnitSpecs.vehicleDetailsModel()).
      withCookies(CookieFactoryForUnitSpecs.disposeFormModel()).
      withCookies(CookieFactoryForUnitSpecs.disposeTransactionId())
    disposeFailure.present(request)
  }
}

Test for display prototype and progress bar

Former-commit-id: 319ca0637904b569fc01056622ae16282641812d


package io.mediachain.api

import java.util.UUID
import io.mediachain.BaseSpec
import io.mediachain.Types._
import org.specs2.specification.BeforeAll
import spray.testkit.Specs2RouteTest
import gremlin.scala._
import io.mediachain.util.orient.MigrationHelper

object LSpaceServiceSpec extends BaseSpec
  with Specs2RouteTest with LSpaceService with BeforeAll {
  def actorRefFactory = system

  val graphFactory = MigrationHelper.newInMemoryGraphFactory()

  val canonicalId = UUID.randomUUID.toString

  def beforeAll: Unit = {
    val graph = graphFactory.getTx()
    graph + Canonical(None, canonicalId)
  }

  def is =
    s2"""
       returns a canonical from GET "/" $returnsFirstCanonical
       returns a canonical by ID $returnsACanonical
       returns a canonical's rev history by ID $returnsASubtree
      """

  def returnsFirstCanonical = {
    Get("/canonicals") ~> baseRoute ~> check {
      responseAs[String] must contain("canonicalID")
    }
  }

  def returnsACanonical = pending {
    Get("/canonicals/" + canonicalId) ~> baseRoute ~> check {
      responseAs[String] must contain("canonicalID")
    }
  }
  def returnsASubtree = pending {
    Get("/canonicals/" + canonicalId + "/history") ~> baseRoute ~> check {
      responseAs[String] must contain("canonicalID")
    }
  }

}

package io.mediachain.api

import java.util.UUID
import io.mediachain.BaseSpec
import io.mediachain.Types._
import org.specs2.specification.BeforeAll
import org.specs2.matcher.JsonMatchers
import spray.testkit.Specs2RouteTest
import gremlin.scala._
import io.mediachain.util.orient.MigrationHelper

object LSpaceServiceSpec extends BaseSpec
  with Specs2RouteTest with LSpaceService with BeforeAll with JsonMatchers {
  def actorRefFactory = system

  val graphFactory = MigrationHelper.newInMemoryGraphFactory()

  val canonicalId = UUID.randomUUID.toString

  def beforeAll: Unit = {
    val graph = graphFactory.getTx()
    graph + Canonical(None, canonicalId)
  }

  def is =
    s2"""
       returns a canonical from GET "/" $returnsFirstCanonical
       returns a canonical by ID $returnsACanonical
       returns a canonical's rev history by ID $returnsASubtree
      """

  def returnsFirstCanonical = {
    Get("/canonicals") ~> baseRoute ~> check {
      responseAs[String] must /#(0) /("canonicalID" -> canonicalId)
    }
  }

  def returnsACanonical = pending {
    Get("/canonicals/" + canonicalId) ~> baseRoute ~> check {
      responseAs[String] must /("canonicalID" -> canonicalId)
    }
  }
  def returnsASubtree = pending {
    Get("/canonicals/" + canonicalId + "/history") ~> baseRoute ~> check {
      // TODO: describe tree structure here
      responseAs[String] must /("canonicalID" -> canonicalId)
    }
  }

}

Use JsonMatchers for API tests

Use JsonMatchers for API tests


package org.talkingpuffin.ui.util

import java.awt.Desktop
import java.net.URI
import java.awt._
import javax.swing.ImageIcon
import akka.actor._
import akka.actor.Actor._
import org.talkingpuffin.Main
import org.talkingpuffin.util.Loggable

object DesktopUtil extends Loggable {
  private case class Browse(uri: String)

  private val browser = actorOf(new Actor() {
    def receive = {
      case browse: Browse => {
        info("Before desktop.browse")
        Desktop.getDesktop.browse(new URI(browse.uri))
        info("After desktop.browse")
      }
    }
  }).start()

  private val trayIcon: TrayIcon = SystemTray.isSupported match {
    case true =>
      val icon = new TrayIcon(new ImageIcon(getClass.getResource("/TalkingPuffin_16.png")).getImage,Main.title)
      SystemTray.getSystemTray.add(icon)
      icon
    case _ => null
  }

  def browse(uri: String) = if (Desktop.isDesktopSupported) browser ! Browse(uri)

  def notify(message: String, header: String) = if (SystemTray.isSupported)
    trayIcon.displayMessage(header, message, TrayIcon.MessageType.INFO)
}

package org.talkingpuffin.ui.util

import java.awt.Desktop
import java.net.URI
import java.awt._
import javax.swing.ImageIcon
import akka.actor._
import akka.actor.Actor._
import org.talkingpuffin.Main
import org.talkingpuffin.util.Loggable

object DesktopUtil extends Loggable {
  private case class Browse(uri: String)

  private lazy val browser = actorOf(new Actor() {
    def receive = {
      case browse: Browse => {
        info("Before desktop.browse")
        Desktop.getDesktop.browse(new URI(browse.uri))
        info("After desktop.browse")
      }
    }
  }).start()

  private lazy val trayIcon: Option[TrayIcon] =
    if (SystemTray.isSupported) {
      val icon = new TrayIcon(new ImageIcon(getClass.getResource("/TalkingPuffin_16.png")).getImage,Main.title)
      SystemTray.getSystemTray.add(icon)
      Some(icon)
    } else None

  def browse(uri: String) = if (Desktop.isDesktopSupported) browser ! Browse(uri)

  def notify(message: String, header: String) =
    trayIcon.foreach(_.displayMessage(header, message, TrayIcon.MessageType.INFO))
}

Use Option to simplify logic

Use Option to simplify logic


addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.28")
addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "0.6.1")

addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.29")
addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "0.6.1")

Update sbt-scalajs, scalajs-compiler to 0.6.29


package com.twitter.server.util

import com.twitter.finagle.stats.StatsReceiver
import com.twitter.concurrent.Scheduler

object TwitterStats {

  def register(statsReceiver: StatsReceiver) = {
    val sched = statsReceiver.scope("scheduler")

    // Productivity is a very rough estimate of time spent not
    // blocking. It measures the proportion of a thread's execution
    // time spent on the CPU. This cannot take into account issues
    // like CPU scheduling effects.
    sched.provideGauge("productivity") {
      val cpu = Scheduler.cpuTime
      val wall = Scheduler.wallTime
      if (wall.toFloat <= 0F) 0F
      else cpu.toFloat / wall.toFloat
    }

    sched.addGauge("dispatches") {
      Scheduler.numDispatches.toFloat
    }
  }
}

package com.twitter.server.util

import com.twitter.finagle.stats.StatsReceiver
import com.twitter.concurrent.Scheduler

object TwitterStats {

  def register(statsReceiver: StatsReceiver) = {
    val sched = statsReceiver.scope("scheduler")

    // Productivity is a very rough estimate of time spent not
    // blocking. It measures the proportion of a thread's execution
    // time spent on the CPU. This cannot take into account issues
    // like CPU scheduling effects.
    sched.provideGauge("productivity") {
      val cpu = Scheduler.cpuTime
      val wall = Scheduler.wallTime
      if (wall.toFloat <= 0F) 0F
      else cpu.toFloat / wall.toFloat
    }

    sched.provideGauge("dispatches") {
      Scheduler.numDispatches.toFloat
    }
  }
}

twitter-server: Add strong reference to scheduler/dispatches gauge

Problem

`TwitterStats.register` was not keeping a strong reference to
the "scheduler/dispatches" stat which means that it could be
garbage collected.

Solution

Use `StatsReceiver.provideGauge` to keep the strong reference.

RB_ID=514883


/**
 * Copyright (C) 2010-2012 LShift Ltd.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package net.lshift.diffa.schema.migrations.steps

import org.hibernate.cfg.Configuration
import net.lshift.hibernate.migrations.MigrationBuilder
import java.sql.Types
import net.lshift.diffa.schema.migrations.HibernateMigrationStep

object Step0033 extends HibernateMigrationStep {
  def versionId = 33
  def name = "Add the collation field to Endpoints"

  def createMigration(config: Configuration) = {
    val migration = new MigrationBuilder(config)

    migration.alterTable("endpoint").addColumn("collation_type", Types.VARCHAR, 16, false, "ascii")
    migration
  }
}
                                       p

Remove random character that crept in.

Remove random character that crept in.


abstract class Coll[E] extends java.util.Collection[E] {
  override def toArray[T](a: Array[T with Object]): Array[T with Object] = ??? // error: cannot override
}

abstract class Coll[E] extends java.util.Collection[E] {
  override def toArray[T](a: Array[T with Object]): Array[T with Object] = ???
}

Remove error tag from positive test

Remove error tag from positive test

It looks like it's copypasta from when the test was added.


package knub.master_thesis

import java.io.File
import java.util.regex.Pattern

import cc.mallet.pipe.{Pipe, SerialPipes}
import cc.mallet.pipe._
import knub.master_thesis.preprocessing.UseFixedVocabulary

import scala.collection.mutable
import scala.collection.JavaConverters._
import scala.io.Source

object PreprocessingPipe {
    def pipeList(stopWordsFileName: String): mutable.ArrayBuffer[Pipe] = {
        val vocabulary = Source.fromFile("/san2/data/wikipedia/2016-06-21/vocab.txt").getLines().toSet

        mutable.ArrayBuffer[Pipe](
            new CharSequenceLowercase(),
            // Regex explanation: lowercase [lowercase punctuation]+ lowercase  --- at least three characters,
            // no punctuation at the end
            new CharSequence2TokenSequence(Pattern.compile("\\p{L}[\\p{L}\\p{P}]+\\p{L}")),
//            new UseOnlyFirstNWordsOfDocument(1000),
            new TokenSequenceRemoveStopwords(
                new File(stopWordsFileName), "UTF-8", false, false, false),
            new UseFixedVocabulary(vocabulary),
            new TokenSequence2FeatureSequence()
        )
    }
    def pipe(stopWordsFileName: String): Pipe = {
        new SerialPipes(pipeList(stopWordsFileName).asJava)

    }
}

package knub.master_thesis

import java.io.File
import java.util.regex.Pattern

import cc.mallet.pipe.{Pipe, SerialPipes}
import cc.mallet.pipe._
import knub.master_thesis.preprocessing.{UseFixedVocabulary, UseOnlyFirstNWordsOfDocument}

import scala.collection.mutable
import scala.collection.JavaConverters._
import scala.io.Source

object PreprocessingPipe {
    def pipeList(stopWordsFileName: String): mutable.ArrayBuffer[Pipe] = {
        val vocabulary = Source.fromFile("/san2/data/wikipedia/2016-06-21/vocab.txt").getLines().toSet

        mutable.ArrayBuffer[Pipe](
            new CharSequenceLowercase(),
            // Regex explanation: lowercase [lowercase punctuation]+ lowercase  --- at least three characters,
            // no punctuation at the end
            new CharSequence2TokenSequence(Pattern.compile("\\p{L}[\\p{L}\\p{P}]+\\p{L}")),
            new UseOnlyFirstNWordsOfDocument(2000),
            new TokenSequenceRemoveStopwords(
                new File(stopWordsFileName), "UTF-8", false, false, false),
//            new UseFixedVocabulary(vocabulary),
            new TokenSequence2FeatureSequence()
        )
    }
    def pipe(stopWordsFileName: String): Pipe = {
        new SerialPipes(pipeList(stopWordsFileName).asJava)

    }
}

Use 2000 first words of document

Use 2000 first words of document


name := "streamz"

organization in ThisBuild := "com.github.krasserm"

version in ThisBuild := "0.1-SNAPSHOT"

scalaVersion in ThisBuild := "2.11.2"

resolvers in ThisBuild += "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases"

scalacOptions in ThisBuild ++= Seq("-feature", "-language:higherKinds", "-language:implicitConversions")

libraryDependencies in ThisBuild ++= Seq(
  "org.scalaz.stream" %% "scalaz-stream" % Version.ScalazStream,
  "com.typesafe.akka" %% "akka-testkit"  % Version.Akka          % "test",
  "org.scalatest"     %% "scalatest"     % Version.Scalatest     % "test"
)

lazy val root = project.in(file(".")).aggregate(akkaCamel, akkaPersistence, akkaStream)

lazy val akkaCamel = project.in(file("streamz-akka-camel"))

lazy val akkaPersistence = project.in(file("streamz-akka-persistence"))

lazy val akkaStream = project.in(file("streamz-akka-stream"))

name := "streamz"

organization in ThisBuild := "com.github.krasserm"

version in ThisBuild := "0.2-SNAPSHOT"

scalaVersion in ThisBuild := "2.11.2"

resolvers in ThisBuild += "Scalaz Bintray Repo" at "http://dl.bintray.com/scalaz/releases"

scalacOptions in ThisBuild ++= Seq("-feature", "-language:higherKinds", "-language:implicitConversions")

libraryDependencies in ThisBuild ++= Seq(
  "org.scalaz.stream" %% "scalaz-stream" % Version.ScalazStream,
  "com.typesafe.akka" %% "akka-testkit"  % Version.Akka          % "test",
  "org.scalatest"     %% "scalatest"     % Version.Scalatest     % "test"
)

lazy val root = project.in(file(".")).aggregate(akkaCamel, akkaPersistence, akkaStream)

lazy val akkaCamel = project.in(file("streamz-akka-camel"))

lazy val akkaPersistence = project.in(file("streamz-akka-persistence"))

lazy val akkaStream = project.in(file("streamz-akka-stream"))

Set development version to 0.2-SNAPSHOT

Set development version to 0.2-SNAPSHOT


import play.PlayImport.PlayKeys._

name := "lib-play"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.11"

crossScalaVersions := Seq("2.11.11")

version := "0.3.31"

lazy val root = project
  .in(file("."))
  .enablePlugins(PlayScala)
  .settings(
    libraryDependencies ++= Seq(
      ws,
      filters,
      "com.jason-goodwin" %% "authentikat-jwt" % "0.4.3",
      "org.scalatestplus" %% "play" % "1.4.0" % "test"
    ),
    resolvers += "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/",
    resolvers += "scalaz-bintray" at "https://dl.bintray.com/scalaz/releases",
    resolvers += "Artifactory" at "https://flow.artifactoryonline.com/flow/libs-release/",
    credentials += Credentials(
      "Artifactory Realm",
      "flow.artifactoryonline.com",
      System.getenv("ARTIFACTORY_USERNAME"),
      System.getenv("ARTIFACTORY_PASSWORD")
    )
)

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

import play.PlayImport.PlayKeys._

name := "lib-play"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.11"

crossScalaVersions := Seq("2.11.11")

version := "0.3.32"

lazy val root = project
  .in(file("."))
  .enablePlugins(PlayScala)
  .settings(
    libraryDependencies ++= Seq(
      ws,
      filters,
      "com.jason-goodwin" %% "authentikat-jwt" % "0.4.3",
      "org.scalatestplus" %% "play" % "1.4.0" % "test"
    ),
    resolvers += "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/",
    resolvers += "scalaz-bintray" at "https://dl.bintray.com/scalaz/releases",
    resolvers += "Artifactory" at "https://flow.artifactoryonline.com/flow/libs-release/",
    credentials += Credentials(
      "Artifactory Realm",
      "flow.artifactoryonline.com",
      System.getenv("ARTIFACTORY_USERNAME"),
      System.getenv("ARTIFACTORY_PASSWORD")
    )
)

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

autocommit: Update configuration to record version 0.3.32


name := "apibuilder-validation"

organization := "io.flow"

scalaVersion in ThisBuild := "2.12.8"

crossScalaVersions := Seq("2.12.8")

version := "0.3.11"

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "com.typesafe.play" %% "play-json" % "2.6.10",
      "org.apache.commons" % "commons-compress" % "1.18",
      "org.scalatest" %% "scalatest" % "3.0.5" % Test
    ),
    credentials += Credentials(
      "Artifactory Realm",
      "flow.jfrog.io",
      System.getenv("ARTIFACTORY_USERNAME"),
      System.getenv("ARTIFACTORY_PASSWORD")
    )
  )

publishTo := {
  val host = "https://flow.jfrog.io/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

name := "apibuilder-validation"

organization := "io.flow"

scalaVersion in ThisBuild := "2.12.8"

crossScalaVersions := Seq("2.12.8")

version := "0.3.12"

lazy val root = project
  .in(file("."))
  .settings(
    libraryDependencies ++= Seq(
      "com.typesafe.play" %% "play-json" % "2.6.10",
      "org.apache.commons" % "commons-compress" % "1.18",
      "org.scalatest" %% "scalatest" % "3.0.5" % Test
    ),
    credentials += Credentials(
      "Artifactory Realm",
      "flow.jfrog.io",
      System.getenv("ARTIFACTORY_USERNAME"),
      System.getenv("ARTIFACTORY_PASSWORD")
    )
  )

publishTo := {
  val host = "https://flow.jfrog.io/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

autocommit: Update configuration to record version 0.3.12


import play.PlayScala

scalaVersion := "2.11.1"

name := "play-silhouette-seed"

version := "1.0"

libraryDependencies ++= Seq(
  "com.mohiva" %% "play-silhouette" % "1.0",
  "org.webjars" %% "webjars-play" % "2.3.0",
  "org.webjars" % "bootstrap" % "3.1.1",
  "org.webjars" % "jquery" % "1.11.0",
  "net.codingwell" %% "scala-guice" % "4.0.0-beta4",
  "com.typesafe.slick" %% "slick" % "2.1.0-M2",
  "mysql" % "mysql-connector-java" % "5.1.18",
  cache
)

lazy val root = (project in file(".")).enablePlugins(PlayScala)

import play.PlayScala

scalaVersion := "2.10.4"

name := "play-silhouette-seed"

version := "1.0"

libraryDependencies ++= Seq(
  "com.mohiva" %% "play-silhouette" % "1.0",
  "org.webjars" %% "webjars-play" % "2.3.0",
  "org.webjars" % "bootstrap" % "3.1.1",
  "org.webjars" % "jquery" % "1.11.0",
  "net.codingwell" %% "scala-guice" % "4.0.0-beta4",
  "com.typesafe.play" %% "play-slick" % "0.7.0-M1",
  "mysql" % "mysql-connector-java" % "5.1.18",
  cache
)

lazy val root = (project in file(".")).enablePlugins(PlayScala)

Replace "Slick" dependency by "play-sick" and change Scala version becase play-slick is'nt yet compatible with Scala 2.11


name := "projectname"

version := "0.1.0"

scalaVersion := "2.11.1"

crossScalaVersions := Seq("2.11.1", "2.10.4")

organization := "orgname"

libraryDependencies ++= Seq(
    "com.google.gdata"      %   "core"                  % "1.47.1",
    "ch.qos.logback"        %   "logback-classic"       % "1.1.2",
    "org.mockito"           %   "mockito-all"           % "1.9.0",
    "org.specs2"            %%  "specs2"                % "2.3.12"
)

resolvers ++= Seq(  "oss-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
				    "oss-releases"  at "https://oss.sonatype.org/content/repositories/releases",
				    "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/")

jacoco.settings

credentials += Credentials(Path.userHome / ".ivy2" / ".credentials")

publishTo := Some("Cloudbees releases" at "https://repository-themillhousegroup.forge.cloudbees.com/"+ "release")

scalariformSettings

name := "projectname"

version := "0.1.0"

scalaVersion := "2.11.1"

crossScalaVersions := Seq("2.11.1", "2.10.4")

organization := "orgname"

libraryDependencies ++= Seq(
    "com.google.gdata"      %   "core"                  % "1.47.1",
    "ch.qos.logback"        %   "logback-classic"       % "1.1.2",
    "org.mockito"           %   "mockito-all"           % "1.9.0",
    "org.specs2"            %%  "specs2"                % "2.3.12"      % "test"
)

resolvers ++= Seq(  "oss-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
                    "oss-releases"  at "https://oss.sonatype.org/content/repositories/releases",
                    "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/")

jacoco.settings

credentials += Credentials(Path.userHome / ".ivy2" / ".credentials")

publishTo := Some("Cloudbees releases" at "https://repository-themillhousegroup.forge.cloudbees.com/"+ "release")

scalariformSettings

Set the specs2 dependency scope to test

Set the specs2 dependency scope to test


package com.twitter.ostrich.stats

import scala.io.Source
import com.twitter.json.Json
import com.twitter.ostrich.admin._
import org.specs.Specification

object JsonStatsFetcherSpec extends Specification {
  def exec(args: String*) = Runtime.getRuntime.exec(args.toArray)

  val hasRuby = try {
    exec("ruby", "--version")
    true
  } catch {
    case e: Throwable => false
  }

  if (hasRuby) {
    "json_stats_fetcher.rb" should {
      var service: AdminHttpService = null
      val script = "./src/scripts/json_stats_fetcher.rb"

      doBefore {
        Stats.clearAll()
        service = new AdminHttpService(0, 20, new RuntimeEnvironment(getClass))
        service.start()
      }

      doAfter {
        service.shutdown()
        Stats.clearAll()
      }

      def getStats = {
        val process = exec(script, "-w", "-p", service.address.getPort.toString, "-n")
        process.waitFor()
        Source.fromInputStream(process.getInputStream).mkString.split("\n")
      }

      "fetch a stat" in {
        Stats.incr("bugs")
        import org.specs.util.TimeConversions._
        getStats must contain("bugs=1").eventually(60, 1.second)
      }
    }
  }
}

package com.twitter.ostrich.stats

import scala.io.Source
import com.twitter.ostrich.admin._
import org.specs.Specification

object JsonStatsFetcherSpec extends Specification {
  def exec(args: String*) = Runtime.getRuntime.exec(args.toArray)

  val hasRuby = try {
    exec("ruby", "--version")
    true
  } catch {
    case e: Throwable => false
  }

  if (hasRuby) {
    "json_stats_fetcher.rb" should {
      var service: AdminHttpService = null
      val script = "./src/scripts/json_stats_fetcher.rb"

      doBefore {
        Stats.clearAll()
        service = new AdminHttpService(0, 20, new RuntimeEnvironment(getClass))
        service.start()
      }

      doAfter {
        service.shutdown()
        Stats.clearAll()
      }

      def getStats = {
        val process = exec(script, "-w", "-p", service.address.getPort.toString, "-n")
        process.waitFor()
        Source.fromInputStream(process.getInputStream).mkString.split("\n")
      }

      "fetch a stat" in {
        Stats.addGauge("bugs") { 1 }
        getStats must contain("bugs=1")
      }
    }
  }
}

Use a gauge to avoid waiting for a latch.


import sbt._, Keys._

object Dependencies {
  val servletApiDep = "jakarta.servlet" % "jakarta.servlet-api" % "5.0.0"

  val specs2Dep = Def.setting {
    "org.specs2" %% "specs2-core" % "4.16.1"
  }

  def okHttp = "com.squareup.okhttp3" % "okhttp" % "4.10.0" :: Nil

  def integrationTestDeps = Def.setting((specs2Dep.value :: okHttp) map { _ % "test" })

  val commonsCodecVersion = "1.15"
  val scalaXmlVersion = "2.1.0"
  val commonsIoVersion = "2.11.0"
  val commonsFileUploadVersion = "1.4"
  val jettyVersion = "11.0.11"
  val nettyVersion = "4.1.78.Final"
  val scalatestVersion = "3.2.12"
  val scalatestScalacheckVersion = s"${scalatestVersion}.0"
  val json4sVersion = "4.0.5"
  val asyncHttpClientVersion = "1.8.17"
  val javaxActivationVersion = "1.1.1"
}

import sbt._, Keys._

object Dependencies {
  val servletApiDep = "jakarta.servlet" % "jakarta.servlet-api" % "5.0.0"

  val specs2Dep = Def.setting {
    "org.specs2" %% "specs2-core" % "4.16.1"
  }

  def okHttp = "com.squareup.okhttp3" % "okhttp" % "4.10.0" :: Nil

  def integrationTestDeps = Def.setting((specs2Dep.value :: okHttp) map { _ % "test" })

  val commonsCodecVersion = "1.15"
  val scalaXmlVersion = "2.1.0"
  val commonsIoVersion = "2.11.0"
  val commonsFileUploadVersion = "1.4"
  val jettyVersion = "11.0.11"
  val nettyVersion = "4.1.79.Final"
  val scalatestVersion = "3.2.12"
  val scalatestScalacheckVersion = s"${scalatestVersion}.0"
  val json4sVersion = "4.0.5"
  val asyncHttpClientVersion = "1.8.17"
  val javaxActivationVersion = "1.1.1"
}

Update netty-codec-http, netty-handler, ... to 4.1.79.Final


package com.azavea.rf.common.cache

import net.spy.memcached._
import com.typesafe.scalalogging.LazyLogging
import scala.concurrent._
import scala.concurrent.duration._
import scala.concurrent.ExecutionContext.Implicits.global

class MemcachedClientMethods(client: MemcachedClient) extends LazyLogging {
  def getOrElseUpdate[CachedType](cacheKey: String, expensiveOperation: Future[CachedType], ttl: Duration)(implicit ec: ExecutionContext): Future[CachedType] = {
    val futureCached = Future { client.asyncGet(cacheKey).get() }
    futureCached.flatMap({ value =>
      if (value != null) { // cache hit
        logger.debug(s"Cache Hit: $cacheKey - $value")
        Future.successful(value.asInstanceOf[CachedType])
      } else { // cache miss
        logger.debug(s"Cache Miss: $cacheKey" )
        val futureCached: Future[CachedType] = expensiveOperation
        futureCached.foreach{ cachedValue =>
          try {
            client.set(cacheKey, ttl.toSeconds.toInt, cachedValue)
            logger.debug(s"Cache Set: $cacheKey - $cachedValue")
          } catch {
            case e: Exception => logger.debug(s"Cache Set Error: ${e.getMessage}")
          }
        }

        futureCached
      }
    })
  }
}

package com.azavea.rf.common.cache

import net.spy.memcached._
import com.typesafe.scalalogging.LazyLogging
import scala.concurrent._
import scala.concurrent.duration._

class MemcachedClientMethods(client: MemcachedClient) extends LazyLogging {
  def getOrElseUpdate[CachedType](cacheKey: String, expensiveOperation: => Future[CachedType], ttl: Duration)(implicit ec: ExecutionContext): Future[CachedType] = blocking {
    val futureCached = Future { client.asyncGet(cacheKey).get() }
    futureCached.flatMap({ value =>
      if (value != null) { // cache hit
        logger.debug(s"Cache Hit: $cacheKey - $value")
        Future.successful(value.asInstanceOf[CachedType])
      } else { // cache miss
        logger.debug(s"Cache Miss: $cacheKey" )
        val futureCached: Future[CachedType] = expensiveOperation
        futureCached.foreach{ cachedValue =>
          try {
            client.set(cacheKey, ttl.toSeconds.toInt, cachedValue)
            logger.debug(s"Cache Set: $cacheKey - $cachedValue")
          } catch {
            case e: Exception => logger.debug(s"Cache Set Error: ${e.getMessage}")
          }
        }

        futureCached
      }
    })
  }
}

Remove global ExecutionContext from being used by memcached


package eu.pepot.eu.spark.inputsplitter.common.file

import eu.pepot.eu.spark.inputsplitter.helper.TestConstants._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.specs2.mutable._

class FilesListerSpec extends Specification {

  implicit val fs = FileSystem.getLocal(new Configuration())

  "FileLister" should {

    "list all regular files (1)" in {
      val inputDir = resourcesBaseDir("scenario-000/input")

      val inputs = FileLister.listFiles(inputDir)

      inputs.files.map(_.path.getName).toSet mustEqual
        Set(
          "big.txt",
          "small1.txt",
          "small2.txt"
        )
    }

    "list all regular files (2)" in {
      val inputDir = resourcesBaseDir("hidden")

      val inputs = FileLister.listFiles(inputDir)

      inputs.files.map(_.path.getName).toSet mustEqual
        Set(
          "_SUCCESS",
          "part-r-00000"
        )
    }

  }

}

package eu.pepot.eu.spark.inputsplitter.common.file

import java.io.FileNotFoundException

import eu.pepot.eu.spark.inputsplitter.helper.TestConstants._
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.specs2.mutable._

class FilesListerSpec extends Specification {

  implicit val fs = FileSystem.getLocal(new Configuration())

  "FileLister" should {

    "list all regular files (1)" in {
      val inputDir = resourcesBaseDir("scenario-000/input")

      val inputs = FileLister.listFiles(inputDir)

      inputs.files.map(_.path.getName).toSet mustEqual
        Set(
          "big.txt",
          "small1.txt",
          "small2.txt"
        )
    }

    "list all regular files (2)" in {
      val inputDir = resourcesBaseDir("hidden")

      val inputs = FileLister.listFiles(inputDir)

      inputs.files.map(_.path.getName).toSet mustEqual
        Set(
          "_SUCCESS",
          "part-r-00000"
        )
    }

    "throw exception if bad path" in {
      val inputDir = resourcesBaseDir("scenario-000/input-inexistent")
      FileLister.listFiles(inputDir) must throwAn[FileNotFoundException]
    }

  }

}

Test for invalid paths passed to FilesLister


package scala.tools.eclipse

package diagnostic

import org.eclipse.jface.dialogs.MessageDialog
import org.eclipse.swt.widgets.Display

object StartupDiagnostics {
  import ScalaPlugin.plugin

  val INSTALLED_VERSION_KEY = plugin.pluginId + ".diagnostic.currentPluginVersion"

  def run {
    val prefStore = plugin.getPreferenceStore
    val previousVersion = prefStore.getString(INSTALLED_VERSION_KEY)
    val currentVersion = plugin.getBundle.getVersion.toString

    println("startup diagnostics: previous version = " + previousVersion)
    println("startup diagnostics: CURRENT version = " + currentVersion)

    if (previousVersion != currentVersion) {
      prefStore.setValue(INSTALLED_VERSION_KEY, currentVersion)

      Display.getDefault asyncExec new Runnable {
        def run() {
          val result = MessageDialog.openQuestion(ScalaPlugin.getShell, "Run Scala Setup Diagnostics?",
            "Upgrade of Scala plugin detected. Run setup diagnostics?")

          if (result) {
            new DiagnosticDialog(ScalaPlugin.getShell).open
          }
        }
      }
    }
  }
}

package scala.tools.eclipse

package diagnostic

import org.eclipse.jface.dialogs.MessageDialog
import org.eclipse.jface.dialogs.IDialogConstants
import org.eclipse.swt.widgets.Display

object StartupDiagnostics {
  import ScalaPlugin.plugin

  val INSTALLED_VERSION_KEY = plugin.pluginId + ".diagnostic.currentPluginVersion"
  val ASK_DIAGNOSTICS = plugin.pluginId + ".diagnostic.askOnUpgrade"

  def run {
    val prefStore = plugin.getPreferenceStore
    val previousVersion = prefStore.getString(INSTALLED_VERSION_KEY)
    val currentVersion = plugin.getBundle.getVersion.toString
    prefStore.setDefault(ASK_DIAGNOSTICS, true)
    val askDiagnostics = prefStore.getBoolean(ASK_DIAGNOSTICS)

    println("startup diagnostics: previous version = " + previousVersion)
    println("startup diagnostics: CURRENT version = " + currentVersion)

    if (previousVersion != currentVersion) {
      prefStore.setValue(INSTALLED_VERSION_KEY, currentVersion)

      if (askDiagnostics) {
        Display.getDefault asyncExec new Runnable {
          def run() {
            val labels = Array(IDialogConstants.YES_LABEL, IDialogConstants.NO_LABEL, "Never")
            val dialog =
              new MessageDialog(ScalaPlugin.getShell, "Run Scala Setup Diagnostics?",
                null, "Upgrade of Scala plugin detected. Run setup diagnostics?", MessageDialog.QUESTION, labels, 0)
            dialog.open match {
              case 0 => // user pressed Yes
                new DiagnosticDialog(ScalaPlugin.getShell).open
              case 2 => // user pressed Never
                plugin.getPreferenceStore.setValue(ASK_DIAGNOSTICS, false)
              case _ => // user pressed close button (-1) or No (1)
            }
          }
        }
      }
    }
  }
}

Add "Never" option to startup diagnostics dialog, so that user will not be prompted to run startup diagnostics even if plugin is upgraded.


// オプショナルな属性を持つXML要素の作り方

package playground.xml

object XmlElementWithAnOptionalAttribute extends App {
  def getElem(foo: Option[String]) = <elem foo={ foo.orNull }></elem>

  def getElem2(foo: String = null) = <elem foo={ foo }></elem>

  assert(getElem(None) == <elem></elem>)
  assert(getElem(Some("bar")) == <elem foo="bar"></elem>)

  assert(getElem2() == <elem></elem>)
  assert(getElem2("bar") == <elem foo="bar"></elem>)
}

// オプショナルな属性を持つXML要素の作り方

package playground.xml

import scala.xml.Text

object XmlElementWithAnOptionalAttribute extends App {
  // orElseを使う方法
  def getElem1(foo: Option[String] = None) = <elem foo={ foo.orNull }></elem>

  // nullを使う方法
  def getElem2(foo: String = null) = <elem foo={ foo }></elem>

  // Option[xml.Text]を使う方法
  def getElem3(foo: Option[Text] = None) = <elem foo={ foo }></elem>

  assert(getElem1() == <elem></elem>)
  assert(getElem1(Some("bar")) == <elem foo="bar"></elem>)

  assert(getElem2() == <elem></elem>)
  assert(getElem2("bar") == <elem foo="bar"></elem>)

  assert(getElem3() == <elem></elem>)
  assert(getElem3(Some(Text("bar"))) == <elem foo="bar"></elem>)
}

Add an example using Option[xml.Text] to create XML element with an optional attribute

Signed-off-by: Hidehito Nozawa <e721f828e7ad4f54e139d41b31892ad174382f42@gmail.com>


import com.typesafe.sbt.SbtProguard.ProguardKeys._

proguardSettings

javaOptions in (Proguard, proguard) := Seq("-Xmx4G")

ProguardKeys.options in Proguard ++= Seq("-dontnote", "-dontwarn", "-ignorewarnings")

ProguardKeys.options in Proguard += """
-dontoptimize
-optimizations !code/simplification/arithmetic,!field/*,!class/merging/*,!code/allocation/variable
-keepnames public class ** { *; }
-keep public enum com.amazonaws.RequestClientOptions$Marker** {
    **[] $VALUES;
    public *;
}
-keepattributes SourceFile,LineNumberTable,*Annotation*,EnclosingMethod
-keep public class org.apache.commons.logging.impl.LogFactoryImpl
-keep public class org.apache.commons.logging.impl.Jdk14Logger { *** <init>(...); }
-keep public class net.shiroka.tools.ofx.aws.Lambda { *; }
"""

ProguardKeys.merge in Proguard := true

ProguardKeys.mergeStrategies in Proguard += ProguardMerge.discard("META-INF/.*".r)

ProguardKeys.mergeStrategies in Proguard += ProguardMerge.append("reference.conf")

ProguardKeys.options in Proguard += ProguardOptions.keepMain("net.shiroka.tools.ofx.*")

scalacOptions += "-target:jvm-1.7"

import com.typesafe.sbt.SbtProguard.ProguardKeys._

proguardSettings

javaOptions in (Proguard, proguard) := Seq("-Xmx4G")

ProguardKeys.options in Proguard ++= Seq("-dontnote", "-dontwarn", "-ignorewarnings")

ProguardKeys.options in Proguard += """
-dontoptimize
-optimizations !code/simplification/arithmetic,!field/*,!class/merging/*,!code/allocation/variable
-keepnames class ** { *; }
-keepnames enum ** { *; }
-keepattributes SourceFile,LineNumberTable,*Annotation*,EnclosingMethod

-keep public enum com.amazonaws.RequestClientOptions$Marker** {
    **[] $VALUES;
    public *;
}
-keep public class org.apache.commons.logging.impl.LogFactoryImpl
-keep public class org.apache.commons.logging.impl.Jdk14Logger { *** <init>(...); }

-keep public class net.shiroka.tools.ofx.aws.Lambda { *; }
"""

ProguardKeys.merge in Proguard := true

ProguardKeys.mergeStrategies in Proguard += ProguardMerge.discard("META-INF/.*".r)

ProguardKeys.mergeStrategies in Proguard += ProguardMerge.append("reference.conf")

ProguardKeys.options in Proguard += ProguardOptions.keepMain("net.shiroka.tools.ofx.*")

scalacOptions += "-target:jvm-1.7"

Add -keepnames on enum and private ones

Add -keepnames on enum and private ones


package io.flow.lint.linters

import io.flow.lint.Linter
import io.apibuilder.spec.v0.models.{Service, Union}

/**
  * Validates that every union type must have a discriminator with
  * value 'discriminator'
  */
case object UnionTypesHaveCommonDiscriminator extends Linter with Helpers {

  override def validate(service: Service): Seq[String] = {
    service.unions.
      filter(op => !ignored(op.attributes, "discriminator")).
      flatMap(validateUnion)
  }

  def validateUnion(union: Union): Seq[String] = {
    val expected = expectedDiscriminator(union.name)

    union.discriminator match {
      case None => Seq(error(union, s"Must have a discriminator with value '$expected'"))
      case Some(actual) if actual == expected => Nil
      case Some(actual) => Seq(error(union, s"Discriminator must have value '$expected' and not '$actual'"))
    }
  }

  private[this] def expectedDiscriminator(typeName: String): String = {
    if (isError(typeName)) {
      "code"
    } else {
      "discriminator"
    }
  }
}

package io.flow.lint.linters

import io.flow.lint.Linter
import io.apibuilder.spec.v0.models.{Service, Union}

/**
  * Validates that every union type must have a discriminator with
  * value 'discriminator'
  */
case object UnionTypesHaveCommonDiscriminator extends Linter with Helpers {

  override def validate(service: Service): Seq[String] = {
    service.unions.flatMap(validateUnion)
  }

  def validateUnion(union: Union): Seq[String] = {
    val expected = expectedDiscriminator(union.name)

    union.discriminator match {
      case None => Seq(error(union, s"Must have a discriminator with value '$expected'"))
      case Some(actual) if actual == expected => Nil
      case Some(actual) => Seq(error(union, s"Discriminator must have value '$expected' and not '$actual'"))
    }
  }

  private[this] def expectedDiscriminator(typeName: String): String = {
    if (isError(typeName)) {
      "code"
    } else {
      "discriminator"
    }
  }
}

Remove linter hint to skip discriminator - union types must have a discriminator

 - otherwise json serialization is odd and unstable


package org.schedoscope.scheduler.driver
import org.scalatest.FlatSpec
import org.scalatest.Matchers
import org.schedoscope.DriverTests
import org.schedoscope.test.resources.LocalTestResources
import org.schedoscope.dsl.transformations.ShellTransformation
class ShellDriverTest extends FlatSpec with Matchers {

  lazy val driver: ShellDriver = new ShellDriver()

    "ShellDriver" should "have transformation name shell" taggedAs (DriverTests) in {
    driver.transformationName shouldBe "shell"
  }

  it should "execute shell tranformations synchronously" taggedAs (DriverTests) in {
    val driverRunState = driver.runAndWait(ShellTransformation("#"))

    driverRunState shouldBe a[DriverRunSucceeded[_]]
  }

  it should "execute another shell tranformations synchronously" taggedAs (DriverTests) in {
    val driverRunState = driver.runAndWait(ShellTransformation("echo test >> /tmp/testout"))
    driverRunState shouldBe a[DriverRunSucceeded[_]]
  }
    it should "execute pig tranformations and return errors when running synchronously" taggedAs (DriverTests) in {
    val driverRunState = driver.runAndWait(ShellTransformation("exit 1"))

    driverRunState shouldBe a[DriverRunFailed[_]]
  }
}

package org.schedoscope.scheduler.driver
import org.scalatest.FlatSpec
import org.scalatest.Matchers
import org.schedoscope.DriverTests
import org.schedoscope.test.resources.LocalTestResources
import org.schedoscope.dsl.transformations.ShellTransformation
import org.schedoscope.DriverSettings
import com.typesafe.config.ConfigFactory
class ShellDriverTest extends FlatSpec with Matchers {

  lazy val driver: ShellDriver = new ShellDriver(new DriverSettings(ConfigFactory.empty(),"shell"))

    "ShellDriver" should "have transformation name shell" taggedAs (DriverTests) in {
    driver.transformationName shouldBe "shell"
  }

  it should "execute shell tranformations synchronously" taggedAs (DriverTests) in {
    val driverRunState = driver.runAndWait(ShellTransformation("#"))

    driverRunState shouldBe a[DriverRunSucceeded[_]]
  }

  it should "execute another shell tranformations synchronously" taggedAs (DriverTests) in {
    val driverRunState = driver.runAndWait(ShellTransformation("echo test >> /tmp/testout"))
    driverRunState shouldBe a[DriverRunSucceeded[_]]
  }
    it should "execute pig tranformations and return errors when running synchronously" taggedAs (DriverTests) in {
    val driverRunState = driver.runAndWait(ShellTransformation("exit 1"))

    driverRunState shouldBe a[DriverRunFailed[_]]
  }
}

Fix initialization of ShellDriver in test


package latis.server

import javax.servlet.http.HttpServletResponse
import java.io.PrintWriter
import javax.xml.ws.http.HTTPException

class ErrorWriter(response: HttpServletResponse) {

  def write(e: Throwable) = e match {

    //pass along http errors we receive
    case httpe: HTTPException => response.sendError(httpe.getStatusCode, httpe.getMessage)

    case _ => {
      response.reset //TODO: what are the side effects?
      //Note, must set status before getting output stream?
      //TODO: consider more specific errors, this is always 500
      response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR)
      response.setContentType("text/plain")

      val errorType = e.getClass.getSimpleName
      val rawMsg = e.getMessage
      val errorMsg = if (rawMsg == null) "" else rawMsg

      val writer = new PrintWriter(response.getOutputStream)
      writer.println("LaTiS Error: {")
      writer.println(s"""  $errorType: "$errorMsg"""") // hello world
      writer.println("}")
      writer.flush()
      response.flushBuffer()
    }

  }
}

object ErrorWriter {
  def apply(response: HttpServletResponse) = new ErrorWriter(response)
}

package latis.server

import javax.servlet.http.HttpServletResponse
import java.io.PrintWriter
import javax.xml.ws.http.HTTPException

class ErrorWriter(response: HttpServletResponse) {

  def write(e: Throwable) = e match {

    //pass along http errors we receive
    case httpe: HTTPException => response.sendError(httpe.getStatusCode, httpe.getMessage)

    case _ => {
      if(response.isCommitted()) {
        throw e;
      }
      else {
        response.reset //TODO: what are the side effects?
        //Note, must set status before getting output stream?
        //TODO: consider more specific errors, this is always 500
        response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR)
        response.setContentType("text/plain")

        val errorType = e.getClass.getSimpleName
        val rawMsg = e.getMessage
        val errorMsg = if (rawMsg == null) "" else rawMsg

        val writer = new PrintWriter(response.getOutputStream)
        writer.println("LaTiS Error: {")
        writer.println(s"""  $errorType: "$errorMsg"""") // hello world
        writer.println("}")
        writer.flush()
        response.flushBuffer()
      }
    }

  }
}

object ErrorWriter {
  def apply(response: HttpServletResponse) = new ErrorWriter(response)
}

Check if response is committed before resetting

Former-commit-id: ab124fc854154563828047e79035decf2fa2b829


import sbt._
import sbt.Keys._
import android.Keys._
import android.Plugin._

object Build extends android.AutoBuild
{
	lazy val main = Project( "toolbelt", file( "." ) )
		.settings( androidBuildAar: _* )
		.settings(
			libraryDependencies ++= Seq(
				"com.android.support" % "appcompat-v7" % "21.0.0",
				"com.github.japgolly.android" % "svg-android" % "2.0.6",
				"com.jpardogo.materialtabstrip" % "library" % "1.0.5",
				"org.scala-lang" % "scala-reflect" % scalaVersion.value
			),
			name := "Toolbelt",
			organization := "com.taig.android",
			publishArtifact in ( Compile, packageDoc ) := false,
			scalaVersion := "2.11.4",
			scalacOptions ++= Seq(
				"-deprecation",
				"-feature",
				"-language:dynamics",
				"-language:existentials",
				"-language:implicitConversions",
				"-language:reflectiveCalls"
			),
			// @see https://github.com/pfn/android-sdk-plugin/issues/88
			sourceGenerators in Compile <<= ( sourceGenerators in Compile ) ( generators => Seq( generators.last ) ),
			version := "0.3.17-BETA",
			minSdkVersion in Android := "10",
			platformTarget in Android := "android-21",
			targetSdkVersion in Android := "21"
		)
}

import sbt._
import sbt.Keys._
import android.Keys._
import android.Plugin._

object Build extends android.AutoBuild
{
	lazy val main = Project( "toolbelt", file( "." ) )
		.settings( androidBuildAar: _* )
		.settings(
			libraryDependencies ++= Seq(
				"com.android.support" % "appcompat-v7" % "21.0.0",
				"com.github.japgolly.android" % "svg-android" % "2.0.6",
				"com.jpardogo.materialtabstrip" % "library" % "1.0.5",
				"org.scala-lang" % "scala-reflect" % scalaVersion.value
			),
			name := "Toolbelt",
			organization := "com.taig.android",
			publishArtifact in ( Compile, packageDoc ) := false,
			scalaVersion := "2.11.4",
			scalacOptions ++= Seq(
				"-deprecation",
				"-feature",
				"-language:dynamics",
				"-language:existentials",
				"-language:implicitConversions",
				"-language:reflectiveCalls"
			),
			// @see https://github.com/pfn/android-sdk-plugin/issues/88
			sourceGenerators in Compile <<= ( sourceGenerators in Compile ) ( generators => Seq( generators.last ) ),
			version := "0.3.18-BETA",
			minSdkVersion in Android := "10",
			platformTarget in Android := "android-21",
			targetSdkVersion in Android := "21"
		)
}

Set project version to 0.3.18-BETA

Set project version to 0.3.18-BETA


import play.PlayImport.PlayKeys._

name := "lib-play"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

crossScalaVersions := Seq("2.11.8")

version := "0.1.69"

lazy val root = project
  .in(file("."))
  .enablePlugins(PlayScala)
  .settings(
    libraryDependencies ++= Seq(
      ws,
      filters,
      "com.jason-goodwin" %% "authentikat-jwt" % "0.4.3",
      "org.scalatestplus" %% "play" % "1.4.0" % "test"
    ),
    resolvers += "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/",
    resolvers += "scalaz-bintray" at "https://dl.bintray.com/scalaz/releases",
    resolvers += "Artifactory" at "https://flow.artifactoryonline.com/flow/libs-release/",
    credentials += Credentials(
      "Artifactory Realm",
      "flow.artifactoryonline.com",
      System.getenv("ARTIFACTORY_USERNAME"),
      System.getenv("ARTIFACTORY_PASSWORD")
    )
)

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

import play.PlayImport.PlayKeys._

name := "lib-play"

organization := "io.flow"

scalaVersion in ThisBuild := "2.11.8"

crossScalaVersions := Seq("2.11.8")

version := "0.1.70"

lazy val root = project
  .in(file("."))
  .enablePlugins(PlayScala)
  .settings(
    libraryDependencies ++= Seq(
      ws,
      filters,
      "com.jason-goodwin" %% "authentikat-jwt" % "0.4.3",
      "org.scalatestplus" %% "play" % "1.4.0" % "test"
    ),
    resolvers += "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/",
    resolvers += "scalaz-bintray" at "https://dl.bintray.com/scalaz/releases",
    resolvers += "Artifactory" at "https://flow.artifactoryonline.com/flow/libs-release/",
    credentials += Credentials(
      "Artifactory Realm",
      "flow.artifactoryonline.com",
      System.getenv("ARTIFACTORY_USERNAME"),
      System.getenv("ARTIFACTORY_PASSWORD")
    )
)

publishTo := {
  val host = "https://flow.artifactoryonline.com/flow"
  if (isSnapshot.value) {
    Some("Artifactory Realm" at s"$host/libs-snapshot-local;build.timestamp=" + new java.util.Date().getTime)
  } else {
    Some("Artifactory Realm" at s"$host/libs-release-local")
  }
}

autocommit: Update configuration to record version 0.1.70


package molmed.hercules.processes

import molmed.hercules.Runfolder

case class WriteTestFileToDirectoryProcess(runfolder: Runfolder)
    extends BiotankProcess {
  val command = "touch " + runfolder.runfolder + "/testfile"
}

package molmed.hercules.processes

import molmed.hercules.Runfolder

case class WriteTestFileToDirectoryProcess(runfolder: Runfolder)
    extends BiotankProcess {
  val command = "hostname > " + runfolder.runfolder + "/testfile"
}

Write the host name to the test file...

Write the host name to the test file...

... this makes it easier to see that it actually ran the command on the
correct host.


package orc.compile

class AnalysisCache {
  // TODO: Replace with LRU cache.
  val cache = collection.mutable.HashMap[(AnalysisRunner[_, _], Any), Any]()

  def get[T, P](runner: AnalysisRunner[P, T])(params: P): T = {
    cache.getOrElseUpdate((runner, params), runner.compute(this)(params)).asInstanceOf[T]
  }

  def clean(): Unit = {

  }
}

trait AnalysisRunner[P, T] {
  def compute(cache: AnalysisCache)(params: P): T
}

package orc.compile

import java.util.WeakHashMap
import scala.ref.WeakReference
import scala.ref.ReferenceQueue

class AnalysisCache {
  private val cache = collection.mutable.HashMap[AnalysisCacheKey, Any]()

  def get[P <: Product, T](runner: AnalysisRunner[P, T])(params: P): T = {
    incrClean()
    cache.getOrElseUpdate(new AnalysisCacheKey(runner, params), runner.compute(this)(params)).asInstanceOf[T]
  }

  def clean(): Unit = {
    cache.clear()
  }

  private class AssociatedWeakRef(v: AnyRef, val key: AnalysisCacheKey, q: ReferenceQueue[AnyRef]) extends WeakReference[AnyRef](v, q)

  private class AnalysisCacheKey(val analysis: AnalysisRunner[_, _], params1: Product) {
    val params = params1.productIterator.map(v => new AssociatedWeakRef(v.asInstanceOf[AnyRef], this, queue)).toSeq
    override def hashCode() = analysis.hashCode() + params.map(_().hashCode()).sum
    override def equals(o: Any) = o match {
      case o: AnalysisCacheKey => {
        analysis == o.analysis &&
          (params zip o.params).forall({ p =>
            p._1.get == p._2.get
          })
      }
      case _ => false
    }
  }

  val queue = new ReferenceQueue[AnyRef]()

  def incrClean(n: Int = 10): Unit = {
    if (n > 0) {
      queue.poll match {
        case Some(k: AssociatedWeakRef) =>
          cache -= k.key
          incrClean(n - 1)
        case Some(_) =>
          throw new AssertionError("Not an AssociatedWeakRef")
        case None =>
          ()
      }
    }
  }
}

trait AnalysisRunner[P <: Product, T] {
  def compute(cache: AnalysisCache)(params: P): T
}

Add analysis result weak caching.

Add analysis result weak caching.


/*
 * Copyright 2011-2015 Asakusa Framework Team.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.asakusafw.spark.compiler

import java.io.File

import scala.collection.mutable
import scala.collection.JavaConversions._

import com.asakusafw.lang.compiler.api.CompilerOptions
import com.asakusafw.lang.compiler.api.JobflowProcessor.{ Context => JPContext }
import com.asakusafw.lang.compiler.api.testing.MockJobflowProcessorContext
import com.asakusafw.spark.compiler.subplan.{
  BranchKeysClassBuilder,
  BroadcastIdsClassBuilder
}

trait CompilerContext {

  def newContext(
    flowId: String,
    outputDir: File): SparkClientCompiler.Context = {
    newContext(
      flowId,
      new MockJobflowProcessorContext(
        new CompilerOptions("buildid", "", Map.empty[String, String]),
        Thread.currentThread.getContextClassLoader,
        outputDir))
  }

  def newContext(
    flowId: String,
    jpContext: JPContext): SparkClientCompiler.Context = {
    SparkClientCompiler.Context(
      flowId = flowId,
      jpContext = jpContext,
      externalInputs = mutable.Map.empty,
      branchKeys = new BranchKeysClassBuilder(flowId),
      broadcastIds = new BroadcastIdsClassBuilder(flowId))
  }
}

Add license header for a new file.

Add license header for a new file.


import skinny._
import skinny.controller._
import _root_.controller._

class ScalatraBootstrap extends SkinnyLifeCycle {

  override def initSkinnyApp(ctx: ServletContext) {
    Controllers.root.mount(ctx)
    AssetsController.mount(ctx)
  }

}

import skinny._
import skinny.controller._
import _root_.controller._

class ScalatraBootstrap extends SkinnyLifeCycle {

  // If you prefer more logging, configure this settings
  scalikejdbc.GlobalSettings.loggingSQLAndTime = scalikejdbc.LoggingSQLAndTimeSettings(
    singleLineMode = true
  )

  override def initSkinnyApp(ctx: ServletContext) {
    Controllers.root.mount(ctx)
    AssetsController.mount(ctx)
  }

}

Set singleLineMode logging as default

Set singleLineMode logging as default