SparkVersion { SparkVersion(String versionString) { this.versionString = versionString; try { int pos = versionString.indexOf('-'); String numberPart = versionString; if (pos > 0) { numberPart = versionString.substring(0, pos); } version = Integer.parseInt(numberPart.replaceAll("\\.", "")); } catch (Exception e) { logger.error("Can not recognize Spark version " + versionString + ". Assume it's a future release", e); version = 999; } } SparkVersion(String versionString); int toNumber(); String toString(); boolean isUnsupportedVersion(); static SparkVersion fromVersionString(String versionString); boolean isPysparkSupported(); boolean isSparkRSupported(); boolean hasDataFrame(); boolean getProgress1_0(); boolean oldLoadFilesMethodName(); boolean oldSqlContextImplicits(); boolean equals(Object versionToCompare); boolean newerThan(SparkVersion versionToCompare); boolean newerThanEquals(SparkVersion versionToCompare); boolean olderThan(SparkVersion versionToCompare); boolean olderThanEquals(SparkVersion versionToCompare); static final SparkVersion SPARK_1_0_0; static final SparkVersion SPARK_1_1_0; static final SparkVersion SPARK_1_2_0; static final SparkVersion SPARK_1_3_0; static final SparkVersion SPARK_1_4_0; static final SparkVersion SPARK_1_5_0; static final SparkVersion SPARK_1_6_0; static final SparkVersion SPARK_1_7_0; static final SparkVersion MIN_SUPPORTED_VERSION; static final SparkVersion UNSUPPORTED_FUTURE_VERSION; }
@Test public void testSparkVersion() { assertEquals(SparkVersion.SPARK_1_2_0, SparkVersion.fromVersionString("1.2.0")); assertEquals(SparkVersion.SPARK_1_5_0, SparkVersion.fromVersionString("1.5.0-SNAPSHOT")); assertFalse(SparkVersion.SPARK_1_2_0.newerThan(SparkVersion.SPARK_1_2_0)); assertFalse(SparkVersion.SPARK_1_2_0.newerThan(SparkVersion.SPARK_1_3_0)); assertTrue(SparkVersion.SPARK_1_2_0.newerThan(SparkVersion.SPARK_1_1_0)); assertTrue(SparkVersion.SPARK_1_2_0.newerThanEquals(SparkVersion.SPARK_1_2_0)); assertFalse(SparkVersion.SPARK_1_2_0.newerThanEquals(SparkVersion.SPARK_1_3_0)); assertTrue(SparkVersion.SPARK_1_2_0.newerThanEquals(SparkVersion.SPARK_1_1_0)); assertFalse(SparkVersion.SPARK_1_2_0.olderThan(SparkVersion.SPARK_1_2_0)); assertFalse(SparkVersion.SPARK_1_2_0.olderThan(SparkVersion.SPARK_1_1_0)); assertTrue(SparkVersion.SPARK_1_2_0.olderThan(SparkVersion.SPARK_1_3_0)); assertTrue(SparkVersion.SPARK_1_2_0.olderThanEquals(SparkVersion.SPARK_1_2_0)); assertFalse(SparkVersion.SPARK_1_2_0.olderThanEquals(SparkVersion.SPARK_1_1_0)); assertTrue(SparkVersion.SPARK_1_2_0.olderThanEquals(SparkVersion.SPARK_1_3_0)); assertEquals(120, SparkVersion.SPARK_1_2_0.toNumber()); assertEquals("1.2.0", SparkVersion.SPARK_1_2_0.toString()); }