SparkModelParser { public static Model parse(String pathToMdl, SupportedSparkModels parsedSparkMdl) throws IllegalArgumentException { File mdlDir = IgniteUtils.resolveIgnitePath(pathToMdl); if (mdlDir == null) throw new IllegalArgumentException("Directory not found or empty [directory_path=" + pathToMdl + "]"); if (!mdlDir.isDirectory()) throw new IllegalArgumentException("Spark Model Parser supports loading from directory only. " + "The specified path " + pathToMdl + " is not the path to directory."); String[] files = mdlDir.list(); if (files.length == 0) throw new IllegalArgumentException("Directory contain 0 files and sub-directories [directory_path=" + pathToMdl + "]"); if (Arrays.stream(files).noneMatch("data"::equals)) throw new IllegalArgumentException("Directory should contain data sub-directory [directory_path=" + pathToMdl + "]"); if (Arrays.stream(files).noneMatch("metadata"::equals)) throw new IllegalArgumentException("Directory should contain metadata sub-directory [directory_path=" + pathToMdl + "]"); String pathToData = pathToMdl + File.separator + "data"; File dataDir = IgniteUtils.resolveIgnitePath(pathToData); File[] dataParquetFiles = dataDir.listFiles((dir, name) -> name.matches("^part-.*\\.snappy\\.parquet$")); if (dataParquetFiles.length == 0) throw new IllegalArgumentException("Directory should contain parquet file " + "with model [directory_path=" + pathToData + "]"); if (dataParquetFiles.length > 1) throw new IllegalArgumentException("Directory should contain only one parquet file " + "with model [directory_path=" + pathToData + "]"); String pathToMdlFile = dataParquetFiles[0].getPath(); String pathToMetadata = pathToMdl + File.separator + "metadata"; File metadataDir = IgniteUtils.resolveIgnitePath(pathToMetadata); String[] metadataFiles = metadataDir.list(); if (Arrays.stream(metadataFiles).noneMatch("part-00000"::equals)) throw new IllegalArgumentException("Directory should contain json file with model metadata " + "with name part-00000 [directory_path=" + pathToMetadata + "]"); try { validateMetadata(pathToMetadata, parsedSparkMdl); } catch (FileNotFoundException e) { throw new IllegalArgumentException("Directory should contain json file with model metadata " + "with name part-00000 [directory_path=" + pathToMetadata + "]"); } if (shouldContainTreeMetadataSubDirectory(parsedSparkMdl)) { if (Arrays.stream(files).noneMatch("treesMetadata"::equals)) throw new IllegalArgumentException("Directory should contain treeMetadata sub-directory [directory_path=" + pathToMdl + "]"); String pathToTreesMetadata = pathToMdl + File.separator + "treesMetadata"; File treesMetadataDir = IgniteUtils.resolveIgnitePath(pathToTreesMetadata); File[] treesMetadataParquetFiles = treesMetadataDir.listFiles((dir, name) -> name.matches("^part-.*\\.snappy\\.parquet$")); if (treesMetadataParquetFiles.length == 0) throw new IllegalArgumentException("Directory should contain parquet file " + "with model treesMetadata [directory_path=" + pathToTreesMetadata + "]"); if (treesMetadataParquetFiles.length > 1) throw new IllegalArgumentException("Directory should contain only one parquet file " + "with model [directory_path=" + pathToTreesMetadata + "]"); String pathToTreesMetadataFile = treesMetadataParquetFiles[0].getPath(); return parseDataWithMetadata(pathToMdlFile, pathToTreesMetadataFile, parsedSparkMdl); } else return parseData(pathToMdlFile, parsedSparkMdl); }  static Model parse(String pathToMdl, SupportedSparkModels parsedSparkMdl);  }
@Test public void failOnNullDirectory() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( "incorrectPath", SupportedSparkModels.LINEAR_REGRESSION ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Directory not found or empty")); } }
@Test public void failOnEmptyDirectory() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( url.getPath() + "empty", SupportedSparkModels.LINEAR_REGRESSION ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Directory not found or empty")); } }
@Test public void failOnEmptyOrNonExistingDataDirectory() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( url.getPath() + File.separator + "nodatafolder", SupportedSparkModels.LINEAR_REGRESSION ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Directory should contain data sub-directory")); } }
@Test public void failOnEmptyOrNonExistingMetadataDirectory() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( url.getPath() + File.separator + "nometadatafolder", SupportedSparkModels.LINEAR_REGRESSION ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Directory should contain metadata sub-directory")); } }
@Test public void failOnNonExistingModelParquetFile() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( url.getPath() + File.separator + "nomodelfilefolder", SupportedSparkModels.LINEAR_REGRESSION ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Directory should contain parquet file")); } }
@Test public void failOnTwoExistingModelParquetFile() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( url.getPath() + File.separator + "twomodelfilefolder", SupportedSparkModels.LINEAR_REGRESSION ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Directory should contain only one parquet file")); } }
@Test public void failOnNonExistingMetadataFile() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( url.getPath() + File.separator + "nometadatafilefolder", SupportedSparkModels.LINEAR_REGRESSION ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Directory should contain json file with model metadata")); } }
@Test public void failOnNonExistingTreeMetadataFolder() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( url.getPath() + File.separator + "notreesmetadatafolder", SupportedSparkModels.GRADIENT_BOOSTED_TREES ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Directory should contain treeMetadata sub-directory")); } }
@Test public void failOnNonExistingTreeMetadataFile() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( url.getPath() + File.separator + "notreesmetadatafile", SupportedSparkModels.GRADIENT_BOOSTED_TREES ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Directory should contain parquet file")); } }
@Test public void failOnTwoExistingTreeMetadataFile() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( url.getPath() + File.separator + "twotreesmetadatafiles", SupportedSparkModels.GRADIENT_BOOSTED_TREES ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("Directory should contain only one parquet file")); } }
@Test public void failOnIncorrectModelClassLoading() { URL url = getClass().getClassLoader().getResource(SPARK_MDL_PATH); try { SparkModelParser.parse( url.getPath() + File.separator + "gbt", SupportedSparkModels.DECISION_TREE ); fail("Expected IllegalArgumentException exception"); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().contains("The metadata file contains incorrect model metadata.")); } }