@Test public void testIssue7068() throws Exception { val batchSize = 5; val featSize = 10; val labelSize = 2; val random = new Random(0); List<org.nd4j.linalg.dataset.api.MultiDataSet> l = new ArrayList<>(); for( int i=0; i<10; i++ ) { org.nd4j.linalg.dataset.MultiDataSet mds = new org.nd4j.linalg.dataset.MultiDataSet( new INDArray[]{Nd4j.rand(batchSize, featSize).castTo(DataType.DOUBLE), Nd4j.rand(batchSize, featSize).castTo(DataType.DOUBLE)}, new INDArray[]{Nd4j.rand(batchSize, labelSize).castTo(DataType.DOUBLE)}); l.add(mds); } JavaRDD<org.nd4j.linalg.dataset.api.MultiDataSet> rdd = sc.parallelize(l); val modelConf = new NeuralNetConfiguration.Builder() .updater(new Adam(0.01)) .weightInit(WeightInit.XAVIER_UNIFORM) .biasInit(0) .graphBuilder() .addInputs("input1", "input2") .addVertex("avg",new ElementWiseVertex(ElementWiseVertex.Op.Average),"input1","input2") .addLayer("dense",new DenseLayer.Builder().dropOut(0.9).nIn(featSize).nOut(featSize / 2).build(),"avg") .addLayer("output",new OutputLayer.Builder().nIn(featSize / 2).nOut(2).lossFunction(LossFunctions.LossFunction.MCXENT).activation(Activation.SOFTMAX).hasBias(false).build(),"dense") .setOutputs("output") .build(); val model = new ComputationGraph(modelConf); model.init(); val trainingMaster = new ParameterAveragingTrainingMaster.Builder(batchSize) .rddTrainingApproach(RDDTrainingApproach.Direct) .build(); val sparkModel = new SparkComputationGraph(sc, model, trainingMaster); for( int i=0; i<3; i++ ){ sparkModel.fitMultiDataSet(rdd); } }
public ComputationGraph fitMultiDataSet(RDD<MultiDataSet> rdd) { return fitMultiDataSet(rdd.toJavaRDD()); }
SparkComputationGraph extends SparkListenable { public ComputationGraph fitMultiDataSet(RDD<MultiDataSet> rdd) { return fitMultiDataSet(rdd.toJavaRDD()); } }
SparkComputationGraph extends SparkListenable { public ComputationGraph fitMultiDataSet(RDD<MultiDataSet> rdd) { return fitMultiDataSet(rdd.toJavaRDD()); } SparkComputationGraph(SparkContext sparkContext, ComputationGraph network, TrainingMaster trainingMaster);  SparkComputationGraph(JavaSparkContext javaSparkContext, ComputationGraph network,
                    TrainingMaster trainingMaster);  SparkComputationGraph(SparkContext sparkContext, ComputationGraphConfiguration conf,
                    TrainingMaster trainingMaster);  SparkComputationGraph(JavaSparkContext sparkContext, ComputationGraphConfiguration conf,
                    TrainingMaster trainingMaster); }
SparkComputationGraph extends SparkListenable { public ComputationGraph fitMultiDataSet(RDD<MultiDataSet> rdd) { return fitMultiDataSet(rdd.toJavaRDD()); } SparkComputationGraph(SparkContext sparkContext, ComputationGraph network, TrainingMaster trainingMaster);  SparkComputationGraph(JavaSparkContext javaSparkContext, ComputationGraph network,
                    TrainingMaster trainingMaster);  SparkComputationGraph(SparkContext sparkContext, ComputationGraphConfiguration conf,
                    TrainingMaster trainingMaster);  SparkComputationGraph(JavaSparkContext sparkContext, ComputationGraphConfiguration conf,
                    TrainingMaster trainingMaster); JavaSparkContext getSparkContext(); void setCollectTrainingStats(boolean collectTrainingStats); SparkTrainingStats getSparkTrainingStats(); ComputationGraph getNetwork(); TrainingMaster getTrainingMaster(); void setNetwork(ComputationGraph network); int getDefaultEvaluationWorkers(); void setDefaultEvaluationWorkers(int workers); ComputationGraph fit(RDD<DataSet> rdd); ComputationGraph fit(JavaRDD<DataSet> rdd); ComputationGraph fit(String path); @Deprecated ComputationGraph fit(String path, int minPartitions); ComputationGraph fitPaths(JavaRDD<String> paths); ComputationGraph fitPaths(JavaRDD<String> paths, DataSetLoader loader); ComputationGraph fitMultiDataSet(RDD<MultiDataSet> rdd); ComputationGraph fitMultiDataSet(JavaRDD<MultiDataSet> rdd); ComputationGraph fitMultiDataSet(String path); ComputationGraph fitPathsMultiDataSet(JavaRDD<String> paths); ComputationGraph fitPaths(JavaRDD<String> paths, MultiDataSetLoader loader); @Deprecated ComputationGraph fitMultiDataSet(String path, int minPartitions); double getScore(); void setScore(double lastScore); double calculateScore(JavaRDD<DataSet> data, boolean average); double calculateScore(JavaRDD<DataSet> data, boolean average, int minibatchSize); double calculateScoreMultiDataSet(JavaRDD<MultiDataSet> data, boolean average); double calculateScoreMultiDataSet(JavaRDD<MultiDataSet> data, boolean average, int minibatchSize); JavaDoubleRDD scoreExamples(JavaRDD<DataSet> data, boolean includeRegularizationTerms); JavaDoubleRDD scoreExamples(JavaRDD<DataSet> data, boolean includeRegularizationTerms, int batchSize); JavaPairRDD<K, Double> scoreExamples(JavaPairRDD<K, DataSet> data, boolean includeRegularizationTerms); JavaPairRDD<K, Double> scoreExamples(JavaPairRDD<K, DataSet> data, boolean includeRegularizationTerms,
                    int batchSize); JavaDoubleRDD scoreExamplesMultiDataSet(JavaRDD<MultiDataSet> data, boolean includeRegularizationTerms); JavaDoubleRDD scoreExamplesMultiDataSet(JavaRDD<MultiDataSet> data, boolean includeRegularizationTerms,
                    int batchSize); JavaPairRDD<K, Double> scoreExamplesMultiDataSet(JavaPairRDD<K, MultiDataSet> data,
                    boolean includeRegularizationTerms); JavaPairRDD<K, INDArray> feedForwardWithKeySingle(JavaPairRDD<K, INDArray> featuresData, int batchSize); JavaPairRDD<K, INDArray[]> feedForwardWithKey(JavaPairRDD<K, INDArray[]> featuresData, int batchSize); JavaPairRDD<K, Double> scoreExamplesMultiDataSet(JavaPairRDD<K, MultiDataSet> data,
                    boolean includeRegularizationTerms, int batchSize); Evaluation evaluate(String path, DataSetLoader loader); Evaluation evaluate(String path, MultiDataSetLoader loader); T evaluate(RDD<DataSet> data); T evaluate(JavaRDD<DataSet> data); T evaluate(RDD<DataSet> data, List<String> labelsList); T evaluateRegression(JavaRDD<DataSet> data); T evaluateRegression(JavaRDD<DataSet> data, int minibatchSize); T evaluate(JavaRDD<DataSet> data, List<String> labelsList); T evaluateROC(JavaRDD<DataSet> data); T evaluateROC(JavaRDD<DataSet> data, int thresholdSteps, int evaluationMinibatchSize); T evaluateROCMultiClass(JavaRDD<DataSet> data); T evaluateROCMultiClass(JavaRDD<DataSet> data, int thresholdSteps, int evaluationMinibatchSize); T evaluate(JavaRDD<DataSet> data, List<String> labelsList, int evalBatchSize); T evaluateMDS(JavaRDD<MultiDataSet> data); T evaluateMDS(JavaRDD<MultiDataSet> data, int minibatchSize); T evaluateRegressionMDS(JavaRDD<MultiDataSet> data); T evaluateRegressionMDS(JavaRDD<MultiDataSet> data, int minibatchSize); ROC evaluateROCMDS(JavaRDD<MultiDataSet> data); T evaluateROCMDS(JavaRDD<MultiDataSet> data, int rocThresholdNumSteps, int minibatchSize); @SuppressWarnings("unchecked") T doEvaluation(JavaRDD<DataSet> data, T emptyEvaluation, int evalBatchSize); T[] doEvaluation(JavaRDD<DataSet> data, int evalBatchSize, T... emptyEvaluations); T[] doEvaluation(JavaRDD<DataSet> data, int evalNumWorkers, int evalBatchSize, T... emptyEvaluations); @SuppressWarnings("unchecked") T[] doEvaluationMDS(JavaRDD<MultiDataSet> data, int evalBatchSize, T... emptyEvaluations); T[] doEvaluationMDS(JavaRDD<MultiDataSet> data, int evalNumWorkers, int evalBatchSize, T... emptyEvaluations); IEvaluation[] doEvaluation(JavaRDD<String> data, DataSetLoader loader, IEvaluation... emptyEvaluations); IEvaluation[] doEvaluation(JavaRDD<String> data, int evalNumWorkers, int evalBatchSize, DataSetLoader loader, IEvaluation... emptyEvaluations); IEvaluation[] doEvaluation(JavaRDD<String> data, MultiDataSetLoader loader, IEvaluation... emptyEvaluations); IEvaluation[] doEvaluation(JavaRDD<String> data, int evalNumWorkers, int evalBatchSize, MultiDataSetLoader loader, IEvaluation... emptyEvaluations); }
SparkComputationGraph extends SparkListenable { public ComputationGraph fitMultiDataSet(RDD<MultiDataSet> rdd) { return fitMultiDataSet(rdd.toJavaRDD()); } SparkComputationGraph(SparkContext sparkContext, ComputationGraph network, TrainingMaster trainingMaster);  SparkComputationGraph(JavaSparkContext javaSparkContext, ComputationGraph network,
                    TrainingMaster trainingMaster);  SparkComputationGraph(SparkContext sparkContext, ComputationGraphConfiguration conf,
                    TrainingMaster trainingMaster);  SparkComputationGraph(JavaSparkContext sparkContext, ComputationGraphConfiguration conf,
                    TrainingMaster trainingMaster); JavaSparkContext getSparkContext(); void setCollectTrainingStats(boolean collectTrainingStats); SparkTrainingStats getSparkTrainingStats(); ComputationGraph getNetwork(); TrainingMaster getTrainingMaster(); void setNetwork(ComputationGraph network); int getDefaultEvaluationWorkers(); void setDefaultEvaluationWorkers(int workers); ComputationGraph fit(RDD<DataSet> rdd); ComputationGraph fit(JavaRDD<DataSet> rdd); ComputationGraph fit(String path); @Deprecated ComputationGraph fit(String path, int minPartitions); ComputationGraph fitPaths(JavaRDD<String> paths); ComputationGraph fitPaths(JavaRDD<String> paths, DataSetLoader loader); ComputationGraph fitMultiDataSet(RDD<MultiDataSet> rdd); ComputationGraph fitMultiDataSet(JavaRDD<MultiDataSet> rdd); ComputationGraph fitMultiDataSet(String path); ComputationGraph fitPathsMultiDataSet(JavaRDD<String> paths); ComputationGraph fitPaths(JavaRDD<String> paths, MultiDataSetLoader loader); @Deprecated ComputationGraph fitMultiDataSet(String path, int minPartitions); double getScore(); void setScore(double lastScore); double calculateScore(JavaRDD<DataSet> data, boolean average); double calculateScore(JavaRDD<DataSet> data, boolean average, int minibatchSize); double calculateScoreMultiDataSet(JavaRDD<MultiDataSet> data, boolean average); double calculateScoreMultiDataSet(JavaRDD<MultiDataSet> data, boolean average, int minibatchSize); JavaDoubleRDD scoreExamples(JavaRDD<DataSet> data, boolean includeRegularizationTerms); JavaDoubleRDD scoreExamples(JavaRDD<DataSet> data, boolean includeRegularizationTerms, int batchSize); JavaPairRDD<K, Double> scoreExamples(JavaPairRDD<K, DataSet> data, boolean includeRegularizationTerms); JavaPairRDD<K, Double> scoreExamples(JavaPairRDD<K, DataSet> data, boolean includeRegularizationTerms,
                    int batchSize); JavaDoubleRDD scoreExamplesMultiDataSet(JavaRDD<MultiDataSet> data, boolean includeRegularizationTerms); JavaDoubleRDD scoreExamplesMultiDataSet(JavaRDD<MultiDataSet> data, boolean includeRegularizationTerms,
                    int batchSize); JavaPairRDD<K, Double> scoreExamplesMultiDataSet(JavaPairRDD<K, MultiDataSet> data,
                    boolean includeRegularizationTerms); JavaPairRDD<K, INDArray> feedForwardWithKeySingle(JavaPairRDD<K, INDArray> featuresData, int batchSize); JavaPairRDD<K, INDArray[]> feedForwardWithKey(JavaPairRDD<K, INDArray[]> featuresData, int batchSize); JavaPairRDD<K, Double> scoreExamplesMultiDataSet(JavaPairRDD<K, MultiDataSet> data,
                    boolean includeRegularizationTerms, int batchSize); Evaluation evaluate(String path, DataSetLoader loader); Evaluation evaluate(String path, MultiDataSetLoader loader); T evaluate(RDD<DataSet> data); T evaluate(JavaRDD<DataSet> data); T evaluate(RDD<DataSet> data, List<String> labelsList); T evaluateRegression(JavaRDD<DataSet> data); T evaluateRegression(JavaRDD<DataSet> data, int minibatchSize); T evaluate(JavaRDD<DataSet> data, List<String> labelsList); T evaluateROC(JavaRDD<DataSet> data); T evaluateROC(JavaRDD<DataSet> data, int thresholdSteps, int evaluationMinibatchSize); T evaluateROCMultiClass(JavaRDD<DataSet> data); T evaluateROCMultiClass(JavaRDD<DataSet> data, int thresholdSteps, int evaluationMinibatchSize); T evaluate(JavaRDD<DataSet> data, List<String> labelsList, int evalBatchSize); T evaluateMDS(JavaRDD<MultiDataSet> data); T evaluateMDS(JavaRDD<MultiDataSet> data, int minibatchSize); T evaluateRegressionMDS(JavaRDD<MultiDataSet> data); T evaluateRegressionMDS(JavaRDD<MultiDataSet> data, int minibatchSize); ROC evaluateROCMDS(JavaRDD<MultiDataSet> data); T evaluateROCMDS(JavaRDD<MultiDataSet> data, int rocThresholdNumSteps, int minibatchSize); @SuppressWarnings("unchecked") T doEvaluation(JavaRDD<DataSet> data, T emptyEvaluation, int evalBatchSize); T[] doEvaluation(JavaRDD<DataSet> data, int evalBatchSize, T... emptyEvaluations); T[] doEvaluation(JavaRDD<DataSet> data, int evalNumWorkers, int evalBatchSize, T... emptyEvaluations); @SuppressWarnings("unchecked") T[] doEvaluationMDS(JavaRDD<MultiDataSet> data, int evalBatchSize, T... emptyEvaluations); T[] doEvaluationMDS(JavaRDD<MultiDataSet> data, int evalNumWorkers, int evalBatchSize, T... emptyEvaluations); IEvaluation[] doEvaluation(JavaRDD<String> data, DataSetLoader loader, IEvaluation... emptyEvaluations); IEvaluation[] doEvaluation(JavaRDD<String> data, int evalNumWorkers, int evalBatchSize, DataSetLoader loader, IEvaluation... emptyEvaluations); IEvaluation[] doEvaluation(JavaRDD<String> data, MultiDataSetLoader loader, IEvaluation... emptyEvaluations); IEvaluation[] doEvaluation(JavaRDD<String> data, int evalNumWorkers, int evalBatchSize, MultiDataSetLoader loader, IEvaluation... emptyEvaluations); static final int DEFAULT_ROC_THRESHOLD_STEPS; static final int DEFAULT_EVAL_SCORE_BATCH_SIZE; static final int DEFAULT_EVAL_WORKERS; }