elephas/ml_model.py
def _fit(self, df):
        """Private fit method of the Estimator, which trains the model.
        """
        simple_rdd = df_to_simple_rdd(df, categorical=self.get_categorical_labels(), nb_classes=self.get_nb_classes(),
                                      features_col=self.getFeaturesCol(), label_col=self.getLabelCol())
        simple_rdd = simple_rdd.repartition(self.get_num_workers())
        elephas_optimizer = None
        if self.get_elephas_optimizer_config() is not None:
            elephas_optimizer = get({'class_name': self.get_optimizer_config()['class_name'],
                                     'config': self.get_optimizer_config()})

        keras_model = model_from_yaml(self.get_keras_model_config())
        metrics = self.get_metrics()
        loss = self.get_loss()
        optimizer = get_optimizer(self.get_optimizer_config())
        keras_model.compile(loss=loss, optimizer=optimizer, metrics=metrics)

        spark_model = SparkModel(model=keras_model, elephas_optimizer=elephas_optimizer,
                                 mode=self.get_mode(), frequency=self.get_frequency(),
                                 num_workers=self.get_num_workers())
        spark_model.fit(simple_rdd, epochs=self.get_epochs(), batch_size=self.get_batch_size(),
                        verbose=self.get_verbosity(), validation_split=self.get_validation_split())

        model_weights = spark_model.master_network.get_weights()
        weights = simple_rdd.ctx.broadcast(model_weights)
        return ElephasTransformer(labelCol=self.getLabelCol(),
                                  outputCol='prediction',  # TODO: Set default value
                                  keras_model_config=spark_model.master_network.to_yaml(),
                                  weights=weights)
Private fit method of the Estimator, which trains the model.
        
Private fit method of the Estimator, which trains the model.
Private fit method of the Estimator which trains the model .
def _fit(self, df):
        simple_rdd = df_to_simple_rdd(df, categorical=self.get_categorical_labels(), nb_classes=self.get_nb_classes(),
                                      features_col=self.getFeaturesCol(), label_col=self.getLabelCol())
        simple_rdd = simple_rdd.repartition(self.get_num_workers())
        elephas_optimizer = None
        if self.get_elephas_optimizer_config() is not None:
            elephas_optimizer = get({'class_name': self.get_optimizer_config()['class_name'],
                                     'config': self.get_optimizer_config()})
        keras_model = model_from_yaml(self.get_keras_model_config())
        metrics = self.get_metrics()
        loss = self.get_loss()
        optimizer = get_optimizer(self.get_optimizer_config())
        keras_model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
        spark_model = SparkModel(model=keras_model, elephas_optimizer=elephas_optimizer,
                                 mode=self.get_mode(), frequency=self.get_frequency(),
                                 num_workers=self.get_num_workers())
        spark_model.fit(simple_rdd, epochs=self.get_epochs(), batch_size=self.get_batch_size(),
                        verbose=self.get_verbosity(), validation_split=self.get_validation_split())
        model_weights = spark_model.master_network.get_weights()
        weights = simple_rdd.ctx.broadcast(model_weights)
        return ElephasTransformer(labelCol=self.getLabelCol(),
                                  outputCol='prediction',  
                                  keras_model_config=spark_model.master_network.to_yaml(),
                                  weights=weights)
def _fit ( self , df ) : simple_rdd = df_to_simple_rdd ( df , categorical = self . get_categorical_labels ( ) , nb_classes = self . get_nb_classes ( ) , features_col = self . getFeaturesCol ( ) , label_col = self . getLabelCol ( ) ) simple_rdd = simple_rdd . repartition ( self . get_num_workers ( ) ) elephas_optimizer = None if self . get_elephas_optimizer_config ( ) is not None : elephas_optimizer = get ( { 'class_name' : self . get_optimizer_config ( ) [ 'class_name' ] , 'config' : self . get_optimizer_config ( ) } ) keras_model = model_from_yaml ( self . get_keras_model_config ( ) ) metrics = self . get_metrics ( ) loss = self . get_loss ( ) optimizer = get_optimizer ( self . get_optimizer_config ( ) ) keras_model . compile ( loss = loss , optimizer = optimizer , metrics = metrics ) spark_model = SparkModel ( model = keras_model , elephas_optimizer = elephas_optimizer , mode = self . get_mode ( ) , frequency = self . get_frequency ( ) , num_workers = self . get_num_workers ( ) ) spark_model . fit ( simple_rdd , epochs = self . get_epochs ( ) , batch_size = self . get_batch_size ( ) , verbose = self . get_verbosity ( ) , validation_split = self . get_validation_split ( ) ) model_weights = spark_model . master_network . get_weights ( ) weights = simple_rdd . ctx . broadcast ( model_weights ) return ElephasTransformer ( labelCol = self . getLabelCol ( ) , outputCol = 'prediction' , keras_model_config = spark_model . master_network . to_yaml ( ) , weights = weights )
Private fit method of the Estimator, which trains the model.
Private fit method of the Estimator which trains the model .
"""Private fit method of the Estimator, which trains the model.
        """ # TODO: Set default value
self
df