h2o-py/h2o/frame.py
def convert_H2OFrame_2_DMatrix(self, predictors, yresp, h2oXGBoostModel):
        '''
        This method requires that you import the following toolboxes: xgboost, pandas, numpy and scipy.sparse.

        This method will convert an H2OFrame to a DMatrix that can be used by native XGBoost.  The H2OFrame contains
        numerical and enum columns alone.  Note that H2O one-hot-encoding introduces a missing(NA)
        column. There can be NAs in any columns.

        Follow the steps below to compare H2OXGBoost and native XGBoost:

        1. Train the H2OXGBoost model with H2OFrame trainFile and generate a prediction:
        h2oModelD = H2OXGBoostEstimator(**h2oParamsD) # parameters specified as a dict()
        h2oModelD.train(x=myX, y=y, training_frame=trainFile) # train with H2OFrame trainFile
        h2oPredict = h2oPredictD = h2oModelD.predict(trainFile)

        2. Derive the DMatrix from H2OFrame:
        nativeDMatrix = trainFile.convert_H2OFrame_2_DMatrix(myX, y, h2oModelD)

        3. Derive the parameters for native XGBoost:
        nativeParams = h2oModelD.convert_H2OXGBoostParams_2_XGBoostParams()

        4. Train your native XGBoost model and generate a prediction:
        nativeModel = xgb.train(params=nativeParams[0], dtrain=nativeDMatrix, num_boost_round=nativeParams[1])
        nativePredict = nativeModel.predict(data=nativeDMatrix, ntree_limit=nativeParams[1].

        5. Compare the predictions h2oPredict from H2OXGBoost, nativePredict from native XGBoost.

        :param h2oFrame: H2OFrame to be converted to DMatrix for native XGBoost
        :param predictors: List of predictor columns, can be column names or indices
        :param yresp: response column, can be column index or name
        :param h2oXGBoostModel: H2OXGboost model that are built with the same H2OFrame as input earlier
        :return: DMatrix that can be an input to a native XGBoost model
        '''
        import xgboost as xgb
        import pandas as pd
        import numpy as np
        from scipy.sparse import csr_matrix

        assert isinstance(predictors, list) or isinstance(predictors, tuple)
        assert h2oXGBoostModel._model_json['algo'] == 'xgboost', \
            "convert_H2OFrame_2_DMatrix is used for H2OXGBoost model only."

        tempFrame = self[predictors].cbind(self[yresp])
        colnames = tempFrame.names
        if type(predictors[0])==type(1): # convert integer indices to column names
            temp = []
            for colInd in predictors:
                temp.append(colnames[colInd])
            predictors = temp

        if (type(yresp) == type(1)):
            tempy = colnames[yresp]
            yresp = tempy # column name of response column

        enumCols = [] # extract enum columns out to process them
        enumColsIndices = []     # store enum column indices
        typeDict = self.types
        for predName in predictors:
            if str(typeDict[predName])=='enum':
                enumCols.append(predName)
                enumColsIndices.append(colnames.index(predName))

        pandaFtrain = tempFrame.as_data_frame(use_pandas=True, header=True)
        nrows = tempFrame.nrow

        # convert H2OFrame to DMatrix starts here
        if len(enumCols) > 0:   # enumCols contain all enum column names
            allDomain = tempFrame.levels() # list all domain levels with column indices
            domainLen = []
            for enumIndex in enumColsIndices:
                if len(allDomain[enumIndex])>0:
                    domainLen.append(len(allDomain[enumIndex])*-1)
            incLevel = np.argsort(domainLen) # indices of enum column indices with decreasing domain length

            # need to move enum columns to the front, highest level first
            c2 = tempFrame[enumCols[incLevel[0]]]
            tempFrame = tempFrame.drop(enumCols[incLevel[0]])
            for index in range(1, len(incLevel)):
                c2 = c2.cbind(tempFrame[enumCols[incLevel[index]]])
                tempFrame = tempFrame.drop(enumCols[incLevel[index]])
               
            enumCols = c2.names
            tempFrame = c2.cbind(tempFrame)
            pandaFtrain = tempFrame.as_data_frame(use_pandas=True, header=True) # redo translation from H2O to panda
        
            pandaTrainPart = generatePandaEnumCols(pandaFtrain, enumCols[0], nrows, tempFrame[enumCols[0]].categories())
            pandaFtrain.drop([enumCols[0]], axis=1, inplace=True)

            for colInd in range(1, len(enumCols)):
                cname=enumCols[colInd]
                ctemp = generatePandaEnumCols(pandaFtrain, cname,  nrows, tempFrame[enumCols[colInd]].categories())
                pandaTrainPart=pd.concat([pandaTrainPart, ctemp], axis=1)
                pandaFtrain.drop([cname], axis=1, inplace=True)

            pandaFtrain = pd.concat([pandaTrainPart, pandaFtrain], axis=1)

        c0= tempFrame[yresp].asnumeric().as_data_frame(use_pandas=True, header=True)
        pandaFtrain.drop([yresp], axis=1, inplace=True)
        pandaF = pd.concat([c0, pandaFtrain], axis=1)
        pandaF.rename(columns={c0.columns[0]:yresp}, inplace=True)
        newX = list(pandaFtrain.columns.values)
        data = pandaF.as_matrix(newX)
        label = pandaF.as_matrix([yresp])

        return xgb.DMatrix(data=csr_matrix(data), label=label) \
            if h2oXGBoostModel._model_json['output']['sparse'] else xgb.DMatrix(data=data, label=label)
def convert_H2OFrame_2_DMatrix(self, predictors, yresp, h2oXGBoostModel):
        '''
        This method requires that you import the following toolboxes: xgboost, pandas, numpy and scipy.sparse.

        This method will convert an H2OFrame to a DMatrix that can be used by native XGBoost.  The H2OFrame contains
        numerical and enum columns alone.  Note that H2O one-hot-encoding introduces a missing(NA)
        column. There can be NAs in any columns.

        Follow the steps below to compare H2OXGBoost and native XGBoost:

        1. Train the H2OXGBoost model with H2OFrame trainFile and generate a prediction:
        h2oModelD = H2OXGBoostEstimator(**h2oParamsD) # parameters specified as a dict()
        h2oModelD.train(x=myX, y=y, training_frame=trainFile) # train with H2OFrame trainFile
        h2oPredict = h2oPredictD = h2oModelD.predict(trainFile)

        2. Derive the DMatrix from H2OFrame:
        nativeDMatrix = trainFile.convert_H2OFrame_2_DMatrix(myX, y, h2oModelD)

        3. Derive the parameters for native XGBoost:
        nativeParams = h2oModelD.convert_H2OXGBoostParams_2_XGBoostParams()

        4. Train your native XGBoost model and generate a prediction:
        nativeModel = xgb.train(params=nativeParams[0], dtrain=nativeDMatrix, num_boost_round=nativeParams[1])
        nativePredict = nativeModel.predict(data=nativeDMatrix, ntree_limit=nativeParams[1].

        5. Compare the predictions h2oPredict from H2OXGBoost, nativePredict from native XGBoost.

        :param h2oFrame: H2OFrame to be converted to DMatrix for native XGBoost
        :param predictors: List of predictor columns, can be column names or indices
        :param yresp: response column, can be column index or name
        :param h2oXGBoostModel: H2OXGboost model that are built with the same H2OFrame as input earlier
        :return: DMatrix that can be an input to a native XGBoost model
        '''
        import xgboost as xgb
        import pandas as pd
        import numpy as np
        from scipy.sparse import csr_matrix

        assert isinstance(predictors, list) or isinstance(predictors, tuple)
        assert h2oXGBoostModel._model_json['algo'] == 'xgboost', \
            "convert_H2OFrame_2_DMatrix is used for H2OXGBoost model only."

        tempFrame = self[predictors].cbind(self[yresp])
        colnames = tempFrame.names
        if type(predictors[0])==type(1): # convert integer indices to column names
            temp = []
            for colInd in predictors:
                temp.append(colnames[colInd])
            predictors = temp

        if (type(yresp) == type(1)):
            tempy = colnames[yresp]
            yresp = tempy # column name of response column

        enumCols = [] # extract enum columns out to process them
        enumColsIndices = []     # store enum column indices
        typeDict = self.types
        for predName in predictors:
            if str(typeDict[predName])=='enum':
                enumCols.append(predName)
                enumColsIndices.append(colnames.index(predName))

        pandaFtrain = tempFrame.as_data_frame(use_pandas=True, header=True)
        nrows = tempFrame.nrow

        # convert H2OFrame to DMatrix starts here
        if len(enumCols) > 0:   # enumCols contain all enum column names
            allDomain = tempFrame.levels() # list all domain levels with column indices
            domainLen = []
            for enumIndex in enumColsIndices:
                if len(allDomain[enumIndex])>0:
                    domainLen.append(len(allDomain[enumIndex])*-1)
            incLevel = np.argsort(domainLen) # indices of enum column indices with decreasing domain length

            # need to move enum columns to the front, highest level first
            c2 = tempFrame[enumCols[incLevel[0]]]
            tempFrame = tempFrame.drop(enumCols[incLevel[0]])
            for index in range(1, len(incLevel)):
                c2 = c2.cbind(tempFrame[enumCols[incLevel[index]]])
                tempFrame = tempFrame.drop(enumCols[incLevel[index]])
               
            enumCols = c2.names
            tempFrame = c2.cbind(tempFrame)
            pandaFtrain = tempFrame.as_data_frame(use_pandas=True, header=True) # redo translation from H2O to panda
        
            pandaTrainPart = generatePandaEnumCols(pandaFtrain, enumCols[0], nrows, tempFrame[enumCols[0]].categories())
            pandaFtrain.drop([enumCols[0]], axis=1, inplace=True)

            for colInd in range(1, len(enumCols)):
                cname=enumCols[colInd]
                ctemp = generatePandaEnumCols(pandaFtrain, cname,  nrows, tempFrame[enumCols[colInd]].categories())
                pandaTrainPart=pd.concat([pandaTrainPart, ctemp], axis=1)
                pandaFtrain.drop([cname], axis=1, inplace=True)

            pandaFtrain = pd.concat([pandaTrainPart, pandaFtrain], axis=1)

        c0= tempFrame[yresp].asnumeric().as_data_frame(use_pandas=True, header=True)
        pandaFtrain.drop([yresp], axis=1, inplace=True)
        pandaF = pd.concat([c0, pandaFtrain], axis=1)
        pandaF.rename(columns={c0.columns[0]:yresp}, inplace=True)
        newX = list(pandaFtrain.columns.values)
        data = pandaF.as_matrix(newX)
        label = pandaF.as_matrix([yresp])

        return xgb.DMatrix(data=csr_matrix(data), label=label) \
            if h2oXGBoostModel._model_json['output']['sparse'] else xgb.DMatrix(data=data, label=label)
def convert_H2OFrame_2_DMatrix ( self , predictors , yresp , h2oXGBoostModel ) : import xgboost as xgb import pandas as pd import numpy as np from scipy . sparse import csr_matrix assert isinstance ( predictors , list ) or isinstance ( predictors , tuple ) assert h2oXGBoostModel . _model_json [ 'algo' ] == 'xgboost' , "convert_H2OFrame_2_DMatrix is used for H2OXGBoost model only." tempFrame = self [ predictors ] . cbind ( self [ yresp ] ) colnames = tempFrame . names if type ( predictors [ 0 ] ) == type ( 1 ) : # convert integer indices to column names temp = [ ] for colInd in predictors : temp . append ( colnames [ colInd ] ) predictors = temp if ( type ( yresp ) == type ( 1 ) ) : tempy = colnames [ yresp ] yresp = tempy # column name of response column enumCols = [ ] # extract enum columns out to process them enumColsIndices = [ ] # store enum column indices typeDict = self . types for predName in predictors : if str ( typeDict [ predName ] ) == 'enum' : enumCols . append ( predName ) enumColsIndices . append ( colnames . index ( predName ) ) pandaFtrain = tempFrame . as_data_frame ( use_pandas = True , header = True ) nrows = tempFrame . nrow # convert H2OFrame to DMatrix starts here if len ( enumCols ) > 0 : # enumCols contain all enum column names allDomain = tempFrame . levels ( ) # list all domain levels with column indices domainLen = [ ] for enumIndex in enumColsIndices : if len ( allDomain [ enumIndex ] ) > 0 : domainLen . append ( len ( allDomain [ enumIndex ] ) * - 1 ) incLevel = np . argsort ( domainLen ) # indices of enum column indices with decreasing domain length # need to move enum columns to the front, highest level first c2 = tempFrame [ enumCols [ incLevel [ 0 ] ] ] tempFrame = tempFrame . drop ( enumCols [ incLevel [ 0 ] ] ) for index in range ( 1 , len ( incLevel ) ) : c2 = c2 . cbind ( tempFrame [ enumCols [ incLevel [ index ] ] ] ) tempFrame = tempFrame . drop ( enumCols [ incLevel [ index ] ] ) enumCols = c2 . names tempFrame = c2 . cbind ( tempFrame ) pandaFtrain = tempFrame . as_data_frame ( use_pandas = True , header = True ) # redo translation from H2O to panda pandaTrainPart = generatePandaEnumCols ( pandaFtrain , enumCols [ 0 ] , nrows , tempFrame [ enumCols [ 0 ] ] . categories ( ) ) pandaFtrain . drop ( [ enumCols [ 0 ] ] , axis = 1 , inplace = True ) for colInd in range ( 1 , len ( enumCols ) ) : cname = enumCols [ colInd ] ctemp = generatePandaEnumCols ( pandaFtrain , cname , nrows , tempFrame [ enumCols [ colInd ] ] . categories ( ) ) pandaTrainPart = pd . concat ( [ pandaTrainPart , ctemp ] , axis = 1 ) pandaFtrain . drop ( [ cname ] , axis = 1 , inplace = True ) pandaFtrain = pd . concat ( [ pandaTrainPart , pandaFtrain ] , axis = 1 ) c0 = tempFrame [ yresp ] . asnumeric ( ) . as_data_frame ( use_pandas = True , header = True ) pandaFtrain . drop ( [ yresp ] , axis = 1 , inplace = True ) pandaF = pd . concat ( [ c0 , pandaFtrain ] , axis = 1 ) pandaF . rename ( columns = { c0 . columns [ 0 ] : yresp } , inplace = True ) newX = list ( pandaFtrain . columns . values ) data = pandaF . as_matrix ( newX ) label = pandaF . as_matrix ( [ yresp ] ) return xgb . DMatrix ( data = csr_matrix ( data ) , label = label ) if h2oXGBoostModel . _model_json [ 'output' ] [ 'sparse' ] else xgb . DMatrix ( data = data , label = label )
This method requires that you import the following toolboxes: xgboost, pandas, numpy and scipy.sparse.

        This method will convert an H2OFrame to a DMatrix that can be used by native XGBoost.  The H2OFrame contains
        numerical and enum columns alone.  Note that H2O one-hot-encoding introduces a missing(NA)
        column. There can be NAs in any columns.

        Follow the steps below to compare H2OXGBoost and native XGBoost:

        1. Train the H2OXGBoost model with H2OFrame trainFile and generate a prediction:
        h2oModelD = H2OXGBoostEstimator(**h2oParamsD) # parameters specified as a dict()
        h2oModelD.train(x=myX, y=y, training_frame=trainFile) # train with H2OFrame trainFile
        h2oPredict = h2oPredictD = h2oModelD.predict(trainFile)

        2. Derive the DMatrix from H2OFrame:
        nativeDMatrix = trainFile.convert_H2OFrame_2_DMatrix(myX, y, h2oModelD)

        3. Derive the parameters for native XGBoost:
        nativeParams = h2oModelD.convert_H2OXGBoostParams_2_XGBoostParams()

        4. Train your native XGBoost model and generate a prediction:
        nativeModel = xgb.train(params=nativeParams[0], dtrain=nativeDMatrix, num_boost_round=nativeParams[1])
        nativePredict = nativeModel.predict(data=nativeDMatrix, ntree_limit=nativeParams[1].

        5. Compare the predictions h2oPredict from H2OXGBoost, nativePredict from native XGBoost.

        :param h2oFrame: H2OFrame to be converted to DMatrix for native XGBoost
        :param predictors: List of predictor columns, can be column names or indices
        :param yresp: response column, can be column index or name
        :param h2oXGBoostModel: H2OXGboost model that are built with the same H2OFrame as input earlier
        :return: DMatrix that can be an input to a native XGBoost model
This method requires that you import the following toolboxes : xgboost pandas numpy and scipy . sparse .