airflow/hooks/hive_hooks.py
def load_df(
            self,
            df,
            table,
            field_dict=None,
            delimiter=',',
            encoding='utf8',
            pandas_kwargs=None, **kwargs):
        """
        Loads a pandas DataFrame into hive.

        Hive data types will be inferred if not passed but column names will
        not be sanitized.

        :param df: DataFrame to load into a Hive table
        :type df: pandas.DataFrame
        :param table: target Hive table, use dot notation to target a
            specific database
        :type table: str
        :param field_dict: mapping from column name to hive data type.
            Note that it must be OrderedDict so as to keep columns' order.
        :type field_dict: collections.OrderedDict
        :param delimiter: field delimiter in the file
        :type delimiter: str
        :param encoding: str encoding to use when writing DataFrame to file
        :type encoding: str
        :param pandas_kwargs: passed to DataFrame.to_csv
        :type pandas_kwargs: dict
        :param kwargs: passed to self.load_file
        """

        def _infer_field_types_from_df(df):
            DTYPE_KIND_HIVE_TYPE = {
                'b': 'BOOLEAN',    # boolean
                'i': 'BIGINT',     # signed integer
                'u': 'BIGINT',     # unsigned integer
                'f': 'DOUBLE',     # floating-point
                'c': 'STRING',     # complex floating-point
                'M': 'TIMESTAMP',  # datetime
                'O': 'STRING',     # object
                'S': 'STRING',     # (byte-)string
                'U': 'STRING',     # Unicode
                'V': 'STRING'      # void
            }

            d = OrderedDict()
            for col, dtype in df.dtypes.iteritems():
                d[col] = DTYPE_KIND_HIVE_TYPE[dtype.kind]
            return d

        if pandas_kwargs is None:
            pandas_kwargs = {}

        with TemporaryDirectory(prefix='airflow_hiveop_') as tmp_dir:
            with NamedTemporaryFile(dir=tmp_dir, mode="w") as f:

                if field_dict is None:
                    field_dict = _infer_field_types_from_df(df)

                df.to_csv(path_or_buf=f,
                          sep=delimiter,
                          header=False,
                          index=False,
                          encoding=encoding,
                          date_format="%Y-%m-%d %H:%M:%S",
                          **pandas_kwargs)
                f.flush()

                return self.load_file(filepath=f.name,
                                      table=table,
                                      delimiter=delimiter,
                                      field_dict=field_dict,
                                      **kwargs)
def load_df(
            self,
            df,
            table,
            field_dict=None,
            delimiter=',',
            encoding='utf8',
            pandas_kwargs=None, **kwargs):
        """
        Loads a pandas DataFrame into hive.

        Hive data types will be inferred if not passed but column names will
        not be sanitized.

        :param df: DataFrame to load into a Hive table
        :type df: pandas.DataFrame
        :param table: target Hive table, use dot notation to target a
            specific database
        :type table: str
        :param field_dict: mapping from column name to hive data type.
            Note that it must be OrderedDict so as to keep columns' order.
        :type field_dict: collections.OrderedDict
        :param delimiter: field delimiter in the file
        :type delimiter: str
        :param encoding: str encoding to use when writing DataFrame to file
        :type encoding: str
        :param pandas_kwargs: passed to DataFrame.to_csv
        :type pandas_kwargs: dict
        :param kwargs: passed to self.load_file
        """

        def _infer_field_types_from_df(df):
            DTYPE_KIND_HIVE_TYPE = {
                'b': 'BOOLEAN',    # boolean
                'i': 'BIGINT',     # signed integer
                'u': 'BIGINT',     # unsigned integer
                'f': 'DOUBLE',     # floating-point
                'c': 'STRING',     # complex floating-point
                'M': 'TIMESTAMP',  # datetime
                'O': 'STRING',     # object
                'S': 'STRING',     # (byte-)string
                'U': 'STRING',     # Unicode
                'V': 'STRING'      # void
            }

            d = OrderedDict()
            for col, dtype in df.dtypes.iteritems():
                d[col] = DTYPE_KIND_HIVE_TYPE[dtype.kind]
            return d

        if pandas_kwargs is None:
            pandas_kwargs = {}

        with TemporaryDirectory(prefix='airflow_hiveop_') as tmp_dir:
            with NamedTemporaryFile(dir=tmp_dir, mode="w") as f:

                if field_dict is None:
                    field_dict = _infer_field_types_from_df(df)

                df.to_csv(path_or_buf=f,
                          sep=delimiter,
                          header=False,
                          index=False,
                          encoding=encoding,
                          date_format="%Y-%m-%d %H:%M:%S",
                          **pandas_kwargs)
                f.flush()

                return self.load_file(filepath=f.name,
                                      table=table,
                                      delimiter=delimiter,
                                      field_dict=field_dict,
                                      **kwargs)
def load_df ( self , df , table , field_dict = None , delimiter = ',' , encoding = 'utf8' , pandas_kwargs = None , * * kwargs ) : def _infer_field_types_from_df ( df ) : DTYPE_KIND_HIVE_TYPE = { 'b' : 'BOOLEAN' , # boolean 'i' : 'BIGINT' , # signed integer 'u' : 'BIGINT' , # unsigned integer 'f' : 'DOUBLE' , # floating-point 'c' : 'STRING' , # complex floating-point 'M' : 'TIMESTAMP' , # datetime 'O' : 'STRING' , # object 'S' : 'STRING' , # (byte-)string 'U' : 'STRING' , # Unicode 'V' : 'STRING' # void } d = OrderedDict ( ) for col , dtype in df . dtypes . iteritems ( ) : d [ col ] = DTYPE_KIND_HIVE_TYPE [ dtype . kind ] return d if pandas_kwargs is None : pandas_kwargs = { } with TemporaryDirectory ( prefix = 'airflow_hiveop_' ) as tmp_dir : with NamedTemporaryFile ( dir = tmp_dir , mode = "w" ) as f : if field_dict is None : field_dict = _infer_field_types_from_df ( df ) df . to_csv ( path_or_buf = f , sep = delimiter , header = False , index = False , encoding = encoding , date_format = "%Y-%m-%d %H:%M:%S" , * * pandas_kwargs ) f . flush ( ) return self . load_file ( filepath = f . name , table = table , delimiter = delimiter , field_dict = field_dict , * * kwargs )
Loads a pandas DataFrame into hive.

        Hive data types will be inferred if not passed but column names will
        not be sanitized.

        :param df: DataFrame to load into a Hive table
        :type df: pandas.DataFrame
        :param table: target Hive table, use dot notation to target a
            specific database
        :type table: str
        :param field_dict: mapping from column name to hive data type.
            Note that it must be OrderedDict so as to keep columns' order.
        :type field_dict: collections.OrderedDict
        :param delimiter: field delimiter in the file
        :type delimiter: str
        :param encoding: str encoding to use when writing DataFrame to file
        :type encoding: str
        :param pandas_kwargs: passed to DataFrame.to_csv
        :type pandas_kwargs: dict
        :param kwargs: passed to self.load_file
Loads a pandas DataFrame into hive .