Jann/utils.py
def embed_lines(args, unencoded_lines, output_dict,
                unencoded_lines_resps=None):
    """Embed a collection of lines to an output dictionary."""

    # Import the Universal Sentence Encoder's TF Hub module
    module = hub.Module(args.module_path, trainable=False)
    config = tf.ConfigProto(allow_soft_placement=True)

    with tf.Session(config=config) as session:
        # initialize the variables
        session.run(
            [tf.global_variables_initializer(), tf.tables_initializer()])

        sp, embeddings, input_placeholder = None, None, None
        if args.use_sentence_piece:
            # spm_path now contains a path to the SentencePiece
            # model stored inside the TF-Hub module
            spm_path = session.run(module(signature="spm_path"))
            sp = spm.SentencePieceProcessor()
            sp.Load(spm_path)

            # build an input placeholder
            with tf.device('/gpu:0'):
                input_placeholder = tf.sparse_placeholder(
                    tf.int64, shape=[None, None])
                embeddings = module(inputs=dict(
                    values=input_placeholder.values,
                    indices=input_placeholder.indices,
                    dense_shape=input_placeholder.dense_shape
                )
                )

        # size of chunk is how many lines will be encoded
        # with each pass of the model
        size_of_chunk = 256

        # ensure that every line has a response
        assert len(unencoded_lines) == len(unencoded_lines_resps)
        all_id_chunks = get_id_chunks(
            range(len(unencoded_lines)), size_of_chunk)

        max_iter = len(unencoded_lines) // size_of_chunk
        for id_chunk in tqdm(all_id_chunks, total=max_iter):
            # get the chunk of lines and matching responses by list of ids
            chunk_unencoded_lines = [unencoded_lines[x] for x in id_chunk]
            chunck_unenc_resp = [unencoded_lines_resps[x] for x in id_chunk]

            if args.use_sentence_piece:
                # process unencoded lines to values and IDs in sparse format
                values, indices, dense_shape = process_to_IDs_in_sparse_format(
                    sp=sp, sentences=chunk_unencoded_lines)

                # run the session
                with tf.device('/gpu:0'):
                    chunk_line_embds = session.run(
                        embeddings,
                        feed_dict={
                            input_placeholder.values: values,
                            input_placeholder.indices: indices,
                            input_placeholder.dense_shape: dense_shape
                        }
                    )
            else:
                with tf.device('/gpu:0'):
                    chunk_line_embds = session.run(
                        module(chunk_unencoded_lines))

            # hash the object into the full output dataframe
            for i, line_embedding in enumerate(
                    np.array(chunk_line_embds).tolist()):
                if args.verbose:
                    tf.logging.info(
                        "Line: {}".format(chunk_unencoded_lines[i]))
                    tf.logging.info(
                        "Embedding size: {}".format(len(line_embedding)))
                    snippet = ", ".join((str(x) for x in line_embedding[:3]))
                    tf.logging.info(
                        "Embedding: [{}, ...]\n".format(snippet))

                # Encode a hash for the string
                hash_object = hashlib.md5(
                    chunk_unencoded_lines[i].encode('utf-8'))

                # Add a row to the dataframe
                output_dict[hash_object.hexdigest()] = {
                    'line': chunk_unencoded_lines[i],
                    'line_embedding': line_embedding,
                    'response': chunck_unenc_resp[i]
                }
    return output_dict
Embed a collection of lines to an output dictionary.
Embed a collection of lines to an output dictionary.
Embed a collection of lines to an output dictionary .
def embed_lines(args, unencoded_lines, output_dict,
                unencoded_lines_resps=None):
    module = hub.Module(args.module_path, trainable=False)
    config = tf.ConfigProto(allow_soft_placement=True)
    with tf.Session(config=config) as session:
        session.run(
            [tf.global_variables_initializer(), tf.tables_initializer()])
        sp, embeddings, input_placeholder = None, None, None
        if args.use_sentence_piece:
            spm_path = session.run(module(signature="spm_path"))
            sp = spm.SentencePieceProcessor()
            sp.Load(spm_path)
            with tf.device('/gpu:0'):
                input_placeholder = tf.sparse_placeholder(
                    tf.int64, shape=[None, None])
                embeddings = module(inputs=dict(
                    values=input_placeholder.values,
                    indices=input_placeholder.indices,
                    dense_shape=input_placeholder.dense_shape
                )
                )
        size_of_chunk = 256
        assert len(unencoded_lines) == len(unencoded_lines_resps)
        all_id_chunks = get_id_chunks(
            range(len(unencoded_lines)), size_of_chunk)
        max_iter = len(unencoded_lines) // size_of_chunk
        for id_chunk in tqdm(all_id_chunks, total=max_iter):
            chunk_unencoded_lines = [unencoded_lines[x] for x in id_chunk]
            chunck_unenc_resp = [unencoded_lines_resps[x] for x in id_chunk]
            if args.use_sentence_piece:
                values, indices, dense_shape = process_to_IDs_in_sparse_format(
                    sp=sp, sentences=chunk_unencoded_lines)
                with tf.device('/gpu:0'):
                    chunk_line_embds = session.run(
                        embeddings,
                        feed_dict={
                            input_placeholder.values: values,
                            input_placeholder.indices: indices,
                            input_placeholder.dense_shape: dense_shape
                        }
                    )
            else:
                with tf.device('/gpu:0'):
                    chunk_line_embds = session.run(
                        module(chunk_unencoded_lines))
            for i, line_embedding in enumerate(
                    np.array(chunk_line_embds).tolist()):
                if args.verbose:
                    tf.logging.info(
                        "Line: {}".format(chunk_unencoded_lines[i]))
                    tf.logging.info(
                        "Embedding size: {}".format(len(line_embedding)))
                    snippet = ", ".join((str(x) for x in line_embedding[:3]))
                    tf.logging.info(
                        "Embedding: [{}, ...]\n".format(snippet))
                hash_object = hashlib.md5(
                    chunk_unencoded_lines[i].encode('utf-8'))
                output_dict[hash_object.hexdigest()] = {
                    'line': chunk_unencoded_lines[i],
                    'line_embedding': line_embedding,
                    'response': chunck_unenc_resp[i]
                }
    return output_dict
def embed_lines ( args , unencoded_lines , output_dict , unencoded_lines_resps = None ) : module = hub . Module ( args . module_path , trainable = False ) config = tf . ConfigProto ( allow_soft_placement = True ) with tf . Session ( config = config ) as session : session . run ( [ tf . global_variables_initializer ( ) , tf . tables_initializer ( ) ] ) sp , embeddings , input_placeholder = None , None , None if args . use_sentence_piece : spm_path = session . run ( module ( signature = "spm_path" ) ) sp = spm . SentencePieceProcessor ( ) sp . Load ( spm_path ) with tf . device ( '/gpu:0' ) : input_placeholder = tf . sparse_placeholder ( tf . int64 , shape = [ None , None ] ) embeddings = module ( inputs = dict ( values = input_placeholder . values , indices = input_placeholder . indices , dense_shape = input_placeholder . dense_shape ) ) size_of_chunk = 256 assert len ( unencoded_lines ) == len ( unencoded_lines_resps ) all_id_chunks = get_id_chunks ( range ( len ( unencoded_lines ) ) , size_of_chunk ) max_iter = len ( unencoded_lines ) // size_of_chunk for id_chunk in tqdm ( all_id_chunks , total = max_iter ) : chunk_unencoded_lines = [ unencoded_lines [ x ] for x in id_chunk ] chunck_unenc_resp = [ unencoded_lines_resps [ x ] for x in id_chunk ] if args . use_sentence_piece : values , indices , dense_shape = process_to_IDs_in_sparse_format ( sp = sp , sentences = chunk_unencoded_lines ) with tf . device ( '/gpu:0' ) : chunk_line_embds = session . run ( embeddings , feed_dict = { input_placeholder . values : values , input_placeholder . indices : indices , input_placeholder . dense_shape : dense_shape } ) else : with tf . device ( '/gpu:0' ) : chunk_line_embds = session . run ( module ( chunk_unencoded_lines ) ) for i , line_embedding in enumerate ( np . array ( chunk_line_embds ) . tolist ( ) ) : if args . verbose : tf . logging . info ( "Line: {}" . format ( chunk_unencoded_lines [ i ] ) ) tf . logging . info ( "Embedding size: {}" . format ( len ( line_embedding ) ) ) snippet = ", " . join ( ( str ( x ) for x in line_embedding [ : 3 ] ) ) tf . logging . info ( "Embedding: [{}, ...]\n" . format ( snippet ) ) hash_object = hashlib . md5 ( chunk_unencoded_lines [ i ] . encode ( 'utf-8' ) ) output_dict [ hash_object . hexdigest ( ) ] = { 'line' : chunk_unencoded_lines [ i ] , 'line_embedding' : line_embedding , 'response' : chunck_unenc_resp [ i ] } return output_dict
Embed a collection of lines to an output dictionary.
Embed a collection of lines to an output dictionary .
"""Embed a collection of lines to an output dictionary.""" # Import the Universal Sentence Encoder's TF Hub module # initialize the variables # spm_path now contains a path to the SentencePiece # model stored inside the TF-Hub module # build an input placeholder # size of chunk is how many lines will be encoded # with each pass of the model # ensure that every line has a response # get the chunk of lines and matching responses by list of ids # process unencoded lines to values and IDs in sparse format # run the session # hash the object into the full output dataframe # Encode a hash for the string # Add a row to the dataframe
args
unencoded_lines
output_dict
unencoded_lines_resps