jiant/models.py
def build_model(args, vocab, pretrained_embs, tasks):
    """
    Build model according to args
    Returns: model which has attributes set in it with the attrbutes.
    """

    # Build embeddings.
    if args.input_module == "gpt":
        # Note: incompatible with other embedders, but logic in preprocess.py
        # should prevent these from being enabled anyway.
        from .openai_transformer_lm.utils import OpenAIEmbedderModule

        log.info("Using OpenAI transformer model.")
        cove_layer = None
        # Here, this uses openAIEmbedder.
        embedder = OpenAIEmbedderModule(args)
        d_emb = embedder.get_output_dim()
    elif args.input_module.startswith("bert"):
        # Note: incompatible with other embedders, but logic in preprocess.py
        # should prevent these from being enabled anyway.
        from .bert.utils import BertEmbedderModule

        log.info(f"Using BERT model ({args.input_module}).")
        cove_layer = None
        # Set PYTORCH_PRETRAINED_BERT_CACHE environment variable to an existing
        # cache; see
        # https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/pytorch_pretrained_bert/file_utils.py  # noqa
        bert_cache_dir = os.getenv(
            "PYTORCH_PRETRAINED_BERT_CACHE", os.path.join(args.exp_dir, "bert_cache")
        )
        maybe_make_dir(bert_cache_dir)
        embedder = BertEmbedderModule(args, cache_dir=bert_cache_dir)
        d_emb = embedder.get_output_dim()
    else:
        # Default case, used for ELMo, CoVe, word embeddings, etc.
        d_emb, embedder, cove_layer = build_embeddings(args, vocab, tasks, pretrained_embs)
    d_sent_input = args.d_hid

    # d_sent_output is 0 for BERT
    sent_encoder, d_sent_output = build_sent_encoder(
        args, vocab, d_emb, tasks, embedder, cove_layer
    )

    # d_task_input is the input dimension of the task-specific module
    # set skip_emb = 1 if you want to concatenate the encoder input with encoder output to pass
    # into task specific module.
    #XXX skip_embs NEEDS to be 1 for BERT to work as it ought to
    d_task_input = d_sent_output + (args.skip_embs * d_emb)

    # Build model and classifiers
    model = MultiTaskModel(args, sent_encoder, vocab)
    build_task_modules(args, tasks, model, d_task_input, d_emb, embedder, vocab)
    model = model.cuda() if args.cuda >= 0 else model
    log.info("Model specification:")
    log.info(model)
    param_count = 0
    trainable_param_count = 0
    if args.list_params:
        log.info("Model parameters:")
    for name, param in model.named_parameters():
        param_count += np.prod(param.size())
        if param.requires_grad:
            trainable_param_count += np.prod(param.size())
            if args.list_params:
                log.info(
                    "\t%s: Trainable parameter, count %d with %s",
                    name,
                    np.prod(param.size()),
                    str(param.size()),
                )
        elif args.list_params:
            log.info(
                "\t%s: Non-trainable parameter, count %d with %s",
                name,
                np.prod(param.size()),
                str(param.size()),
            )
    log.info("Total number of parameters: {ct:d} ({ct:g})".format(ct=param_count))
    log.info("Number of trainable parameters: {ct:d} ({ct:g})".format(ct=trainable_param_count))
    return model

    Build model according to args
    Returns: model which has attributes set in it with the attrbutes.
    
Build model according to args
Returns: model which has attributes set in it with the attrbutes.
Build model according to args Returns : model which has attributes set in it with the attrbutes .
def build_model(args, vocab, pretrained_embs, tasks):
    if args.input_module == "gpt":
        from .openai_transformer_lm.utils import OpenAIEmbedderModule
        log.info("Using OpenAI transformer model.")
        cove_layer = None
        embedder = OpenAIEmbedderModule(args)
        d_emb = embedder.get_output_dim()
    elif args.input_module.startswith("bert"):
        from .bert.utils import BertEmbedderModule
        log.info(f"Using BERT model ({args.input_module}).")
        cove_layer = None
        bert_cache_dir = os.getenv(
            "PYTORCH_PRETRAINED_BERT_CACHE", os.path.join(args.exp_dir, "bert_cache")
        )
        maybe_make_dir(bert_cache_dir)
        embedder = BertEmbedderModule(args, cache_dir=bert_cache_dir)
        d_emb = embedder.get_output_dim()
    else:
        d_emb, embedder, cove_layer = build_embeddings(args, vocab, tasks, pretrained_embs)
    d_sent_input = args.d_hid
    sent_encoder, d_sent_output = build_sent_encoder(
        args, vocab, d_emb, tasks, embedder, cove_layer
    )
    d_task_input = d_sent_output + (args.skip_embs * d_emb)
    model = MultiTaskModel(args, sent_encoder, vocab)
    build_task_modules(args, tasks, model, d_task_input, d_emb, embedder, vocab)
    model = model.cuda() if args.cuda >= 0 else model
    log.info("Model specification:")
    log.info(model)
    param_count = 0
    trainable_param_count = 0
    if args.list_params:
        log.info("Model parameters:")
    for name, param in model.named_parameters():
        param_count += np.prod(param.size())
        if param.requires_grad:
            trainable_param_count += np.prod(param.size())
            if args.list_params:
                log.info(
                    "\t%s: Trainable parameter, count %d with %s",
                    name,
                    np.prod(param.size()),
                    str(param.size()),
                )
        elif args.list_params:
            log.info(
                "\t%s: Non-trainable parameter, count %d with %s",
                name,
                np.prod(param.size()),
                str(param.size()),
            )
    log.info("Total number of parameters: {ct:d} ({ct:g})".format(ct=param_count))
    log.info("Number of trainable parameters: {ct:d} ({ct:g})".format(ct=trainable_param_count))
    return model
def build_model ( args , vocab , pretrained_embs , tasks ) : if args . input_module == "gpt" : from . openai_transformer_lm . utils import OpenAIEmbedderModule log . info ( "Using OpenAI transformer model." ) cove_layer = None embedder = OpenAIEmbedderModule ( args ) d_emb = embedder . get_output_dim ( ) elif args . input_module . startswith ( "bert" ) : from . bert . utils import BertEmbedderModule log . info ( f"Using BERT model ({args.input_module})." ) cove_layer = None bert_cache_dir = os . getenv ( "PYTORCH_PRETRAINED_BERT_CACHE" , os . path . join ( args . exp_dir , "bert_cache" ) ) maybe_make_dir ( bert_cache_dir ) embedder = BertEmbedderModule ( args , cache_dir = bert_cache_dir ) d_emb = embedder . get_output_dim ( ) else : d_emb , embedder , cove_layer = build_embeddings ( args , vocab , tasks , pretrained_embs ) d_sent_input = args . d_hid sent_encoder , d_sent_output = build_sent_encoder ( args , vocab , d_emb , tasks , embedder , cove_layer ) d_task_input = d_sent_output + ( args . skip_embs * d_emb ) model = MultiTaskModel ( args , sent_encoder , vocab ) build_task_modules ( args , tasks , model , d_task_input , d_emb , embedder , vocab ) model = model . cuda ( ) if args . cuda >= 0 else model log . info ( "Model specification:" ) log . info ( model ) param_count = 0 trainable_param_count = 0 if args . list_params : log . info ( "Model parameters:" ) for name , param in model . named_parameters ( ) : param_count += np . prod ( param . size ( ) ) if param . requires_grad : trainable_param_count += np . prod ( param . size ( ) ) if args . list_params : log . info ( "\t%s: Trainable parameter, count %d with %s" , name , np . prod ( param . size ( ) ) , str ( param . size ( ) ) , ) elif args . list_params : log . info ( "\t%s: Non-trainable parameter, count %d with %s" , name , np . prod ( param . size ( ) ) , str ( param . size ( ) ) , ) log . info ( "Total number of parameters: {ct:d} ({ct:g})" . format ( ct = param_count ) ) log . info ( "Number of trainable parameters: {ct:d} ({ct:g})" . format ( ct = trainable_param_count ) ) return model
Build model according to args
Returns: model which has attributes set in it with the attrbutes.
Build model according to args Returns : model which has attributes set in it with the attrbutes .
"""
    Build model according to args
    Returns: model which has attributes set in it with the attrbutes.
    """ # Build embeddings. # Note: incompatible with other embedders, but logic in preprocess.py # should prevent these from being enabled anyway. # Here, this uses openAIEmbedder. # Note: incompatible with other embedders, but logic in preprocess.py # should prevent these from being enabled anyway. # Set PYTORCH_PRETRAINED_BERT_CACHE environment variable to an existing # cache; see # https://github.com/huggingface/pytorch-pretrained-BERT/blob/master/pytorch_pretrained_bert/file_utils.py  # noqa # Default case, used for ELMo, CoVe, word embeddings, etc. # d_sent_output is 0 for BERT # d_task_input is the input dimension of the task-specific module # set skip_emb = 1 if you want to concatenate the encoder input with encoder output to pass # into task specific module. #XXX skip_embs NEEDS to be 1 for BERT to work as it ought to # Build model and classifiers
args
vocab
pretrained_embs
tasks